Remove uncessary columns and invalid rows from csv format data
970807
def raw_filter raw_data, opts={}
      raw_header = raw_data[header_row].map.with_index do |h, idx|
        is_valid = valid_header?(h)
        {
          idx: idx,
          is_remove: !is_valid,
          is_required: require_header?(h),
          name: is_valid && h.gsub(/\s*/, '').gsub(/!/, '').downcase
        }
      end

      rows = raw_data[(header_row + 1)..-1].map.with_index do |raw, rdx|
        row = {}.with_indifferent_access
        raw_header.each do |h|
          val = raw[h[:idx]]
          next if h[:is_remove]
          if h[:is_required] && val.blank?
            row = {}
            break
          end

          row[h[:name]] = standardize_cell_val val
        end

        next if row.values.all?(&:blank?)

        row[:id] = rdx + 1 if find_columns.include?(:id) && row[:id].blank?
        row
      end.compact
         .reject(&:blank?)
      rows = rows.select do |row|
        if required_columns.present?
          required_columns.all? {|col| row[col].present? }
        else
          true
        end
      end

      rows
    end
460587
def csv_to_pickle(path=ROOT / "raw", clean=False):
    """"""

    for file in os.listdir(path):
        base, ext = os.path.splitext(file)
        if ext != ".csv":
            continue

        LOG(f"converting {file} to pickle")
        df = pd.read_csv(path / file, low_memory=True)
        WRITE_DF(df, path / (base + "." + FILE_EXTENSION), **WRITE_DF_OPTS)
        if clean:
            os.remove(path / file)
            LOG(f"removed {file}")
832965
def _ReadCSV(self, file_name, cols, required, deprecated):
    """"""
    contents = self._GetUtf8Contents(file_name)
    if not contents:
      return

    eol_checker = util.EndOfLineChecker(StringIO.StringIO(contents),
                                   file_name, self._problems)
    reader = csv.reader(eol_checker)  # Use excel dialect

    header = next(reader)
    header = map(lambda x: x.strip(), header)  # trim any whitespace
    header_occurrences = util.defaultdict(lambda: 0)
    for column_header in header:
      header_occurrences[column_header] += 1

    for name, count in header_occurrences.items():
      if count > 1:
        self._problems.DuplicateColumn(
            header=name,
            file_name=file_name,
            count=count)

    # check for unrecognized columns, which are often misspellings
    header_context = (file_name, 1, [''] * len(header), header)
    valid_cols = cols + [deprecated_name for (deprecated_name, _) in deprecated]
    unknown_cols = set(header).difference(set(valid_cols))
    for col in unknown_cols:
      # this is provided in order to create a nice colored list of
      # columns in the validator output
      self._problems.UnrecognizedColumn(file_name, col, header_context)

    # check for missing required columns
    col_index = [-1] * len(cols)
    for i in range(len(cols)):
      if cols[i] in header:
        col_index[i] = header.index(cols[i])
      elif cols[i] in required:
        self._problems.MissingColumn(file_name, cols[i], header_context)

    # check for deprecated columns
    for (deprecated_name, new_name) in deprecated:
      if deprecated_name in header:
        self._problems.DeprecatedColumn(file_name, deprecated_name, new_name,
                                        header_context)

    row_num = 1
    for row in reader:
      row_num += 1
      if len(row) == 0:  # skip extra empty lines in file
        continue

      if len(row) > len(header):
        self._problems.OtherProblem('Found too many cells (commas) in line '
                                    '%d of file "%s".  Every row in the file '
                                    'should have the same number of cells as '
                                    'the header (first line) does.' %
                                    (row_num, file_name), (file_name, row_num),
                                    type=problems.TYPE_WARNING)

      if len(row) < len(header):
        self._problems.OtherProblem('Found missing cells (commas) in line '
                                    '%d of file "%s".  Every row in the file '
                                    'should have the same number of cells as '
                                    'the header (first line) does.' %
                                    (row_num, file_name), (file_name, row_num),
                                    type=problems.TYPE_WARNING)

      result = [None] * len(cols)
      unicode_error_columns = []  # A list of column numbers with an error
      for i in range(len(cols)):
        ci = col_index[i]
        if ci >= 0:
          if len(row) <= ci:  # handle short CSV rows
            result[i] = u''
          else:
            try:
              result[i] = row[ci].decode('utf-8').strip()
            except UnicodeDecodeError:
              # Replace all invalid characters with
              # REPLACEMENT CHARACTER (U+FFFD)
              result[i] = codecs.getdecoder("utf8")(row[ci],
                                                    errors="replace")[0].strip()
              unicode_error_columns.append(i)

      for i in unicode_error_columns:
        self._problems.InvalidValue(cols[i], result[i],
                                    'Unicode error',
                                    (file_name, row_num, result, cols))
      yield (result, row_num, cols)
1186195
def to_arrays(data, columns, coerce_float=False, dtype=None):
    
    if isinstance(data, ABCDataFrame):
        if columns is not None:
            arrays = [data._ixs(i, axis=1).values
                      for i, col in enumerate(data.columns) if col in columns]
        else:
            columns = data.columns
            arrays = [data._ixs(i, axis=1).values for i in range(len(columns))]

        return arrays, columns

    if not len(data):
        if isinstance(data, np.ndarray):
            columns = data.dtype.names
            if columns is not None:
                return [[]] * len(columns), columns
        return [], []  # columns if columns is not None else []
    if isinstance(data[0], (list, tuple)):
        return _list_to_arrays(data, columns, coerce_float=coerce_float,
                               dtype=dtype)
    elif isinstance(data[0], abc.Mapping):
        return _list_of_dict_to_arrays(data, columns,
                                       coerce_float=coerce_float, dtype=dtype)
    elif isinstance(data[0], ABCSeries):
        return _list_of_series_to_arrays(data, columns,
                                         coerce_float=coerce_float,
                                         dtype=dtype)
    elif isinstance(data[0], Categorical):
        if columns is None:
            columns = ibase.default_index(len(data))
        return data, columns
    elif (isinstance(data, (np.ndarray, ABCSeries, Index)) and
          data.dtype.names is not None):

        columns = list(data.dtype.names)
        arrays = [data[k] for k in columns]
        return arrays, columns
    else:
        # last ditch effort
        data = lmap(tuple, data)
        return _list_to_arrays(data, columns, coerce_float=coerce_float,
                               dtype=dtype)
1255507
def clean_df(df, header=None, **read_csv_kwargs):
    
    df = read_csv(df, header=header, **read_csv_kwargs)
    df = df.fillna(' ')
    for col in df.columns:
        df[col] = df[col].apply(unicode2ascii)
    return df
162246
def clean_column_names(df: DataFrame) -> DataFrame:
    
    f = df.copy()
    f.columns = [col.strip() for col in f.columns]
    return f
572347
def _clean_data(self, str_value, file_data, obj_value):
        """"""
        str_value = str_value or None
        obj_value = obj_value or None
        return (str_value, None, obj_value)
1138332
def clean_win_tsv(filepath=os.path.join(DATA_PATH, 'Products.txt'),
                  index_col=False, sep='\t', lineterminator='\r', error_bad_lines=False, **kwargs):
    
    df = pd.read_csv(filepath, index_col=index_col, sep=sep, lineterminator=lineterminator,
                     error_bad_lines=error_bad_lines, **kwargs)
    index_col = df.columns[0]
    original_len = len(df)
    if df[index_col].values[-1] == '\n':
        df.iloc[-1, 0] = np.nan
        original_len = len(df) - 1
    df.dropna(how='all', inplace=True)
    df[index_col] = df[index_col].str.strip().apply(lambda x: x if x else str(INT_MIN)).astype(int)
    df = df[~(df[index_col] == INT_NAN)]
    df.set_index(index_col, inplace=True)
    if len(df) != original_len:
        logger.warning(('Loaded {} rows from tsv. Original file, "{}", contained {} seemingly valid lines.' +
                        'Index column: {}').format(len(df), original_len, filepath, index_col))
    return df
1374146
def _DataClean_(fieldVal):
    

    fieldValNew = fieldVal

    fieldValNew = fieldValNew.upper()

    fieldValNew = fieldValNew.strip()

    fieldValNew = re.sub("[\s\n\t]+", " ", fieldValNew)

    return fieldValNew
1319523
function () {
		var self = this,
			options = this.options,
			csv = options.csv,
			columns = this.columns,
			startRow = options.startRow || 0,
			endRow = options.endRow || Number.MAX_VALUE,
			startColumn = options.startColumn || 0,
			endColumn = options.endColumn || Number.MAX_VALUE,
			lines,
			activeRowNo = 0;
			
		if (csv) {
			
			lines = csv
				.replace(/\r\n/g, "\n") // Unix
				.replace(/\r/g, "\n") // Mac
				.split(options.lineDelimiter || "\n");
			
			each(lines, function (line, rowNo) {
				var trimmed = self.trim(line),
					isComment = trimmed.indexOf('#') === 0,
					isBlank = trimmed === '',
					items;
				
				if (rowNo >= startRow && rowNo <= endRow && !isComment && !isBlank) {
					items = line.split(options.itemDelimiter || ',');
					each(items, function (item, colNo) {
						if (colNo >= startColumn && colNo <= endColumn) {
							if (!columns[colNo - startColumn]) {
								columns[colNo - startColumn] = [];					
							}
							
							columns[colNo - startColumn][activeRowNo] = item;
						}
					});
					activeRowNo += 1;
				}
			});

			this.dataFound();
		}
	}
1291926
def __rm_names_on_csv_cols(d):
        
        _names = []
        _data = []
        for name, data in d.items():
            _names.append(name)
            _data.append(data)
        return _names, _data
261020
function processCsv (csvContent) {
  UNSAFE_SYMBOLS.forEach(function (symbolParams) {
    csvContent = csvContent.replace(symbolParams.pattern, symbolParams.replace);
  });
  return csvContent;
}
364843
def clean(cls, cnpj):
        
        """
        if isinstance(cnpj, six.string_types):
            cnpj = int(re.sub('[^0-9]', '', cnpj))

        return '{0:014d}'.format(cnpj)
1052561
def csv_format(csv_data, c_headers=None, r_headers=None, rows=None, **kwargs):
    
    result = None
    c_headers = [] if c_headers is None else c_headers
    r_headers = [] if r_headers is None else r_headers
    rows = [] if rows is None else rows

    result_format = kwargs.get('result_format', ARRAY_RAW_FORMAT)

    # DICT FORMAT
    if result_format == DICT_FORMAT:
        result = csv_dict_format(csv_data, c_headers, r_headers)

    # ARRAY_RAW_FORMAT
    elif result_format == ARRAY_RAW_FORMAT:
        result = rows

    # ARRAY_CLEAN_FORMAT
    elif result_format == ARRAY_CLEAN_FORMAT:
        result = csv_array_clean_format(csv_data, c_headers, r_headers)

    else:
        result = None

    # DEFAULT
    if result and result_format < DICT_FORMAT:
        result = [result]

    return result
527246
def strip_columns(tab):
    """"""
    for colname in tab.colnames:
        if tab[colname].dtype.kind in ['S', 'U']:
            tab[colname] = np.core.defchararray.strip(tab[colname])
554775
def _sanitize_column(self, key, value, **kwargs):
        
        def sp_maker(x, index=None):
            return SparseArray(x, index=index,
                               fill_value=self._default_fill_value,
                               kind=self._default_kind)
        if isinstance(value, SparseSeries):
            clean = value.reindex(self.index).as_sparse_array(
                fill_value=self._default_fill_value, kind=self._default_kind)

        elif isinstance(value, SparseArray):
            if len(value) != len(self.index):
                raise ValueError('Length of values does not match '
                                 'length of index')
            clean = value

        elif hasattr(value, '__iter__'):
            if isinstance(value, Series):
                clean = value.reindex(self.index)
                if not isinstance(value, SparseSeries):
                    clean = sp_maker(clean)
            else:
                if len(value) != len(self.index):
                    raise ValueError('Length of values does not match '
                                     'length of index')
                clean = sp_maker(value)

        # Scalar
        else:
            clean = sp_maker(value, self.index)

        # always return a SparseArray!
        return clean
1105463
def clean(self, data):
        
        try:
            value = data[self.column_name]
        except KeyError:
            raise KeyError("Column '%s' not found in dataset. Available "
                           "columns are: %s" % (self.column_name, list(data)))

        # If ValueError is raised here, import_obj() will handle it
        value = self.widget.clean(value, row=data)

        if value in self.empty_values and self.default != NOT_PROVIDED:
            if callable(self.default):
                return self.default()
            return self.default

        return value