Convert Excel files to LiPD files. LiPD data is returned directly from this function.

    | Example
    | 1: lipd.readExcel()
    | 2: D = lipd.excel()

    :return dict _d: Metadata
548150
def excel():
    
    global files, cwd, settings
    _d = {}
    # Turn off verbose. We don't want to clutter the console with extra reading/writing output statements
    settings["verbose"] = False
    # Find excel files
    print("Found " + str(len(files[".xls"])) + " Excel files")
    logger_start.info("found excel files: {}".format(len(files[".xls"])))
    # Start the clock
    start = clock()
    # Loop for each excel file
    for file in files[".xls"]:
        # Convert excel file to LiPD
        dsn = excel_main(file)
        try:
            # Read the new LiPD file back in, to get fixes, inferred calculations, updates, etc.
            _d[dsn] = readLipd(os.path.join(file["dir"], dsn + ".lpd"))
            # Write the modified LiPD file back out again.
            writeLipd(_d[dsn], cwd)
        except Exception as e:
            logger_start.error("excel: Unable to read new LiPD file, {}".format(e))
            print("Error: Unable to read new LiPD file: {}, {}".format(dsn, e))
    # Time!
    end = clock()
    logger_benchmark.info(log_benchmark("excel", start, end))
    # Start printing stuff again.
    settings["verbose"] = True
    return _d
272405
def find_files():
    
    _dir = os.getcwd()
    _files = os.listdir()
    # Look for a jsonld file
    for _file in _files:
        if _file.endswith(".jsonld"):
            return os.getcwd()
    # No jsonld file found, try to chdir into "bag" (LiPD v1.3)
    if "bag" in _files:
        os.chdir("bag")
        _dir = find_files()
    # No "bag" dir. Try chdir into whatever dataset name dir we find (< LiPD v1.2)
    else:
        for _file in _files:
            if os.path.isdir(_file):
                os.chdir(_file)
                _dir = find_files()
    return _dir
1321768
def lipd_to_df(metadata, csvs):
    
    dfs = {}
    logger_dataframes.info("enter lipd_to_df")

    # Flatten the dictionary, but ignore the chron data items
    dict_in_dotted = {}
    logger_dataframes.info("enter dot_notation")
    _dotnotation_for_nested_dictionary(metadata, '', dict_in_dotted)
    dict_in_dotted = collections.OrderedDict(sorted(dict_in_dotted.items()))

    # Create one data frame for metadata items
    dfs["metadata"] = pd.DataFrame(list(dict_in_dotted.items()), columns=["Key", "Value"])

    # Create data frames for paleo data and chron data items. This does not use LiPD data, it uses the csv data
    dfs.update(_get_dfs(csvs))

    return dfs
300470
def read_json(self):
        
        with reading_ancillary_files(self):
            metadata = super(GenericLayerMetadata, self).read_json()

        return metadata
971209
def get_dataset(self, dataset_id, ds_info, xslice=slice(None), yslice=slice(None)):
        """"""
        var_path = ds_info.get('file_key', '{}'.format(dataset_id.name))
        metadata = self.get_metadata(dataset_id, ds_info)
        shape = metadata['shape']
        file_shape = self[var_path + '/shape']
        if isinstance(shape, tuple) and len(shape) == 2:
            # 2D array
            if xslice.start is not None:
                shape = (shape[0], xslice.stop - xslice.start)
            if yslice.start is not None:
                shape = (yslice.stop - yslice.start, shape[1])
        elif isinstance(shape, tuple) and len(shape) == 1 and yslice.start is not None:
            shape = ((yslice.stop - yslice.start) / yslice.step,)
        metadata['shape'] = shape

        valid_min = self[var_path + '/attr/valid_min']
        valid_max = self[var_path + '/attr/valid_max']
        # no need to check fill value since we are using valid min/max
        scale_factor = self.get(var_path + '/attr/scale_factor')
        add_offset = self.get(var_path + '/attr/add_offset')

        if isinstance(file_shape, tuple) and len(file_shape) == 3:
            data = self[var_path][0, yslice, xslice]
        elif isinstance(file_shape, tuple) and len(file_shape) == 2:
            data = self[var_path][yslice, xslice]
        elif isinstance(file_shape, tuple) and len(file_shape) == 1:
            data = self[var_path][yslice]
        else:
            data = self[var_path]
        data = data.where((data >= valid_min) & (data <= valid_max))
        if scale_factor is not None:
            data = data * scale_factor + add_offset

        if ds_info.get('cloud_clear', False):
            # clear-sky if bit 15-16 are 00
            clear_sky_mask = (self['l2p_flags'][0] & 0b1100000000000000) != 0
            data = data.where(~clear_sky_mask)

        data.attrs.update(metadata)
        return data
716854
def to_data(force_level = :none)
      cells       = [] 
      merges      = []
      col_widths  = {}
      row_heights = {}

      css.reset(force_level)

      tb  = doc.css('table').first 

      # ignoring specified formats for anything other than table tr td/th
      tb_format = css.format_from_node(tb) 

      row = 0
      doc.css('tr').each do |tr|
        tr_format = tb_format.merge(@css.format_from_node(tr))

        increment = true
        tr.css('td, th').each_with_index do |td, col|
           
          # TODO Do we really need rowheight and colwidth now that there
          # is css parsing?
          rowheight = td.attributes["rowheight"]
          colwidth  = td.attributes["colwidth"]
          rowspan   = td.attributes["rowspan"]
          colspan   = td.attributes["colspan"]

          rowheight &&= rowheight.value.to_i
          colwidth  &&= colwidth.value.to_i
          rowspan   &&= rowspan.value.to_i
          colspan   &&= colspan.value.to_i

          add_td_to_cells(row, col, td, tr_format, cells)
          if colspan
            (1..colspan-1).each {|t| 
              add_td_to_cells(row, col+t, td, tr_format, cells)
            }
          end
          if rowspan
            (1..rowspan-2).each {|t| 
              add_td_to_cells(row+t, col, td, tr_format, cells)
            }
            increment = false
          end
          if colspan || rowspan
            merges << [
              row, col, row + (rowspan || 2)-2, col + (colspan || 1)-1
            ]
          end
        end

        row += 1 if increment
      end

      puts cells.inspect
      { cells: cells, merges: { 0 => merges } }
    end
309855
def readLipd(usr_path=""):
    
    global cwd, settings, files
    if settings["verbose"]:
        __disclaimer(opt="update")
    start = clock()
    files[".lpd"] = []
    __read(usr_path, ".lpd")
    _d = __read_lipd_contents()
    end = clock()
    logger_benchmark.info(log_benchmark("readLipd", start, end))
    return _d
834379
def read_hdf(filename: PathLike, key: str) -> AnnData:
    
    with h5py.File(filename, 'r') as f:
        # the following is necessary in Python 3, because only
        # a view and not a list is returned
        keys = [k for k in f.keys()]
        if key == '':
            raise ValueError((
                'The file {} stores the following sheets:\n{}\n'
                'Call read/read_hdf5 with one of them.'
            ).format(filename, keys))
        # read array
        X = f[key][()]
        # try to find row and column names
        rows_cols = [{}, {}]
        for iname, name in enumerate(['row_names', 'col_names']):
            if name in keys:
                rows_cols[iname][name] = f[name][()]
    adata = AnnData(X, rows_cols[0], rows_cols[1], dtype=X.dtype.name)
    return adata
731428
def read_data( self, step, filename = None ):
        """"""
        filename = get_default( filename, self.filename )

        out = {}

        fd = open( self.filename, 'r' )
        while 1:
            line = skip_read_line(fd, no_eof=True).split()
            if line[0] == 'POINT_DATA':
                break

        n_nod = int(line[1])

        while 1:
            line = skip_read_line(fd)
            if not line:
                break

            line = line.split()

            if line[0] == 'SCALARS':
                name, dtype, nc = line[1:]
                assert_(int(nc) == 1)
                fd.readline() # skip lookup table line
                
                data = nm.zeros((n_nod,), dtype=nm.float64)
                ii = 0
                while ii < n_nod:
                    data[ii] = float(fd.readline())
                    ii += 1

                out[name] = Struct( name = name,
                                    mode = 'vertex',
                                    data = data,
                                    dofs = None )

            elif line[0] == 'VECTORS':
                name, dtype = line[1:]
                data = []
                ii = 0
                while ii < n_nod:
                    data.append([float(val) for val in fd.readline().split()])
                    ii += 1

                out[name] = Struct( name = name,
                                    mode = 'vertex',
                                    data = nm.array(data, dtype=nm.float64),
                                    dofs = None )

            elif line[0] == 'CELL_DATA':
                break

            line = fd.readline()

        fd.close()

        return out
527043
def get_filenames_in_lipd(path, name=""):
    
    _filenames = []
    try:
        # in the top level, list all files and skip the "data" directory
        _top = [os.path.join(name, f) for f in os.listdir(path) if f != "data"]
        # in the data directory, list all files
        _dir_data = [os.path.join(name, "data", f) for f in os.listdir(os.path.join(path, "data"))]
        # combine the two lists
        _filenames = _top + _dir_data
    except Exception:
        pass
    return _filenames
1369439
def csv_to_dict(file_name, file_location):
    
    file = __os.path.join(file_location, file_name)
    try:
        csv_read = open(file, "r")
    except Exception as e:
        LOGGER.critical('Function csv_to_dict Error {error} ignoring any errors'.format(error=e))
        print('Error {error} ignoring any errors'.format(error=e))
        csv_read = open(file, "r", errors='ignore')
    data_row = __csv.DictReader(csv_read, dialect="excel")
    dict_key = 1
    temp_dict = dict()
    for row in data_row:
        temp_dict[dict_key] = row
        dict_key += 1
    csv_read.close()
    return temp_dict
157781
def readExcel(usr_path=""):
    
    global cwd, files
    start = clock()
    files[".xls"] = []
    __read(usr_path, ".xls")
    end = clock()
    logger_benchmark.info(log_benchmark("readExcel", start, end))
    return cwd
15415
def database_to_excel(engine, excel_file_path):
    
    from sqlalchemy import MetaData, select

    metadata = MetaData()
    metadata.reflect(engine)

    writer = pd.ExcelWriter(excel_file_path)
    for table in metadata.tables.values():
        sql = select([table])
        df = pd.read_sql(sql, engine)
        df.to_excel(writer, table.name, index=False)

    writer.save()
259735
def get_data(self, data_format):
        
        if data_format not in self.output_formats:
            raise Exception("Output format {0} not available with this class. Available formats are {1}.".format(data_format, self.output_formats))
        data_converter = getattr(self, "to_" + data_format)
        return data_converter()
513259
def _create_raw_data(self):
        
        result = {}
        for section in self.get_sections():
            result[section.get('id')] = 'all'
        return result
1213752
def _get_header_keys(row):
    
    # Swap out NOAA keys for LiPD keys
    for idx, key in enumerate(row):
        key_low = key.value.lower()

        # Simple case: Nothing fancy here, just map to the LiPD key counterpart.
        if key_low in EXCEL_LIPD_MAP_FLAT:
            row[idx] = EXCEL_LIPD_MAP_FLAT[key_low]

        # Nested data case: Check if this is a calibration, interpretation, or some other data that needs to be nested.
        # elif key_low:
        #     pass

        # Unknown key case: Store the key as-is because we don't have a LiPD mapping for it.
        else:
            try:
                row[idx] = key.value
            except AttributeError as e:
                logger_excel.warn("excel_main: get_header_keys: unknown header key, unable to add: {}".format(e))

    # Since we took a whole row of cells, we have to drop off the empty cells at the end of the row.
    header_keys = _rm_cells_reverse(row)
    return header_keys
114442
def parse_meta(self, selected_meta_data):
        
        # reads all meta data files
        files = self._get_files("meta", self.path)
        df = pd.DataFrame()
        print("Parsing the metadata files...")
        for f in tqdm(files):
            data = self.parse_single_meta(f, selected_meta_data)
            if data is not None:
                df = pd.concat([df, data], axis=0)
        df.index = df['sample']
        #
        # df = df.drop('sample', 1)  # 1 for the columns
        return df