Read in a calibration file and parse into a dictionary.
969039
def read_calib_file(filepath):
    """"""
    data = {}

    with open(filepath, 'r') as f:
        for line in f.readlines():
            key, value = line.split(':', 1)
            # The only non-float values in these files are dates, which
            # we don't care about anyway
            try:
                data[key] = np.array([float(x) for x in value.split()])
            except ValueError:
                pass

    return data
1250971
def _calibrate(self, data):
        """"""
        idx = self.mda['calibration_parameters']['indices']
        val = self.mda['calibration_parameters']['values']
        data.data = da.where(data.data == 0, np.nan, data.data)
        ddata = data.data.map_blocks(np.interp, idx, val, dtype=val.dtype)
        res = xr.DataArray(ddata,
                           dims=data.dims, attrs=data.attrs,
                           coords=data.coords)
        res = res.clip(min=0)
        units = {'percent': '%'}
        unit = self.mda['calibration_parameters'][b'_UNIT']
        res.attrs['units'] = units.get(unit, unit)
        return res
1040547
def _parse_from_file(self, filepath, fname,
                         dependencies, recursive, greedy):
        """"""
        #Now that we have the file contents, we can parse them using the parsers
        string = self.tramp.read(filepath)
        pmodules = self.modulep.parse(string, self, filepath=filepath)
        file_mtime = self._get_mod_mtime(filepath)

        for module in pmodules:
            module.change_time = file_mtime
            self.modules[module.name.lower()] = module
            self._modulefiles[fname].append(module.name.lower())

        pprograms = self.modulep.parse(string, self, False)
        for program in pprograms:
            program.change_time = file_mtime
            self.programs[program.name.lower()] = program
            self._programfiles[fname].append(program.name.lower())

        #There may be xml files for the docstrings that also need to be parsed.
        self._parse_docstrings(filepath)
        
        return (pmodules, pprograms)
173583
def parse_file(filename):
        
        poscar_read = False
        poscar_string = []
        dataset = []
        all_dataset = []
        # for holding any strings in input that are not Poscar
        # or VolumetricData (typically augmentation charges)
        all_dataset_aug = {}
        dim = None
        dimline = None
        read_dataset = False
        ngrid_pts = 0
        data_count = 0
        poscar = None
        with zopen(filename, "rt") as f:
            for line in f:
                original_line = line
                line = line.strip()
                if read_dataset:
                    toks = line.split()
                    for tok in toks:
                        if data_count < ngrid_pts:
                            # This complicated procedure is necessary because
                            # vasp outputs x as the fastest index, followed by y
                            # then z.
                            x = data_count % dim[0]
                            y = int(math.floor(data_count / dim[0])) % dim[1]
                            z = int(math.floor(data_count / dim[0] / dim[1]))
                            dataset[x, y, z] = float(tok)
                            data_count += 1
                    if data_count >= ngrid_pts:
                        read_dataset = False
                        data_count = 0
                        all_dataset.append(dataset)
                elif not poscar_read:
                    if line != "" or len(poscar_string) == 0:
                        poscar_string.append(line)
                    elif line == "":
                        poscar = Poscar.from_string("\n".join(poscar_string))
                        poscar_read = True
                elif not dim:
                    dim = [int(i) for i in line.split()]
                    ngrid_pts = dim[0] * dim[1] * dim[2]
                    dimline = line
                    read_dataset = True
                    dataset = np.zeros(dim)
                elif line == dimline:
                    # when line == dimline, expect volumetric data to follow
                    # so set read_dataset to True
                    read_dataset = True
                    dataset = np.zeros(dim)
                else:
                    # store any extra lines that were not part of the
                    # volumetric data so we know which set of data the extra
                    # lines are associated with
                    key = len(all_dataset) - 1
                    if key not in all_dataset_aug:
                        all_dataset_aug[key] = []
                    all_dataset_aug[key].append(original_line)
            if len(all_dataset) == 4:

                data = {"total": all_dataset[0], "diff_x": all_dataset[1],
                        "diff_y": all_dataset[2], "diff_z": all_dataset[3]}
                data_aug = {"total": all_dataset_aug.get(0, None),
                            "diff_x": all_dataset_aug.get(1, None),
                            "diff_y": all_dataset_aug.get(2, None),
                            "diff_z": all_dataset_aug.get(3, None)}

                # construct a "diff" dict for scalar-like magnetization density,
                # referenced to an arbitrary direction (using same method as
                # pymatgen.electronic_structure.core.Magmom, see
                # Magmom documentation for justification for this)
                # TODO: re-examine this, and also similar behavior in
                # Magmom - @mkhorton
                # TODO: does CHGCAR change with different SAXIS?
                diff_xyz = np.array([data["diff_x"], data["diff_y"],
                                     data["diff_z"]])
                diff_xyz = diff_xyz.reshape((3, dim[0] * dim[1] * dim[2]))
                ref_direction = np.array([1.01, 1.02, 1.03])
                ref_sign = np.sign(np.dot(ref_direction, diff_xyz))
                diff = np.multiply(np.linalg.norm(diff_xyz, axis=0), ref_sign)
                data["diff"] = diff.reshape((dim[0], dim[1], dim[2]))

            elif len(all_dataset) == 2:
                data = {"total": all_dataset[0], "diff": all_dataset[1]}
                data_aug = {"total": all_dataset_aug.get(0, None),
                            "diff": all_dataset_aug.get(1, None)}
            else:
                data = {"total": all_dataset[0]}
                data_aug = {"total": all_dataset_aug.get(0, None)}
            return poscar, data, data_aug
331543
def decode_from_file(estimator,
                     filename,
                     hparams,
                     decode_hp,
                     decode_to_file=None,
                     checkpoint_path=None):
  """"""
  if not decode_hp.batch_size:
    decode_hp.batch_size = 32
    tf.logging.info(
        "decode_hp.batch_size not specified; default=%d" % decode_hp.batch_size)

  # Inputs vocabulary is set to targets if there are no inputs in the problem,
  # e.g., for language models where the inputs are just a prefix of targets.
  p_hp = hparams.problem_hparams
  has_input = "inputs" in p_hp.vocabulary
  inputs_vocab_key = "inputs" if has_input else "targets"
  inputs_vocab = p_hp.vocabulary[inputs_vocab_key]
  targets_vocab = p_hp.vocabulary["targets"]
  problem_name = FLAGS.problem
  filename = _add_shard_to_filename(filename, decode_hp)
  tf.logging.info("Performing decoding from file (%s)." % filename)
  if has_input:
    sorted_inputs, sorted_keys = _get_sorted_inputs(
        filename, decode_hp.delimiter)
  else:
    sorted_inputs = _get_language_modeling_inputs(
        filename, decode_hp.delimiter, repeat=decode_hp.num_decodes)
    sorted_keys = range(len(sorted_inputs))
  num_sentences = len(sorted_inputs)
  num_decode_batches = (num_sentences - 1) // decode_hp.batch_size + 1

  if estimator.config.use_tpu:
    length = getattr(hparams, "length", 0) or hparams.max_length
    batch_ids = []
    for line in sorted_inputs:
      if has_input:
        ids = inputs_vocab.encode(line.strip()) + [1]
      else:
        ids = targets_vocab.encode(line)
      if len(ids) < length:
        ids.extend([0] * (length - len(ids)))
      else:
        ids = ids[:length]
      batch_ids.append(ids)
    np_ids = np.array(batch_ids, dtype=np.int32)
    def input_fn(params):
      batch_size = params["batch_size"]
      dataset = tf.data.Dataset.from_tensor_slices({"inputs": np_ids})
      dataset = dataset.map(
          lambda ex: {"inputs": tf.reshape(ex["inputs"], (length, 1, 1))})
      dataset = dataset.batch(batch_size)
      return dataset
  else:
    def input_fn():
      input_gen = _decode_batch_input_fn(
          num_decode_batches, sorted_inputs,
          inputs_vocab, decode_hp.batch_size,
          decode_hp.max_input_size,
          task_id=decode_hp.multiproblem_task_id, has_input=has_input)
      gen_fn = make_input_fn_from_generator(input_gen)
      example = gen_fn()
      return _decode_input_tensor_to_features_dict(example, hparams)
  decodes = []
  result_iter = estimator.predict(input_fn, checkpoint_path=checkpoint_path)

  start_time = time.time()
  total_time_per_step = 0
  total_cnt = 0

  def timer(gen):
    while True:
      try:
        start_time = time.time()
        item = next(gen)
        elapsed_time = time.time() - start_time
        yield elapsed_time, item
      except StopIteration:
        break

  for elapsed_time, result in timer(result_iter):
    if decode_hp.return_beams:
      beam_decodes = []
      beam_scores = []
      output_beams = np.split(result["outputs"], decode_hp.beam_size, axis=0)
      scores = None
      if "scores" in result:
        if np.isscalar(result["scores"]):
          result["scores"] = result["scores"].reshape(1)
        scores = np.split(result["scores"], decode_hp.beam_size, axis=0)
      for k, beam in enumerate(output_beams):
        tf.logging.info("BEAM %d:" % k)
        score = scores and scores[k]
        _, decoded_outputs, _ = log_decode_results(
            result["inputs"],
            beam,
            problem_name,
            None,
            inputs_vocab,
            targets_vocab,
            log_results=decode_hp.log_results,
            skip_eos_postprocess=decode_hp.skip_eos_postprocess)
        beam_decodes.append(decoded_outputs)
        if decode_hp.write_beam_scores:
          beam_scores.append(score)
      if decode_hp.write_beam_scores:
        decodes.append("\t".join([
            "\t".join([d, "%.2f" % s])
            for d, s in zip(beam_decodes, beam_scores)
        ]))
      else:
        decodes.append("\t".join(beam_decodes))
    else:
      _, decoded_outputs, _ = log_decode_results(
          result["inputs"],
          result["outputs"],
          problem_name,
          None,
          inputs_vocab,
          targets_vocab,
          log_results=decode_hp.log_results,
          skip_eos_postprocess=decode_hp.skip_eos_postprocess)
      decodes.append(decoded_outputs)
    total_time_per_step += elapsed_time
    total_cnt += result["outputs"].shape[-1]
  duration = time.time() - start_time
  tf.logging.info("Elapsed Time: %5.5f" % duration)
  tf.logging.info("Averaged Single Token Generation Time: %5.7f "
                  "(time %5.7f count %d)" %
                  (total_time_per_step / total_cnt,
                   total_time_per_step, total_cnt))
  if decode_hp.batch_size == 1:
    tf.logging.info("Inference time %.4f seconds "
                    "(Latency = %.4f ms/setences)" %
                    (duration, 1000.0*duration/num_sentences))
  else:
    tf.logging.info("Inference time %.4f seconds "
                    "(Throughput = %.4f sentences/second)" %
                    (duration, num_sentences/duration))

  # If decode_to_file was provided use it as the output filename without change
  # (except for adding shard_id if using more shards for decoding).
  # Otherwise, use the input filename plus model, hp, problem, beam, alpha.
  decode_filename = decode_to_file if decode_to_file else filename
  if not decode_to_file:
    decode_filename = _decode_filename(decode_filename, problem_name, decode_hp)
  else:
    decode_filename = _add_shard_to_filename(decode_filename, decode_hp)
  tf.logging.info("Writing decodes into %s" % decode_filename)
  outfile = tf.gfile.Open(decode_filename, "w")
  for index in range(len(sorted_inputs)):
    outfile.write("%s%s" % (decodes[sorted_keys[index]], decode_hp.delimiter))
  outfile.flush()
  outfile.close()

  output_dir = os.path.join(estimator.model_dir, "decode")
  tf.gfile.MakeDirs(output_dir)

  run_postdecode_hooks(DecodeHookArgs(
      estimator=estimator,
      problem=hparams.problem,
      output_dirs=[output_dir],
      hparams=hparams,
      decode_hparams=decode_hp,
      predictions=list(result_iter)
  ), None)
311059
def read(file, system):
    """"""
    try:
        fid = open(file, 'r')
        raw_file = fid.readlines()
    except IOError:
        print('* IOError while reading input card file.')
        return

    ret_dict = dict()
    ret_dict['outfile'] = file.split('.')[0].lower() + '.py'
    key, val = None, None
    for idx, line in enumerate(raw_file):
        line = line.strip()
        if not line:
            continue
        if line.startswith('#'):
            continue
        elif '#' in line:
            line = line.split('#')[0]
        if '=' in line:  # defining a field
            key, val = line.split('=')
            key, val = key.strip(), val.strip()
            val = [] if val == '' else val
            ret_dict.update({key: val})
            if val:
                val = val.split(';')
        else:
            val.extend(line.split(';'))
        if val:
            val = de_blank(val)
            ret_dict[key] = val

    ret_dict_ord = dict(ret_dict)

    for key, val in ret_dict.items():
        if not val:
            continue
        if type(val) == list:
            if ':' in val[0]:
                new_val = {}  # return in a dictionary
                new_val_ord = [
                ]  # return in an ordered list with the dict keys at 0
                for item in val:
                    try:
                        m, n = item.split(':')
                    except ValueError:
                        print('* Error: check line <{}>'.format(item))
                        return
                    m, n = m.strip(), n.strip()
                    if ',' in n:
                        n = n.split(',')
                        n = de_blank(n)
                        n = [to_number(i) for i in n]
                    else:
                        n = to_number(n)
                    new_val.update({m.strip(): n})
                    new_val_ord.append([m.strip(), n])
                ret_dict[key] = new_val
                ret_dict_ord[key] = new_val_ord

    ret_dict['name'] = ret_dict['name'][0]
    ret_dict['doc_string'] = ret_dict['doc_string'][0]
    ret_dict['group'] = ret_dict['group'][0]
    ret_dict['service_keys'] = list(ret_dict['service_eq'].keys())
    ret_dict['consts'] = list(ret_dict['data'].keys()) + list(
        ret_dict['service_eq'].keys())
    ret_dict['init1_eq'] = ret_dict_ord['init1_eq']
    ret_dict['service_eq'] = ret_dict_ord['service_eq']
    ret_dict['ctrl'] = ret_dict_ord['ctrl']

    copy_algebs = []
    copy_states = []
    for item in ret_dict['ctrl']:
        key, val = item
        if val[3] == 'y':
            copy_algebs.append(key)
        elif val[3] == 'x':
            copy_states.append(key)
        elif val[3] == 'c':
            ret_dict['consts'].append(key)
    ret_dict['copy_algebs'] = copy_algebs
    ret_dict['copy_states'] = copy_states

    return run(system, **ret_dict)
1087716
def _compile_column_metadata(row, keys, number):
    
    # Store the variable keys by index in a dictionary
    _column = {}
    _interpretation = {}
    _calibration = {}
    _physical = {}

    # Use the header keys to place the column data in the dictionary
    if keys:
        for idx, key in enumerate(keys):
            _key_low = key.lower()

            # Special case: Calibration data
            if re.match(re_calibration, _key_low):
                m = re.match(re_calibration, _key_low)
                if m:
                    _key = m.group(1)
                    _calibration[_key] = row[idx].value

            # Special case: PhysicalSample data
            elif re.match(re_physical, _key_low):
                m = re.match(re_physical, _key_low)
                if m:
                    _key = m.group(1)
                    _physical[_key] = row[idx].value

            # Special case: Interpretation data
            elif re.match(re_interpretation, _key_low):
                # Put interpretation data in a tmp dictionary that we'll sort later.
                _interpretation[_key_low] = row[idx].value

            else:
                try:
                    val = row[idx].value
                except Exception:
                    logger_excel.info("compile_column_metadata: Couldn't get value from row cell")
                    val = "n/a"
                try:
                    if key == "variableName":
                        val = _rm_units_from_var_name_single(row[idx].value)
                except Exception:
                    # when a variableName fails to split, keep the name as-is and move on.
                    pass
                _column[key] = val
        _column["number"] = number

        if _calibration:
            _column["calibration"] = _calibration
        # Only allow physicalSample on measured variableTypes. duh.
        if _physical and _column["variableType"] == "measured":
            _column["physicalSample"] = _physical
        if _interpretation:
            _interpretation_data = _compile_interpretation(_interpretation)
            _column["interpretation"] = _interpretation_data

    # If there are not keys, that means it's a header-less metadata section.
    else:
        # Assume we only have one cell, because we have no keys to know what data is here.
        try:
            val = row[0].value.lower()
        except AttributeError:
            val = row[0].value
        except Exception:
            logger_excel.info("compile_column_metadata: Couldn't get value from row cell")
            val = "n/a"
        val = _rm_units_from_var_name_single(val)
        _column["variableName"] = val
        _column["number"] = number

    # Add this column to the overall metadata, but skip if there's no data present
    _column = {k: v for k, v in _column.items() if v}

    return _column
960295
def readmarheader(filename):
    """"""
    with open(filename, 'rb') as f:
        intheader = np.fromstring(f.read(10 * 4), np.int32)
        floatheader = np.fromstring(f.read(15 * 4), '<f4')
        strheader = f.read(24)
        f.read(4)
        otherstrings = [f.read(16) for i in range(29)]
    return {'Xsize': intheader[0], 'Ysize': intheader[1], 'MeasTime': intheader[8],
            'BeamPosX': floatheader[7], 'BeamPosY': floatheader[8],
            'Wavelength': floatheader[9], 'Dist': floatheader[10],
            '__Origin__': 'MarResearch .image', 'recordlength': intheader[2],
            'highintensitypixels': intheader[4],
            'highintensityrecords': intheader[5],
            'Date': dateutil.parser.parse(strheader),
            'Detector': 'MARCCD', '__particle__': 'photon'}
450589
def read_from_file(cls, filename):
        """"""
        with open(filename) as f:
            lines = list(line for line in f if line[0] != '#')
        r = []
        t = []
        for line in lines[:3]:
            values = list(float(word) for word in line.split())
            r.append(values[:3])
            t.append(values[3])
        transformation = Complete(r, t)
        affected_atoms = set(int(word) for word in lines[3].split())
        return cls(affected_atoms, transformation)
1006800
def get_calibration(self, name, shape, chunks=None):
        """"""
        data_items = self.root.findall(".//calibrationVector")
        data, low_res_coords = self.read_xml_array(data_items, name)
        return self.interpolate_xml_array(data, low_res_coords, shape, chunks=chunks)
304507
def read_dict_from_file(file_path):
    
    with open(file_path) as file:
        lines = file.read().splitlines()

    obj = {}
    for line in lines:
        key, value = line.split(':', maxsplit=1)
        obj[key] = eval(value)

    return obj
958524
def read(self, fp):
        ""
        base_size = struct.unpack(str("=I"), fp.read(4))[0]
        self._units.fromfile(fp, base_size)
925391
def __parse(self) -> object:
        """"""
        char = self.data[self.idx: self.idx + 1]
        if char in [b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'0']:
            str_len = int(self.__read_to(b':'))
            return self.__read(str_len)
        elif char == b'i':
            self.idx += 1
            return int(self.__read_to(b'e'))
        elif char == b'd':
            return self.__parse_dict()
        elif char == b'l':
            return self.__parse_list()
        elif char == b'':
            raise bencodepy.DecodingError('Unexpected End of File at index position of {0}.'.format(str(self.idx)))
        else:
            raise bencodepy.DecodingError(
                'Invalid token character ({0}) at position {1}.'.format(str(char), str(self.idx)))
712022
def _get_calibration_params(self):
        """"""
        params = {}
        idx_table = []
        val_table = []
        for elt in self.mda['image_data_function'].split(b'\r\n'):
            try:
                key, val = elt.split(b':=')
                try:
                    idx_table.append(int(key))
                    val_table.append(float(val))
                except ValueError:
                    params[key] = val
            except ValueError:
                pass
        params['indices'] = np.array(idx_table)
        params['values'] = np.array(val_table, dtype=np.float32)
        return params
1304371
def parse_text(file_name):
    """"""

    # Read the header
    fid = open(file_name, 'r')
    point_names = np.array(fid.readline().split())[2:-1:3]
    fid.close()
    for i in range(len(point_names)):
        point_names[i] = point_names[i][0:-2]

    # Read the matrix data
    S = np.loadtxt(file_name, skiprows=1)
    field = np.uint(S[:, 0])
    times = S[:, 1]
    S = S[:, 2:]

    # Set the -9999.99 markers to be not present
    S[S==-9999.99] = np.NaN

    # Store x, y and z in different arrays
    points = []
    points.append(S[:, 0:-1:3])
    points.append(S[:, 1:-1:3])
    points.append(S[:, 2:-1:3])

    return points, point_names, times
1351232
def parse_readme():
    """"""
    # Get the long description from the relevant file
    here = os.path.abspath(os.path.dirname(__file__))
    readme_path = os.path.join(here, 'README.md')
    with codecs.open(readme_path, encoding='utf-8') as handle:
        long_description = handle.read()

    return long_description
761906
def ParseInput(self, a_file):
    
    input_lines = a_file.read().splitlines()
    self.ParseLines(input_lines)