Collect statistics on nanopub file

    input_fn can be json, jsonl or yaml and additionally gzipped
432671
def nanopub_stats(ctx, input_fn):
    

    counts = {
        "nanopubs": 0,
        "assertions": {"total": 0, "subject_only": 0, "nested": 0, "relations": {}},
    }

    for np in bnf.read_nanopubs(input_fn):
        if "nanopub" in np:
            counts["nanopubs"] += 1
            counts["assertions"]["total"] += len(np["nanopub"]["assertions"])
            for assertion in np["nanopub"]["assertions"]:
                if assertion["relation"] is None:
                    counts["assertions"]["subject_only"] += 1
                else:
                    if re.match("\s*\(", assertion["object"]):
                        counts["assertions"]["nested"] += 1

                    if (
                        not assertion.get("relation")
                        in counts["assertions"]["relations"]
                    ):
                        counts["assertions"]["relations"][assertion.get("relation")] = 1
                    else:
                        counts["assertions"]["relations"][
                            assertion.get("relation")
                        ] += 1

    counts["assertions"]["relations"] = sorted(counts["assertions"]["relations"])

    print("DumpVar:\n", json.dumps(counts, indent=4))
313920
function getSourceFilesFromGlob(globPattern, ignoreGlobs) {
    return new Promise((fulfil, reject) => {
        glob(globPattern, { ignore: ignoreGlobs }, (err, files) => {
            if (err) {
                reject(err);
            } else {
                fulfil(files);
            }
        });
    });
}
890053
def jsonload(model, fp):
    '''
    
    '''
    dumped_list = json.load(fp)
    for link in dumped_list:
        if len(link) == 2:
            sid, (s, p, o, a) = link
        elif len(link) == 4: #canonical
            (s, p, o, a) = link
            tt = a.get('@target-type')
            if tt == '@iri-ref':
                o = I(o)
            a.pop('@target-type', None)
        else:
            continue
        model.add(s, p, o, a)
    return
1168674
def _readPPN(self, fname, sldir):
        '''
        

        '''
        if sldir.endswith(os.sep):
                    #Making sure fname will be formatted correctly
            fname = str(sldir)+str(fname)
        else:
            fname = str(sldir)+os.sep+str(fname)
            self.sldir+=os.sep
        f=open(fname,'r')
        lines=f.readlines()
        for i in range(len(lines)):
            lines[i]=lines[i].strip()

        cols = ['ISOTP', 'ABUNDANCE_MF'] #These are constant, .ppn files have no header to read from
        for i in range(len(lines)):
            if not lines[i].startswith('H'):
                index = i-1
                break

        return cols, index
405786
function readYml(file, fn) {

  var m;
  var api = {};
  var resource = yaml.safeLoad(fs.readFileSync(file, 'utf8'));
  var paths = Object.keys(resource.paths);
  _.each(paths, function(path) {
    if (options.apiPrefix && path.indexOf(options.apiPrefix) === 0){
        path = path.replace(options.apiPrefix, "");
    }
    api.resourcePath = path;
    api.description = resource.description || "";
    descriptor.paths[path] = resource.paths[path];

    // append definitions
    if (descriptor.definitions && Object.keys(descriptor.definitions).length) {
      m = _.merge(descriptor.definitions, resource.definitions);
      descriptor.definitions = m;
    } else {
      descriptor.definitions = resource.definitions;
    }
  });

  fn();
}
846872
def load_existing_json():
        
        dir = logger.get_logger_dir()
        fname = os.path.join(dir, JSONWriter.FILENAME)
        if tf.gfile.Exists(fname):
            with open(fname) as f:
                stats = json.load(f)
                assert isinstance(stats, list), type(stats)
                return stats
        return None
1161867
def dump_json(model, batch_len=200000, use_natural_keys=True, verbosity=1):
    
    model = get_model(model)

    N = model.objects.count()

    if verbosity > 0:
        widgets = [pb.Counter(), '/%d rows: ' % (N,), pb.Percentage(), ' ', pb.RotatingMarker(), ' ', pb.Bar(),' ', pb.ETA()]
        i, pbar = 0, pb.ProgressBar(widgets=widgets, maxval=N).start()

    JSONSerializer = serializers.get_serializer("json")
    jser = JSONSerializer()

    if verbosity > 0:
        pbar.update(0)
    for i, partial_qs in enumerate(util.generate_slices(model.objects.all(), batch_len=batch_len)):
        with open(model._meta.app_label.lower() + '--' + model._meta.object_name.lower() + '--%04d.json' % i, 'w') as fpout:
            if verbosity > 0:
                pbar.update(i*batch_len)
            jser.serialize(partial_qs, indent=1, stream=fpout, use_natural_keys=use_natural_keys)
    if verbosity > 0:
        pbar.finish()
286820
def yield_nanopub(assertions, annotations, line_num):
    """"""

    if not assertions:
        return {}

    anno = copy.deepcopy(annotations)

    evidence = anno.pop("evidence", None)
    stmt_group = anno.pop("statement_group", None)
    citation = anno.pop("citation", None)

    anno_list = []
    for anno_type in anno:
        if isinstance(anno[anno_type], (list, tuple)):
            for val in anno[anno_type]:
                anno_list.append({"type": anno_type, "label": val})
        else:
            anno_list.append({"type": anno_type, "label": anno[anno_type]})

    assertions_list = []
    for assertion in assertions:
        (subj, rel, obj) = split_bel_stmt(assertion, line_num)
        assertions_list.append({"subject": subj, "relation": rel, "object": obj})

    nanopub = {
        "schema_uri": "https://raw.githubusercontent.com/belbio/schemas/master/schemas/nanopub_bel-1.0.0.yaml",
        "type": copy.deepcopy(nanopub_type),
        "annotations": copy.deepcopy(anno_list),
        "citation": copy.deepcopy(citation),
        "assertions": copy.deepcopy(assertions_list),
        "evidence": evidence,
        "metadata": {"statement_group": stmt_group},
    }

    return {"nanopub": copy.deepcopy(nanopub)}
770689
@GwtIncompatible("Unnecessary")
  private List<SourceFile> createSourceInputs(
      List<JsModuleSpec> jsModuleSpecs,
      List<FlagEntry<JsSourceType>> files,
      List<JsonFileSpec> jsonFiles,
      List<String> moduleRoots)
      throws IOException {
    if (isInTestMode()) {
      return inputsSupplierForTesting != null ? inputsSupplierForTesting.get()
          : null;
    }
    if (files.isEmpty() && jsonFiles == null) {
      // Request to read from stdin.
      files = ImmutableList.of(new FlagEntry<JsSourceType>(JsSourceType.JS, "-"));
    }

    for (JSError error : deduplicateIjsFiles(files, moduleRoots, !jsModuleSpecs.isEmpty())) {
      compiler.report(error);
    }

    try {
      if (jsonFiles != null) {
        return createInputs(files, jsonFiles, jsModuleSpecs);
      } else {
        return createInputs(files, true, jsModuleSpecs);
      }
    } catch (FlagUsageException e) {
      throw new FlagUsageException("Bad --js flag. " + e.getMessage());
    }
  }
606491
def serving_input_fn(self, hparams, decode_hparams=None, use_tpu=False):
    """"""
    mode = tf.estimator.ModeKeys.PREDICT
    serialized_example = tf.placeholder(
        dtype=tf.string, shape=[None], name="serialized_example")
    dataset = tf.data.Dataset.from_tensor_slices(serialized_example)
    dataset = dataset.map(self.decode_example)
    dataset = dataset.map(lambda ex: self.preprocess_example(ex, mode, hparams))
    dataset = dataset.map(data_reader.cast_ints_to_int32)

    if use_tpu:
      padded_shapes = data_reader.pad_for_tpu(dataset.output_shapes, hparams,
                                              hparams.max_length)
      batch_size = 1 if not decode_hparams else getattr(decode_hparams,
                                                        "batch_size", 1)
      dataset = dataset.padded_batch(
          batch_size, padded_shapes, drop_remainder=False)
      dataset = dataset.map(
          functools.partial(data_reader.pad_batch, batch_multiple=batch_size))
    else:
      dataset = dataset.padded_batch(
          tf.shape(serialized_example, out_type=tf.int64)[0],
          dataset.output_shapes)

    dataset = dataset.map(data_reader.standardize_shapes)
    features = tf.data.experimental.get_single_element(dataset)

    if self.has_inputs:
      features.pop("targets", None)

    return tf.estimator.export.ServingInputReceiver(
        features=features, receiver_tensors=serialized_example)
585772
def gzipped(fn):
    
    def gzipped_wrapper(*args, **kwargs):
        accepted_encoding = request.get_header("Accept-Encoding")

        if not accepted_encoding or "gzip" not in accepted_encoding:
            return fn(*args, **kwargs)

        response.set_header("Content-Encoding", "gzip")

        return to_gzipped_file(fn(*args, **kwargs))

    return gzipped_wrapper
180542
def packet_get_samples_per_frame(data, fs):
    """"""

    data_pointer = ctypes.c_char_p(data)

    result = _packet_get_nb_frames(data_pointer, ctypes.c_int(fs))
    if result < 0:
        raise OpusError(result)

    return result
1283245
def get_experiment_fn(args):
  

  def get_experiment(output_dir):
    # Read schema, input features, and transforms.
    schema_path_with_target = os.path.join(args.analysis,
                                           feature_transforms.SCHEMA_FILE)
    features_path = os.path.join(args.analysis,
                                 feature_transforms.FEATURES_FILE)
    stats_path = os.path.join(args.analysis,
                              feature_transforms.STATS_FILE)

    schema = read_json_file(schema_path_with_target)
    features = read_json_file(features_path)
    stats = read_json_file(stats_path)

    target_column_name = feature_transforms.get_target_name(features)
    if not target_column_name:
      raise ValueError('target missing from features file.')

    # Make a copy of the schema file without the target column.
    schema_without_target = [col for col in schema if col['name'] != target_column_name]
    schema_path_without_target = os.path.join(args.job_dir, 'schema_without_target.json')
    file_io.recursive_create_dir(args.job_dir)
    file_io.write_string_to_file(schema_path_without_target,
                                 json.dumps(schema_without_target, indent=2))

    # Make list of files to save with the trained model.
    additional_assets_with_target = {
        feature_transforms.FEATURES_FILE: features_path,
        feature_transforms.SCHEMA_FILE: schema_path_with_target}
    additional_assets_without_target = {
        feature_transforms.FEATURES_FILE: features_path,
        feature_transforms.SCHEMA_FILE: schema_path_without_target}

    # Get the model to train.
    target_vocab = read_vocab(args, target_column_name)
    estimator = get_estimator(args, output_dir, features, stats, len(target_vocab))

    export_strategy_csv_notarget = make_export_strategy(
        args=args,
        keep_target=False,
        assets_extra=additional_assets_without_target,
        features=features,
        schema=schema,
        stats=stats)
    export_strategy_csv_target = make_export_strategy(
        args=args,
        keep_target=True,
        assets_extra=additional_assets_with_target,
        features=features,
        schema=schema,
        stats=stats)

    # Build readers for training.
    if args.transform:
      if any(v['transform'] == feature_transforms.IMAGE_TRANSFORM
             for k, v in six.iteritems(features)):
        raise ValueError('"image_to_vec" transform requires transformation step. ' +
                         'Cannot train from raw data.')

      input_reader_for_train = feature_transforms.build_csv_transforming_training_input_fn(
          schema=schema,
          features=features,
          stats=stats,
          analysis_output_dir=args.analysis,
          raw_data_file_pattern=args.train,
          training_batch_size=args.train_batch_size,
          num_epochs=args.max_epochs,
          randomize_input=True,
          min_after_dequeue=10,
          reader_num_threads=multiprocessing.cpu_count())
      input_reader_for_eval = feature_transforms.build_csv_transforming_training_input_fn(
          schema=schema,
          features=features,
          stats=stats,
          analysis_output_dir=args.analysis,
          raw_data_file_pattern=args.eval,
          training_batch_size=args.eval_batch_size,
          num_epochs=1,
          randomize_input=False,
          reader_num_threads=multiprocessing.cpu_count())
    else:
      input_reader_for_train = feature_transforms.build_tfexample_transfored_training_input_fn(
          schema=schema,
          features=features,
          analysis_output_dir=args.analysis,
          raw_data_file_pattern=args.train,
          training_batch_size=args.train_batch_size,
          num_epochs=args.max_epochs,
          randomize_input=True,
          min_after_dequeue=10,
          reader_num_threads=multiprocessing.cpu_count())
      input_reader_for_eval = feature_transforms.build_tfexample_transfored_training_input_fn(
          schema=schema,
          features=features,
          analysis_output_dir=args.analysis,
          raw_data_file_pattern=args.eval,
          training_batch_size=args.eval_batch_size,
          num_epochs=1,
          randomize_input=False,
          reader_num_threads=multiprocessing.cpu_count())

    if args.early_stopping_num_evals == 0:
      train_monitors = None
    else:
      if is_classification_model(args.model):
        early_stop_monitor = tf.contrib.learn.monitors.ValidationMonitor(
            input_fn=input_reader_for_eval,
            every_n_steps=args.min_eval_frequency,
            early_stopping_rounds=(args.early_stopping_num_evals * args.min_eval_frequency),
            early_stopping_metric='accuracy',
            early_stopping_metric_minimize=False)
      else:
        early_stop_monitor = tf.contrib.learn.monitors.ValidationMonitor(
            input_fn=input_reader_for_eval,
            every_n_steps=args.min_eval_frequency,
            early_stopping_rounds=(args.early_stopping_num_evals * args.min_eval_frequency))
      train_monitors = [early_stop_monitor]

    return tf.contrib.learn.Experiment(
        estimator=estimator,
        train_input_fn=input_reader_for_train,
        eval_input_fn=input_reader_for_eval,
        train_steps=args.max_steps,
        train_monitors=train_monitors,
        export_strategies=[export_strategy_csv_notarget, export_strategy_csv_target],
        min_eval_frequency=args.min_eval_frequency,
        eval_steps=None)

  # Return a function to create an Experiment.
  return get_experiment
765476
def process_definition(line, nanopubs_metadata):
    """"""

    matches = re.match('DEFINE\s+(\w+)\s+(\w+)\s+AS\s+URL\s+"(.*?)"\s*$', line)
    if matches:
        def_type = matches.group(1).lower()
        if def_type == "namespace":
            def_type = "namespaces"
        elif def_type == "annotation":
            def_type == "annotations"

        key = matches.group(2)
        val = matches.group(3)

        if def_type in nanopubs_metadata:
            nanopubs_metadata[def_type][key] = val
        else:
            nanopubs_metadata[def_type] = {key: val}

    matches = re.match("DEFINE\s+(\w+)\s+(\w+)\s+AS\s+LIST\s+{(.*?)}\s*$", line)
    if matches:
        def_type = matches.group(1).lower()
        if def_type == "namespace":
            def_type = "namespaces"
        elif def_type == "annotation":
            def_type == "annotations"

        key = matches.group(2)
        val = matches.group(3)
        vals = convert_csv_str_to_list(val)

        if def_type in nanopubs_metadata:
            nanopubs_metadata[def_type][key] = vals
        else:
            nanopubs_metadata[def_type] = {key: vals}

    return nanopubs_metadata
460260
def json_friendly(obj):
    """"""
    converted = True
    typename = get_full_typename(obj)

    if is_tf_tensor_typename(typename):
        obj = obj.eval()
    elif is_pytorch_tensor_typename(typename):
        try:
            if obj.requires_grad:
                obj = obj.detach()
        except AttributeError:
            pass  # before 0.4 is only present on variables

        try:
            obj = obj.data
        except RuntimeError:
            pass  # happens for Tensors before 0.4

        if obj.size():
            obj = obj.numpy()
        else:
            return obj.item(), True

    if np and isinstance(obj, np.ndarray):
        if obj.size == 1:
            obj = obj.flatten()[0]
        elif obj.size <= 32:
            obj = obj.tolist()
    elif np and isinstance(obj, np.generic):
        obj = obj.item()
    elif isinstance(obj, bytes):
        obj = obj.decode('utf-8')
    elif isinstance(obj, (datetime, date)):
        obj = obj.isoformat()
    else:
        converted = False
    if getsizeof(obj) > VALUE_BYTES_LIMIT:
        logger.warning("Object %s is %i bytes", obj, getsizeof(obj))

    return obj, converted
1179123
def nfa_json_importer(input_file: str) -> dict:
    
    file = open(input_file)
    json_file = json.load(file)

    transitions = {}  # key [state in states, action in alphabet]
    #                   value [Set of arriving states in states]
    for p in json_file['transitions']:
        transitions.setdefault((p[0], p[1]), set()).add(p[2])

    nfa = {
        'alphabet': set(json_file['alphabet']),
        'states': set(json_file['states']),
        'initial_states': set(json_file['initial_states']),
        'accepting_states': set(json_file['accepting_states']),
        'transitions': transitions
    }

    return nfa
209328
def get_nanopub(url):
    """"""

    r = bel.utils.get_url(url, cache=False)
    if r and r.json():
        return r.json()
    else:
        return {}