Loads a data file into a list of `InputBatch`s.
428958
def convert_examples_to_features(examples, tokenizer, max_seq_length,
                                 doc_stride, max_query_length, is_training):
    """"""

    unique_id = 1000000000

    features = []
    for (example_index, example) in enumerate(examples):
        query_tokens = tokenizer.tokenize(example.question_text)

        if len(query_tokens) > max_query_length:
            query_tokens = query_tokens[0:max_query_length]

        tok_to_orig_index = []
        orig_to_tok_index = []
        all_doc_tokens = []
        for (i, token) in enumerate(example.doc_tokens):
            orig_to_tok_index.append(len(all_doc_tokens))
            sub_tokens = tokenizer.tokenize(token)
            for sub_token in sub_tokens:
                tok_to_orig_index.append(i)
                all_doc_tokens.append(sub_token)

        tok_start_position = None
        tok_end_position = None
        if is_training and example.is_impossible:
            tok_start_position = -1
            tok_end_position = -1
        if is_training and not example.is_impossible:
            tok_start_position = orig_to_tok_index[example.start_position]
            if example.end_position < len(example.doc_tokens) - 1:
                tok_end_position = orig_to_tok_index[example.end_position + 1] - 1
            else:
                tok_end_position = len(all_doc_tokens) - 1
            (tok_start_position, tok_end_position) = _improve_answer_span(
                all_doc_tokens, tok_start_position, tok_end_position, tokenizer,
                example.orig_answer_text)

        # The -3 accounts for [CLS], [SEP] and [SEP]
        max_tokens_for_doc = max_seq_length - len(query_tokens) - 3

        # We can have documents that are longer than the maximum sequence length.
        # To deal with this we do a sliding window approach, where we take chunks
        # of the up to our max length with a stride of `doc_stride`.
        _DocSpan = collections.namedtuple(  # pylint: disable=invalid-name
            "DocSpan", ["start", "length"])
        doc_spans = []
        start_offset = 0
        while start_offset < len(all_doc_tokens):
            length = len(all_doc_tokens) - start_offset
            if length > max_tokens_for_doc:
                length = max_tokens_for_doc
            doc_spans.append(_DocSpan(start=start_offset, length=length))
            if start_offset + length == len(all_doc_tokens):
                break
            start_offset += min(length, doc_stride)

        for (doc_span_index, doc_span) in enumerate(doc_spans):
            tokens = []
            token_to_orig_map = {}
            token_is_max_context = {}
            segment_ids = []
            tokens.append("[CLS]")
            segment_ids.append(0)
            for token in query_tokens:
                tokens.append(token)
                segment_ids.append(0)
            tokens.append("[SEP]")
            segment_ids.append(0)

            for i in range(doc_span.length):
                split_token_index = doc_span.start + i
                token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index]

                is_max_context = _check_is_max_context(doc_spans, doc_span_index,
                                                       split_token_index)
                token_is_max_context[len(tokens)] = is_max_context
                tokens.append(all_doc_tokens[split_token_index])
                segment_ids.append(1)
            tokens.append("[SEP]")
            segment_ids.append(1)

            input_ids = tokenizer.convert_tokens_to_ids(tokens)

            # The mask has 1 for real tokens and 0 for padding tokens. Only real
            # tokens are attended to.
            input_mask = [1] * len(input_ids)

            # Zero-pad up to the sequence length.
            while len(input_ids) < max_seq_length:
                input_ids.append(0)
                input_mask.append(0)
                segment_ids.append(0)

            assert len(input_ids) == max_seq_length
            assert len(input_mask) == max_seq_length
            assert len(segment_ids) == max_seq_length

            start_position = None
            end_position = None
            if is_training and not example.is_impossible:
                # For training, if our document chunk does not contain an annotation
                # we throw it out, since there is nothing to predict.
                doc_start = doc_span.start
                doc_end = doc_span.start + doc_span.length - 1
                out_of_span = False
                if not (tok_start_position >= doc_start and
                        tok_end_position <= doc_end):
                    out_of_span = True
                if out_of_span:
                    start_position = 0
                    end_position = 0
                else:
                    doc_offset = len(query_tokens) + 2
                    start_position = tok_start_position - doc_start + doc_offset
                    end_position = tok_end_position - doc_start + doc_offset
            if is_training and example.is_impossible:
                start_position = 0
                end_position = 0
            if example_index < 20:
                logger.info("*** Example ***")
                logger.info("unique_id: %s" % (unique_id))
                logger.info("example_index: %s" % (example_index))
                logger.info("doc_span_index: %s" % (doc_span_index))
                logger.info("tokens: %s" % " ".join(tokens))
                logger.info("token_to_orig_map: %s" % " ".join([
                    "%d:%d" % (x, y) for (x, y) in token_to_orig_map.items()]))
                logger.info("token_is_max_context: %s" % " ".join([
                    "%d:%s" % (x, y) for (x, y) in token_is_max_context.items()
                ]))
                logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
                logger.info(
                    "input_mask: %s" % " ".join([str(x) for x in input_mask]))
                logger.info(
                    "segment_ids: %s" % " ".join([str(x) for x in segment_ids]))
                if is_training and example.is_impossible:
                    logger.info("impossible example")
                if is_training and not example.is_impossible:
                    answer_text = " ".join(tokens[start_position:(end_position + 1)])
                    logger.info("start_position: %d" % (start_position))
                    logger.info("end_position: %d" % (end_position))
                    logger.info(
                        "answer: %s" % (answer_text))

            features.append(
                InputFeatures(
                    unique_id=unique_id,
                    example_index=example_index,
                    doc_span_index=doc_span_index,
                    tokens=tokens,
                    token_to_orig_map=token_to_orig_map,
                    token_is_max_context=token_is_max_context,
                    input_ids=input_ids,
                    input_mask=input_mask,
                    segment_ids=segment_ids,
                    start_position=start_position,
                    end_position=end_position,
                    is_impossible=example.is_impossible))
            unique_id += 1

    return features
1091248
def load(self, list_file):
    
    with open(list_file) as f:
      for line in f:
        if line and line[0] != '#':
          splits = line.split()
          bounding_boxes = []
          for i in range(1, len(splits), 4):
            assert splits[i][0] == '[' and splits[i+3][-1] == ']'
            bounding_boxes.append(BoundingBox(topleft=(float(splits[i][1:]), float(splits[i+1])), size=(float(splits[i+2]), float(splits[i+3][:-1]))))
          self.image_paths.append(splits[0])
          self.bounding_boxes.append(bounding_boxes)
1162251
def load_file(self, file_handle, data_set=0, read_data=True):
        """"""
        file_handle.seek(0, 2)
        self._file_size = file_handle.tell()
        file_handle.seek(0)
        data_segments = 0
        # seek the correct data set in fcs
        nextdata_offset = 0
        while data_segments <= data_set:
            self.read_header(file_handle, nextdata_offset)
            self.read_text(file_handle)
            if '$NEXTDATA' in self.annotation:
                data_segments += 1
                nextdata_offset = self.annotation['$NEXTDATA']
                file_handle.seek(nextdata_offset)
                if nextdata_offset == 0 and data_segments < data_set:
                    warnings.warn("File does not contain the number of data sets.")
                    break
            else:
                if data_segments != 0:
                    warnings.warn('File does not contain $NEXTDATA information.')
                break
        if read_data:
            self.read_data(file_handle)
243691
def load_data(self, filename, ext):
        """"""
        from spyder_kernels.utils.iofuncs import iofunctions
        from spyder_kernels.utils.misc import fix_reference_name

        glbs = self._mglobals()

        load_func = iofunctions.load_funcs[ext]
        data, error_message = load_func(filename)

        if error_message:
            return error_message

        for key in list(data.keys()):
            new_key = fix_reference_name(key, blacklist=list(glbs.keys()))
            if new_key != key:
                data[new_key] = data.pop(key)

        try:
            glbs.update(data)
        except Exception as error:
            return str(error)

        return None
146645
public void compileBatch(String []files)
    throws IOException, ClassNotFoundException
  {
    if (_compileParent) {
      try {
        if (_loader instanceof Make)
          ((Make) _loader).make();
      } catch (Exception e) {
        throw new IOException(e);
      }
    }

    if (files.length == 0)
      return;

    // only batch a number of files at a time

    int batchCount = _maxBatch;
    if (batchCount < 0)
      batchCount = Integer.MAX_VALUE / 2;
    else if (batchCount == 0)
      batchCount = 1;

    IOException exn = null;

    ArrayList<String> uniqueFiles = new ArrayList<String>();
    for (int i = 0; i < files.length; i++) {
      if (! uniqueFiles.contains(files[i]))
        uniqueFiles.add(files[i]);
    }
    files = new String[uniqueFiles.size()];
    uniqueFiles.toArray(files);

    LineMap lineMap = null;
    
    _compilerService.compile(this, files, lineMap);

    /*
    synchronized (LOCK) {
      for (int i = 0; i < files.length; i += batchCount) {
        int len = files.length - i;

        len = Math.min(len, batchCount);

        String []batchFiles = new String[len];

        System.arraycopy(files, i, batchFiles, 0, len);

        Arrays.sort(batchFiles);

        try {
          compileInt(batchFiles, null);
        } catch (IOException e) {
          if (exn == null)
            exn = e;
          else
            log.log(Level.WARNING, e.toString(), e);
        }
      }
    }

    if (exn != null)
      throw exn;
      */
  }
101113
public static void createFileBatchesSpark(JavaRDD<String> filePaths, final String rootOutputDir, final int batchSize, @NonNull final org.apache.hadoop.conf.Configuration hadoopConfig) {
        final SerializableHadoopConfig conf = new SerializableHadoopConfig(hadoopConfig);
        //Here: assume input is images. We can't store them as Float32 arrays - that's too inefficient
        // instead: let's store the raw file content in a batch.
        long count = filePaths.count();
        long maxPartitions = count / batchSize;
        JavaRDD<String> repartitioned = filePaths.repartition(Math.max(filePaths.getNumPartitions(), (int) maxPartitions));
        repartitioned.foreachPartition(new VoidFunction<Iterator<String>>() {
            @Override
            public void call(Iterator<String> stringIterator) throws Exception {
                //Construct file batch
                List<String> list = new ArrayList<>();
                List<byte[]> bytes = new ArrayList<>();
                FileSystem fs = FileSystem.get(conf.getConfiguration());
                while (stringIterator.hasNext()) {
                    String inFile = stringIterator.next();
                    byte[] fileBytes;
                    try (BufferedInputStream bis = new BufferedInputStream(fs.open(new Path(inFile)))) {
                        fileBytes = IOUtils.toByteArray(bis);
                    }
                    list.add(inFile);
                    bytes.add(fileBytes);

                    if (list.size() == batchSize) {
                        process(list, bytes);
                    }
                }
                if (list.size() > 0) {
                    process(list, bytes);
                }
            }

            private void process(List<String> paths, List<byte[]> bytes) throws IOException {
                FileBatch fb = new FileBatch(bytes, paths);
                String name = UUID.randomUUID().toString().replaceAll("-", "") + ".zip";
                String outPath = FilenameUtils.concat(rootOutputDir, name);
                FileSystem fileSystem = FileSystem.get(conf.getConfiguration());
                try (BufferedOutputStream bos = new BufferedOutputStream(fileSystem.create(new Path(outPath)))) {
                    fb.writeAsZip(bos);
                }

                paths.clear();
                bytes.clear();
            }
        });
    }
1371747
def loads(content):
    """"""
    lines = _group_lines(line for line in content.split('\n'))
    lines = [
        (i, _parse_envfile_line(line))
        for i, line in lines if line.strip()
    ]
    errors = []
    # Reject files with duplicate variables (no sane default).
    duplicates = _find_duplicates(((i, line[0]) for i, line in lines))
    for i, variable, j in duplicates:
        errors.append(''.join([
                'Line %d: duplicate environment variable "%s": ',
                'already appears on line %d.',
            ]) % (i + 1, variable, j + 1)
        )
    # Done!
    if errors:
        raise ValueError(errors)
    return {k: v for _, (k, v) in lines}
720306
def load_bnf_file(filepath, repository = None):
    """"""
    linelist = []
    with open(filepath,'r') as mlfile:
        for line in mlfile:
            linelist.append(line)
    return strlist_to_production_set(linelist, repository)
233451
def load_data(self, idx):
        """"""
        for name in self:
            ndim = getattr(self, '_%s_ndim' % name)
            diskflag = getattr(self, '_%s_diskflag' % name)
            ramflag = getattr(self, '_%s_ramflag' % name)
            if diskflag:
                file_ = getattr(self, '_%s_file' % name)
                length_tot = 1
                shape = []
                for jdx in range(ndim):
                    length = getattr(self, '_%s_length_%s' % (name, jdx))
                    length_tot *= length
                    shape.append(length)
                raw = file_.read(length_tot*8)
                values = struct.unpack(length_tot*'d', raw)
                if ndim:
                    values = numpy.array(values).reshape(shape)
                else:
                    values = values[0]
            elif ramflag:
                array = getattr(self, '_%s_array' % name)
                values = array[idx]
            if diskflag or ramflag:
                if ndim == 0:
                    setattr(self, name, values)
                else:
                    getattr(self, name)[:] = values
820098
def load_files(self, cursor, target, files, options):
        ""
        for fname in files:
            self.load_file(cursor, target, fname, options)
1023427
protected void loadRemaining() {
        if (!loadedAll) {
            // Put all of the batches from the sequence into the buffer
            assert targetNumRowsInMemory >= 0L;
            assert batchSize != null;
            Batch batch = original.nextBatch();
            boolean loadIntoMemory = inMemoryBatches != null && actualNumRowsInMemory < targetNumRowsInMemory;
            while (batch != null) {
                long rows = loadBatch(batch, loadIntoMemory, null);
                if (batchSize.get() == 0L) batchSize.set(rows);
                if (loadIntoMemory) {
                    assert inMemoryBatches != null;
                    if (actualNumRowsInMemory >= targetNumRowsInMemory) loadIntoMemory = false;
                }
                batch = original.nextBatch();
            }
            long numInMemory = inMemoryBatches != null ? actualNumRowsInMemory : 0L;
            totalSize = offHeapBatchesSupplier.size() + numInMemory;
            loadedAll = true;
            restartBatches();
        }
    }
820519
def __load(self, path):
        """"""
        try:
            path = os.path.abspath(path)
            with open(path, 'rb') as df:
                # loaded_dataset = pickle.load(df)
                self.__data, self.__classes, self.__labels, \
                self.__dtype, self.__description, \
                self.__num_features, self.__feature_names = pickle.load(df)

            # ensure the loaded dataset is valid
            self.__validate(self.__data, self.__classes, self.__labels)

        except IOError as ioe:
            raise IOError('Unable to read the dataset from file: {}', format(ioe))
        except:
            raise
783646
def load_file_to_list(self):
        
        lst = []
        try:
            with open(self.fullname, 'r') as f:
                for line in f:
                    lst.append(line) 
            return lst  
        except IOError:
            return lst
966349
def _serialize_inputs(inputs):
        """"""
        serialized_inputs = {}
        for input_id, input_value in inputs.items():
            if isinstance(input_value, list):
                serialized_list = Task._serialize_input_list(input_value)
                serialized_inputs[input_id] = serialized_list
            else:
                if isinstance(input_value, File):
                    input_value = Task._to_api_file_format(input_value)
                serialized_inputs[input_id] = input_value
        return serialized_inputs
465789
def _load_dataset(dsid, ds_info, file_handlers, dim='y'):
        """"""
        slice_list = []
        failure = True
        for fh in file_handlers:
            try:
                projectable = fh.get_dataset(dsid, ds_info)
                if projectable is not None:
                    slice_list.append(projectable)
                    failure = False
            except KeyError:
                logger.warning("Failed to load {} from {}".format(dsid, fh),
                               exc_info=True)

        if failure:
            raise KeyError(
                "Could not load {} from any provided files".format(dsid))

        if dim not in slice_list[0].dims:
            return slice_list[0]
        res = xr.concat(slice_list, dim=dim)

        combined_info = file_handlers[0].combine_info(
            [p.attrs for p in slice_list])

        res.attrs = combined_info
        return res
88137
@Override
    public List<DataSet> dataSetBatches(int num) {
        List<List<DataSet>> list = Lists.partition(asList(), num);
        List<DataSet> ret = new ArrayList<>();
        for (List<DataSet> l : list)
            ret.add(DataSet.merge(l));
        return ret;

    }
247653
def load_data_batch(self, data_batch):
        """"""
        _load_data(data_batch, self.data_arrays)
        _load_label(data_batch, self.label_arrays)