reads files off disk, resizes them
        and then predicts them, files should
        be a list or itrerable of file paths
        that lead to images, they are then
        loaded with opencv, resized, and predicted
487251
def predict_files(self, files):
        '''
        
        '''
        imgs = [0]*len(files)
        for i, file in enumerate(files):
            img = cv2.imread(file).astype('float64')
            img = cv2.resize(img, (224,224))
            img = preprocess_input(img)
            if img is None:
                print('failed to open: {}, continuing...'.format(file))
            imgs[i] = img
        return self.model.predict(np.array(imgs))
782096
def load_fits(self):
        '''
        

        '''

        log.info("Loading FITS file for %d." % (self.ID))
        with pyfits.open(self.fitsfile) as f:

            # Params and long cadence data
            self.loaded = True
            self.is_parent = False
            try:
                self.X1N = f[2].data['X1N']
            except KeyError:
                self.X1N = None
            self.aperture = f[3].data
            self.aperture_name = f[1].header['APNAME']
            try:
                self.bkg = f[1].data['BKG']
            except KeyError:
                self.bkg = 0.
            self.bpad = f[1].header['BPAD']
            self.cbv_minstars = []
            self.cbv_num = f[1].header.get('CBVNUM', 1)
            self.cbv_niter = f[1].header['CBVNITER']
            self.cbv_win = f[1].header['CBVWIN']
            self.cbv_order = f[1].header['CBVORD']
            self.cadn = f[1].data['CADN']
            self.cdivs = f[1].header['CDIVS']
            self.cdpp = f[1].header['CDPP']
            self.cdppr = f[1].header['CDPPR']
            self.cdppv = f[1].header['CDPPV']
            self.cdppg = f[1].header['CDPPG']
            self.cv_min = f[1].header['CVMIN']
            self.fpix = f[2].data['FPIX']
            self.pixel_images = [f[4].data['STAMP1'],
                                 f[4].data['STAMP2'], f[4].data['STAMP3']]
            self.fraw = f[1].data['FRAW']
            self.fraw_err = f[1].data['FRAW_ERR']
            self.giter = f[1].header['GITER']
            self.gmaxf = f[1].header.get('GMAXF', 200)
            self.gp_factor = f[1].header['GPFACTOR']
            try:
                self.hires = f[5].data
            except:
                self.hires = None
            self.kernel_params = np.array([f[1].header['GPWHITE'],
                                           f[1].header['GPRED'],
                                           f[1].header['GPTAU']])
            try:
                self.kernel = f[1].header['KERNEL']
                self.kernel_params = np.append(
                    self.kernel_params,
                    [f[1].header['GPGAMMA'],
                     f[1].header['GPPER']])
            except KeyError:
                self.kernel = 'Basic'
            self.pld_order = f[1].header['PLDORDER']
            self.lam_idx = self.pld_order
            self.leps = f[1].header['LEPS']
            self.mag = f[0].header['KEPMAG']
            self.max_pixels = f[1].header['MAXPIX']
            self.model = self.fraw - f[1].data['FLUX']
            self.nearby = []
            for i in range(99):
                try:
                    ID = f[1].header['NRBY%02dID' % (i + 1)]
                    x = f[1].header['NRBY%02dX' % (i + 1)]
                    y = f[1].header['NRBY%02dY' % (i + 1)]
                    mag = f[1].header['NRBY%02dM' % (i + 1)]
                    x0 = f[1].header['NRBY%02dX0' % (i + 1)]
                    y0 = f[1].header['NRBY%02dY0' % (i + 1)]
                    self.nearby.append(
                        {'ID': ID, 'x': x, 'y': y,
                         'mag': mag, 'x0': x0, 'y0': y0})
                except KeyError:
                    break
            self.neighbors = []
            for c in range(99):
                try:
                    self.neighbors.append(f[1].header['NEIGH%02d' % (c + 1)])
                except KeyError:
                    break
            self.oiter = f[1].header['OITER']
            self.optimize_gp = f[1].header['OPTGP']
            self.osigma = f[1].header['OSIGMA']
            self.planets = []
            for i in range(99):
                try:
                    t0 = f[1].header['P%02dT0' % (i + 1)]
                    per = f[1].header['P%02dPER' % (i + 1)]
                    dur = f[1].header['P%02dDUR' % (i + 1)]
                    self.planets.append((t0, per, dur))
                except KeyError:
                    break
            self.quality = f[1].data['QUALITY']
            self.saturated = f[1].header['SATUR']
            self.saturation_tolerance = f[1].header['SATTOL']
            self.time = f[1].data['TIME']
            self._norm = np.array(self.fraw)

            # Chunk arrays
            self.breakpoints = []
            self.cdpp_arr = []
            self.cdppv_arr = []
            self.cdppr_arr = []
            for c in range(99):
                try:
                    self.breakpoints.append(f[1].header['BRKPT%02d' % (c + 1)])
                    self.cdpp_arr.append(f[1].header['CDPP%02d' % (c + 1)])
                    self.cdppr_arr.append(f[1].header['CDPPR%02d' % (c + 1)])
                    self.cdppv_arr.append(f[1].header['CDPPV%02d' % (c + 1)])
                except KeyError:
                    break
            self.lam = [[f[1].header['LAMB%02d%02d' % (c + 1, o + 1)]
                        for o in range(self.pld_order)]
                        for c in range(len(self.breakpoints))]
            if self.model_name == 'iPLD':
                self.reclam = [[f[1].header['RECL%02d%02d' % (c + 1, o + 1)]
                               for o in range(self.pld_order)]
                               for c in range(len(self.breakpoints))]

            # Masks
            self.badmask = np.where(self.quality & 2 ** (QUALITY_BAD - 1))[0]
            self.nanmask = np.where(self.quality & 2 ** (QUALITY_NAN - 1))[0]
            self.outmask = np.where(self.quality & 2 ** (QUALITY_OUT - 1))[0]
            self.recmask = np.where(self.quality & 2 ** (QUALITY_REC - 1))[0]
            self.transitmask = np.where(
                self.quality & 2 ** (QUALITY_TRN - 1))[0]

            # CBVs
            self.XCBV = np.empty((len(self.time), 0))
            for i in range(99):
                try:
                    self.XCBV = np.hstack(
                        [self.XCBV,
                         f[1].data['CBV%02d' % (i + 1)].reshape(-1, 1)])
                except KeyError:
                    break

        # These are not stored in the fits file; we don't need them
        self.saturated_aperture_name = None
        self.apertures = None
        self.Xpos = None
        self.Ypos = None
        self.fpix_err = None
        self.parent_model = None
        self.lambda_arr = None
        self.meta = None
        self._transit_model = None
        self.transit_depth = None
747133
def _load_data_and_files(self):
        
        if not _hasattr(self, '_data'):
            self._data, self._files = self._parse()
            if self._files:
                self._full_data = self._data.copy()
                self._full_data.update(self._files)
            else:
                self._full_data = self._data
658601
def _imload(self, filepath, kwds):
        
        start_time = time.time()
        typ, enc = mimetypes.guess_type(filepath)
        if not typ:
            typ = 'image/jpeg'
        typ, subtyp = typ.split('/')
        self.logger.debug("MIME type is %s/%s" % (typ, subtyp))

        data_loaded = False
        if have_opencv and subtyp not in ['gif']:
            # First choice is OpenCv, because it supports high-bit depth
            # multiband images
            means = 'opencv'
            data_np = cv2.imread(filepath,
                                 cv2.IMREAD_ANYDEPTH + cv2.IMREAD_ANYCOLOR)
            if data_np is not None:
                data_loaded = True
                # funky indexing because opencv returns BGR images,
                # whereas PIL and others return RGB
                if len(data_np.shape) >= 3 and data_np.shape[2] >= 3:
                    data_np = data_np[..., :: -1]

                # OpenCv doesn't "do" image metadata, so we punt to piexif
                # library (if installed)
                self.piexif_getexif(filepath, kwds)

                # OpenCv added a feature to do auto-orientation when loading
                # (see https://github.com/opencv/opencv/issues/4344)
                # So reset these values to prevent auto-orientation from
                # happening later
                kwds['Orientation'] = 1
                kwds['Image Orientation'] = 1

                # convert to working color profile, if can
                if self.clr_mgr.can_profile():
                    data_np = self.clr_mgr.profile_to_working_numpy(data_np, kwds)

        if not data_loaded and have_pil:
            means = 'PIL'
            image = PILimage.open(filepath)

            try:
                if hasattr(image, '_getexif'):
                    info = image._getexif()
                    if info is not None:
                        for tag, value in info.items():
                            kwd = TAGS.get(tag, tag)
                            kwds[kwd] = value

                elif have_exif:
                    self.piexif_getexif(image.info["exif"], kwds)

                else:
                    self.logger.warning("Please install 'piexif' module to get image metadata")

            except Exception as e:
                self.logger.warning("Failed to get image metadata: %s" % (str(e)))

            # convert to working color profile, if can
            if self.clr_mgr.can_profile():
                image = self.clr_mgr.profile_to_working_pil(image, kwds)

            # convert from PIL to numpy
            data_np = np.array(image)
            if data_np is not None:
                data_loaded = True

        if (not data_loaded and (typ == 'image') and
            (subtyp in ('x-portable-pixmap', 'x-portable-greymap'))):
            # Special opener for PPM files, preserves high bit depth
            means = 'built-in'
            data_np = open_ppm(filepath)
            if data_np is not None:
                data_loaded = True

        if not data_loaded:
            raise ImageError("No way to load image format '%s/%s'" % (
                typ, subtyp))

        end_time = time.time()
        self.logger.debug("loading (%s) time %.4f sec" % (
            means, end_time - start_time))
        return data_np
1072591
def load(self, patterns, dirs, ignore=None):
        
        for _dir in dirs:
            data = self.read_file(_dir, ignore=ignore)
            if data:
                self.paths[_dir] = data
820519
def __load(self, path):
        """"""
        try:
            path = os.path.abspath(path)
            with open(path, 'rb') as df:
                # loaded_dataset = pickle.load(df)
                self.__data, self.__classes, self.__labels, \
                self.__dtype, self.__description, \
                self.__num_features, self.__feature_names = pickle.load(df)

            # ensure the loaded dataset is valid
            self.__validate(self.__data, self.__classes, self.__labels)

        except IOError as ioe:
            raise IOError('Unable to read the dataset from file: {}', format(ioe))
        except:
            raise
562522
def from_files(filenames, strict_type_checks=True):
    
    if strict_type_checks:
        load_whitelist()

    all_histograms = OrderedDict()
    for filename in filenames:
        parser = FILENAME_PARSERS[os.path.basename(filename)]
        histograms = parser(filename, strict_type_checks)

        # OrderedDicts are important, because then the iteration order over
        # the parsed histograms is stable, which makes the insertion into
        # all_histograms stable, which makes ordering in generated files
        # stable, which makes builds more deterministic.
        if not isinstance(histograms, OrderedDict):
            raise ParserError("Histogram parser did not provide an OrderedDict.")

        for name, definition in iteritems(histograms):
            if name in all_histograms:
                raise ParserError('Duplicate histogram name "%s".' % name)
            all_histograms[name] = definition

    # We require that all USE_COUNTER2_* histograms be defined in a contiguous
    # block.
    use_counter_indices = filter(lambda x: x[1].startswith("USE_COUNTER2_"),
                                 enumerate(iterkeys(all_histograms)))
    if use_counter_indices:
        lower_bound = use_counter_indices[0][0]
        upper_bound = use_counter_indices[-1][0]
        n_counters = upper_bound - lower_bound + 1
        if n_counters != len(use_counter_indices):
            raise ParserError("Use counter histograms must be defined in a contiguous block.")

    # Check that histograms that were removed from Histograms.json etc.
    # are also removed from the whitelists.
    if whitelists is not None:
        all_whitelist_entries = itertools.chain.from_iterable(whitelists.itervalues())
        orphaned = set(all_whitelist_entries) - set(iterkeys(all_histograms))
        if len(orphaned) > 0:
            msg = 'The following entries are orphaned and should be removed from ' \
                  'histogram-whitelists.json:\n%s'
            raise ParserError(msg % (', '.join(sorted(orphaned))))

    for name, definition in iteritems(all_histograms):
        yield Histogram(name, definition, strict_type_checks=strict_type_checks)
408436
def _fromfile(self):
        
        fd = self.parent._fd
        byte_order = self.parent.byte_order
        offset_size = self.parent.offset_size

        fmt = {4: 'I', 8: 'Q'}[offset_size]
        offset = struct.unpack(byte_order + fmt, fd.read(offset_size))[0]
        if not offset:
            raise StopIteration()

        # read standard tags
        tags = self.tags
        fd.seek(offset)
        fmt, size = {4: ('H', 2), 8: ('Q', 8)}[offset_size]
        try:
            numtags = struct.unpack(byte_order + fmt, fd.read(size))[0]
        except Exception:
            warnings.warn("corrupted page list")
            raise StopIteration()

        for _ in range(numtags):
            tag = TIFFtag(self.parent)
            tags[tag.name] = tag

        # read LSM info subrecords
        if self.is_lsm:
            pos = fd.tell()
            for name, reader in CZ_LSM_INFO_READERS.items():
                try:
                    offset = self.cz_lsm_info["offset_"+name]
                except KeyError:
                    continue
                if not offset:
                    continue
                fd.seek(offset)
                try:
                    setattr(self, "cz_lsm_"+name, reader(fd, byte_order))
                except ValueError:
                    pass
            fd.seek(pos)
396653
def rescan(self):
        
        if not self.attached:
            return
        dprint(5, "starting rescan")
        newstuff = {};  # this accumulates names of new or changed files. Keys are paths, values are 'quiet' flag.
        # store timestamp of scan
        self.last_scan_timestamp = time.time()
        # go through watched files/directories, check for mtime changes
        for path, watcher in list(self.watchers.items()):
            # get list of new files from watcher
            newfiles = watcher.newFiles()
            # None indicates access error, so drop it from watcher set
            if newfiles is None:
                if watcher.survive_deletion:
                    dprintf(5, "access error on %s, but will still be watched\n", watcher.path)
                else:
                    dprintf(2, "access error on %s, will no longer be watched\n", watcher.path)
                    del self.watchers[path]
                if not watcher.disappeared:
                    self.emit(SIGNAL("disappearedFile"), path)
                    watcher.disappeared = True
                continue
            dprintf(5, "%s: %d new file(s)\n", watcher.path, len(newfiles))
            # if a file has its own watcher, and is independently reported by a directory watcher, skip the directory's
            # version and let the file's watcher report it. Reason for this is that the file watcher may have a more
            # up-to-date timestamp, so we trust it over the dir watcher.
            newfiles = [p for p in newfiles if p is path or p not in self.watchers]
            # skip files in self._unwatched_paths
            newfiles = [filename for filename in newfiles if
                        self._watching_state.get(os.path.dirname(filename)) > Purr.UNWATCHED]
            # Now go through files and add them to the newstuff dict
            for newfile in newfiles:
                # if quiet flag is explicitly set on watcher, enforce it
                # if not pouncing on directory, also add quietly
                if watcher.quiet or self._watching_state.get(os.path.dirname(newfile)) < Purr.POUNCE:
                    quiet = True
                # else add quietly if file is not in the quiet patterns
                else:
                    quiet = matches_patterns(os.path.basename(newfile), self._quiet_patterns)
                # add file to list of new products. Since a file may be reported by multiple
                # watchers, make the quiet flag a logical AND of all the quiet flags (i.e. DP will be
                # marked as quiet only if all watchers report it as quiet).
                newstuff[newfile] = quiet and newstuff.get(newfile, True)
                dprintf(4, "%s: new data product, quiet=%d (watcher quiet: %s)\n", newfile, quiet, watcher.quiet)
                # add a watcher for this file to the temp_watchers list. this is used below
                # to detect renamed and deleted files
                self.temp_watchers[newfile] = Purrer.WatchedFile(newfile)
        # now, go through temp_watchers to see if any newly pounced-on files have disappeared
        for path, watcher in list(self.temp_watchers.items()):
            # get list of new files from watcher
            if watcher.newFiles() is None:
                dprintf(2, "access error on %s, marking as disappeared", watcher.path)
                del self.temp_watchers[path]
                self.emit(SIGNAL("disappearedFile"), path)
        # if we have new data products, send them to the main window
        return self.makeDataProducts(iter(newstuff.items()))
570122
def _load_bboxes(csv_path, csv_positions, prefix):
  """"""
  logging.info('Loading CSVs %s from positions %s with prefix %s',
               csv_path, csv_positions, prefix)
  boxes = collections.defaultdict(list)
  with tf.io.gfile.GFile(csv_path) as csv_f:
    if csv_positions[0] > 0:
      csv_f.seek(csv_positions[0])
    else:
      csv_f.readline()  # Drop headers
    reader = csv.reader(csv_f)
    for (image_id, source, label, confidence, xmin, xmax, ymin, ymax,
         is_occluded, is_truncated, is_group_of, is_depiction, is_inside,
        ) in reader:
      if prefix and image_id[0] != prefix:
        break
      csv_positions[0] = csv_f.tell()
      image_id = int(image_id, 16)
      del confidence  # always 1 in bounding boxes.
      current_row = _Bbox(
          label, source, tfds.features.BBox(
              float(ymin), float(xmin), float(ymax), float(xmax)),
          int(is_occluded), int(is_truncated),
          int(is_group_of), int(is_depiction), int(is_inside))
      boxes[image_id].append(current_row)
  return dict(boxes)
231294
def _openResources(self):
        
        dct = np.load(self._fileName, allow_pickle=ALLOW_PICKLE)
        check_class(dct, NpzFile)
        self._dictionary = dct
582582
def read_examples(input_files, batch_size, shuffle, num_epochs=None):
  """"""
  files = []
  for e in input_files:
    for path in e.split(','):
      files.extend(file_io.get_matching_files(path))
  thread_count = multiprocessing.cpu_count()

  # The minimum number of instances in a queue from which examples are drawn
  # randomly. The larger this number, the more randomness at the expense of
  # higher memory requirements.
  min_after_dequeue = 1000

  # When batching data, the queue's capacity will be larger than the batch_size
  # by some factor. The recommended formula is (num_threads + a small safety
  # margin). For now, we use a single thread for reading, so this can be small.
  queue_size_multiplier = thread_count + 3

  # Convert num_epochs == 0 -> num_epochs is None, if necessary
  num_epochs = num_epochs or None

  # Build a queue of the filenames to be read.
  filename_queue = tf.train.string_input_producer(files, num_epochs, shuffle)

  example_id, encoded_example = tf.TextLineReader().read_up_to(
      filename_queue, batch_size)

  if shuffle:
    capacity = min_after_dequeue + queue_size_multiplier * batch_size
    return tf.train.shuffle_batch(
        [example_id, encoded_example],
        batch_size,
        capacity,
        min_after_dequeue,
        enqueue_many=True,
        num_threads=thread_count)

  else:
    capacity = queue_size_multiplier * batch_size
    return tf.train.batch(
        [example_id, encoded_example],
        batch_size,
        capacity=capacity,
        enqueue_many=True,
        num_threads=thread_count)
1255044
def predict(self, image_file):
        
        image = misc.imread(image_file)
        gray_image = image
        if len(image.shape) > 2:
            gray_image = cv2.cvtColor(image, code=cv2.COLOR_BGR2GRAY)
        resized_image = cv2.resize(gray_image, self.target_dimensions, interpolation=cv2.INTER_LINEAR)
        final_image = np.array([np.array([resized_image]).reshape(list(self.target_dimensions)+[self.channels])])
        prediction = self.model.predict(final_image)
        # Return the dominant expression
        dominant_expression = self._print_prediction(prediction[0])
        return dominant_expression
1022067
def createImageObjectList(files,instrpars,group=None,
                            undistort=True, inmemory=False):
    
    imageObjList = []
    mtflag = False
    mt_refimg = None
    for img in files:
        image = _getInputImage(img,group=group)
        image.setInstrumentParameters(instrpars)
        image.compute_wcslin(undistort=undistort)
        if 'MTFLAG' in image._image['PRIMARY'].header:
            # check to see whether we are dealing with moving target observations...
            _keyval = image._image['PRIMARY'].header['MTFLAG']
            if not util.is_blank(_keyval):
                if isinstance(_keyval,bool):
                    mtflag = _keyval
                else:
                    if 'T' in _keyval:
                        mtflag = True
                    else:
                        mtflag = False
        else:
            mtflag = False

        if mtflag:
            print("#####\nProcessing Moving Target Observations using reference image as WCS for all inputs!\n#####\n")
            if mt_refimg is None:
                mt_refimg = image
            else:
                image.set_mt_wcs(mt_refimg)
        image.inmemory = inmemory # set flag for inmemory processing
        # Now add (possibly updated) image object to list
        imageObjList.append(image)
    return imageObjList
640980
def loadFeatures(path, opt):
    

    log.info("loading from %s ..." % path)
    data = []
    if opt.in_format == "BED":        
        with open(path) as fd:
            for line in fd:
                cols = line.split()
                data.append( (cols[0], int(cols[1]), int(cols[2]), cols[3]) )
        data = np.array(data, dtype=elem_t)
    else:
        with open(path) as fd:
            for line in fd:
                cols = line.split()
                data.append( (cols[0], int(cols[1]), int(cols[2]), cols[3], int(cols[4]),
                              cols[5], float(cols[6]), float(cols[7]), float(cols[8]),
                              int(cols[-1])+int(cols[1])) )
        data = np.array(data, dtype=narrowPeak_t)
    return data
590769
def _load(self, titles=[], descriptions=[], images=[], urls=[], **kwargs):
        
        enough = lambda items: items # len(items) >= MAX_ITEMS

        if config.GET_ALL_DATA or not enough(self.titles):
            titles = filter(None, map(self._clean_text, titles))
            self.titles.extend(titles)

        if config.GET_ALL_DATA or not enough(self.descriptions):
            descriptions = filter(None, map(self._clean_text, descriptions))
            self.descriptions.extend(descriptions)

        ## Never mind the urls, they can be bad not worth it
        # if config.GET_ALL_DATA or not enough(self.urls):
            # # urls = [self._clean_url(u) for u in urls]
            # urls = filter(None, map(self._clean_url, urls))
            # self.urls.extend(urls)

        if config.GET_ALL_DATA:
            # images = [i for i in [self._filter_image(i) for i in images] if i]
            images = filter(None, map(self._filter_image, images))
            self.images.extend(images)
        elif not enough(self.images):
            for i in images:
                image = self._filter_image(i)
                if image:
                    self.images.append(image)
                if enough(self.images):
                    break
859209
def poll(self, force_rescan = False):
      """"""
      # Check for new, deleted, and rotated files.
      if force_rescan or time.time() > self._last_scan + self._interval:
         self._rescan(skip_to_end = False)
         self._last_scan = time.time()

      filereaders = {}
      for path, tailedfile in self._tailedfiles.iteritems():
         filereaders[path] = tailedfile.readlines()

      # One line is read from each file in turn, in an attempt to read
      # from all files evenly. They'll be in an undefined order because
      # of using a dict for filereaders, but that's not a problem
      # because some entropy here is desirable for evenness.
      while len(filereaders) > 0:
         for path in filereaders.keys():
            lines = filereaders[path]
            try:
               line, offset = lines.next()
            except StopIteration:
               # Reached end the of this file.
               del filereaders[path]
               break
            
            yield (path, offset), line