Calculate some interesting datapoints of the file
420067
def PrintSummary(logfile):
    ''''''
    # Open the log file
    mlog = mavutil.mavlink_connection(filename, notimestamps=args.notimestamps, dialect=args.dialect)

    autonomous_sections = 0 # How many different autonomous sections there are
    autonomous = False # Whether the vehicle is currently autonomous at this point in the logfile
    auto_time = 0.0 # The total time the vehicle was autonomous/guided (seconds)
    start_time = None # The datetime of the first received message (seconds since epoch)
    total_dist = 0.0 # The total ground distance travelled (meters)
    last_gps_msg = None # The first GPS message received
    first_gps_msg = None # The last GPS message received
    true_time = None # Track the first timestamp found that corresponds to a UNIX timestamp

    while True:
        m = mlog.recv_match(condition=args.condition)

        # If there's no match, it means we're done processing the log.
        if m is None:
            break

        # Ignore any failed messages
        if m.get_type() == 'BAD_DATA':
            continue

        # Keep track of the latest timestamp for various calculations
        timestamp = getattr(m, '_timestamp', 0.0)

        # Log the first message time
        if start_time is None:
            start_time = timestamp

        # Log the first timestamp found that is a true timestamp. We first try
        # to get the groundstation timestamp from the log directly. If that fails
        # we try to find a message that outputs a Unix time-since-epoch.
        if true_time is None:
            if not args.notimestamps and timestamp >= 1230768000:
                true_time = timestamp
            elif 'time_unix_usec' in m.__dict__ and m.time_unix_usec >= 1230768000:
                true_time = m.time_unix_usec * 1.0e-6
            elif 'time_usec' in m.__dict__ and m.time_usec >= 1230768000:
                true_time = m.time_usec * 1.0e-6

        # Track the vehicle's speed and status
        if m.get_type() == 'GPS_RAW_INT':

            # Ignore GPS messages without a proper fix
            if m.fix_type < 3 or m.lat == 0 or m.lon == 0:
                continue

            # Log the first GPS location found, being sure to skip GPS fixes
            # that are bad (at lat/lon of 0,0)
            if first_gps_msg is None:
                first_gps_msg = m

            # Track the distance travelled, being sure to skip GPS fixes
            # that are bad (at lat/lon of 0,0)
            if last_gps_msg is None or m.time_usec > last_gps_msg.time_usec or m.time_usec+30e6 < last_gps_msg.time_usec:
                if last_gps_msg is not None:
                    total_dist += distance_two(last_gps_msg, m)

                # Save this GPS message to do simple distance calculations with
                last_gps_msg = m

        elif m.get_type() == 'HEARTBEAT':
            if m.type == mavutil.mavlink.MAV_TYPE_GCS:
                continue
            if (m.base_mode & mavutil.mavlink.MAV_MODE_FLAG_GUIDED_ENABLED or
                m.base_mode & mavutil.mavlink.MAV_MODE_FLAG_AUTO_ENABLED) and autonomous == False:
                autonomous = True
                autonomous_sections += 1
                start_auto_time = timestamp
            elif (not m.base_mode & mavutil.mavlink.MAV_MODE_FLAG_GUIDED_ENABLED and
                not m.base_mode & mavutil.mavlink.MAV_MODE_FLAG_AUTO_ENABLED) and autonomous == True:
                autonomous = False
                auto_time += timestamp - start_auto_time

    # If there were no messages processed, say so
    if start_time is None:
        print("ERROR: No messages found.")
        return

    # If the vehicle ends in autonomous mode, make sure we log the total time
    if autonomous == True:
        auto_time += timestamp - start_auto_time

    # Compute the total logtime, checking that timestamps are 2009 or newer for validity
    # (MAVLink didn't exist until 2009)
    if true_time:
        start_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(true_time))
        print("Log started at about {}".format(start_time_str))
    else:
        print("Warning: No absolute timestamp found in datastream. No starting time can be provided for this log.")

    # Print location data
    if last_gps_msg is not None:
        first_gps_position = (first_gps_msg.lat / 1e7, first_gps_msg.lon / 1e7)
        last_gps_position = (last_gps_msg.lat / 1e7, last_gps_msg.lon / 1e7)
        print("Travelled from ({0[0]}, {0[1]}) to ({1[0]}, {1[1]})".format(first_gps_position, last_gps_position))
        print("Total distance : {:0.2f}m".format(total_dist))
    else:
        print("Warning: No GPS data found, can't give position summary.")

    # Print out the rest of the results.
    total_time = timestamp - start_time
    print("Total time (mm:ss): {:3.0f}:{:02.0f}".format(total_time / 60, total_time % 60))
    # The autonomous time should be good, as a steady HEARTBEAT is required for MAVLink operation
    print("Autonomous sections: {}".format(autonomous_sections))
    if autonomous_sections > 0:
        print("Autonomous time (mm:ss): {:3.0f}:{:02.0f}".format(auto_time / 60, auto_time % 60))

    totals.time += total_time
    totals.distance += total_dist
    totals.flights += 1
1123138
def sparklineData(data, filename):
    
    fd = file(filename, "w+b")
    for name in data.keys():
        action, reward = data[name]

        altName = name.lower().replace("_", "")

        fd.write("\def")
        fd.write("\REWARDDATA%s{" % altName)
        for i, r in enumerate(reward):
            fd.write("(%.2f,%.3f)" % (i / 10.0, r / 10.0)) # dimension too large
        fd.write("}\n")

        maxreward, maxindex = max(izip(reward, count()))
        minreward, minindex = min(izip(reward, count()))
        meanreward = scipy.mean(reward)
        fd.write("\def\REWARDMAX%s{%.1f}\n" % (altName, maxreward))
        fd.write("\def\REWARDMAXIDX%s{%d}\n" % (altName, maxindex))
        fd.write("\def\REWARDMIN%s{%.1f}\n" % (altName, minreward))
        fd.write("\def\REWARDMINIDX%s{%d}\n" % (altName, minindex))
        fd.write("\def\REWARDMEAN%s{%.1f}\n" % (altName, meanreward))

        fd.write("\def")
        fd.write("\ACTIONDATA%s{" % altName)
        for i, a in enumerate(action):
            fd.write("(%.2f,%.3f)" % (i / 10.0, a / 10.0))
        fd.write("}\n")

        maxaction, maxindex = max(izip(reward, count()))
        minaction, minindex = min(izip(reward, count()))
        meanaction = scipy.mean(reward)
        fd.write("\def\ACTIONMAX%s{%.1f}\n" % (altName, maxaction))
        fd.write("\def\ACTIONMAXIDX%s{%d}\n" % (altName, maxindex))
        fd.write("\def\ACTIONMIN%s{%.1f}\n" % (altName, minaction))
        fd.write("\def\ACTIONMINIDX%s{%d}\n" % (altName, minindex))
        fd.write("\def\ACTIONMEAN%s{%.1f}\n" % (altName, meanaction))
    fd.close()
412427
def polydata_fromfile(f, self):
    """"""
    points = []
    data = dict(vertices=[], lines=[], polygons=[], triangle_strips=[])
    l = common._getline(f).decode('ascii')
    k,n,datatype = [s.strip().lower() for s in l.split(' ')]
    if k!='points':
        raise ValueError('expected points but got %s'%(repr(k)))
    n = int(n)
    assert datatype in ['bit','unsigned_char','char','unsigned_short','short','unsigned_int','int','unsigned_long','long','float','double'],repr(datatype)

    log.debug('\tgetting %s points'%n)
    while len(points) < 3*n:
        l = common._getline(f).decode('ascii')
        points += map(eval,l.split(' '))
    assert len(points)==3*n
    while 1:
        l = common._getline(f)
        if l is None:
            break
        l = l.decode('ascii')
        sl = l.split(' ')
        k = sl[0].strip().lower()
        if k not in ['vertices','lines','polygons','triangle_strips']:
            break
        assert len(sl)==3
        n = int(sl[1])
        size = int(sl[2])
        lst = []
        while len(lst) < size:
            l = common._getline(f).decode('ascii')
            lst += map(eval, l.split(' '))
        assert len(lst)==size
        lst2 = []
        j = 0
        for i in range(n):
            lst2.append(lst[j+1:j+lst[j]+1])
            j += lst[j]+1
        data[k] = lst2

    return PolyData(points,data['vertices'], data['lines'], data['polygons'], data['triangle_strips']), l.encode()
738794
def load_mldataset(filename):
    
    user = []
    item = []
    score = []
    with open(filename) as f:
        for line in f:
            tks = line.strip().split('\t')
            if len(tks) != 4:
                continue
            user.append(int(tks[0]))
            item.append(int(tks[1]))
            score.append(float(tks[2]))
    user = mx.nd.array(user)
    item = mx.nd.array(item)
    score = mx.nd.array(score)
    return gluon.data.ArrayDataset(user, item, score)
487251
def predict_files(self, files):
        '''
        
        '''
        imgs = [0]*len(files)
        for i, file in enumerate(files):
            img = cv2.imread(file).astype('float64')
            img = cv2.resize(img, (224,224))
            img = preprocess_input(img)
            if img is None:
                print('failed to open: {}, continuing...'.format(file))
            imgs[i] = img
        return self.model.predict(np.array(imgs))
345756
def validate_data_files(problem, data_files, min_size):
  """"""
  # Check that all files are present
  data_dir = os.path.split(data_files[0])[0]
  out_filepaths = problem.out_filepaths(data_dir)
  missing_filepaths = set(out_filepaths) - set(data_files)
  if missing_filepaths:
    tf.logging.error("Missing %d data files", len(missing_filepaths))

  # Check that each file is at least 100M
  too_small = []
  for data_file in data_files:
    length = get_length(data_file)
    if length < min_size:
      too_small.append(data_file)
  if too_small:
    tf.logging.error("%d files too small", len(too_small))

  bad_files = too_small + list(missing_filepaths)
  return bad_files
1305901
def __calculate_bu_dfs(dfs_data):
    """"""
    u = dfs_data['ordering'][0]
    b = {}
    b[u] = D(u, dfs_data)
    __calculate_bu_dfs_recursively(u, b, dfs_data)
    return b
702971
def collect_data(self):
        
        #collect some measurements resolved per file and save to file
        for measure in ['LFP', 'CSD']:
            if measure in self.savelist:
                self.collectSingleContribs(measure)


        #calculate lfp from all cell contribs
        lfp = self.calc_signal_sum(measure='LFP')

        #calculate CSD in every lamina
        if self.calculateCSD:
            csd = self.calc_signal_sum(measure='CSD')

        if RANK == 0 and self.POPULATION_SIZE > 0:
            #saving LFPs
            if 'LFP' in self.savelist:
                fname = os.path.join(self.populations_path,
                                     self.output_file.format(self.y,
                                                             'LFP')+'.h5')
                f = h5py.File(fname, 'w')
                f['srate'] = 1E3 / self.dt_output
                f.create_dataset('data', data=lfp, compression=4)
                f.close()
                del lfp
                assert(os.path.isfile(fname))
                print('save lfp ok')


            #saving CSDs
            if 'CSD' in self.savelist and self.calculateCSD:
                fname = os.path.join(self.populations_path,
                                     self.output_file.format(self.y,
                                                             'CSD')+'.h5')
                f = h5py.File(fname, 'w')
                f['srate'] = 1E3 / self.dt_output
                f.create_dataset('data', data=csd, compression=4)
                f.close()
                del csd
                assert(os.path.isfile(fname))
                print('save CSD ok')


            #save the somatic placements:
            pop_soma_pos = np.zeros((self.POPULATION_SIZE, 3))
            keys = ['xpos', 'ypos', 'zpos']
            for i in range(self.POPULATION_SIZE):
                for j in range(3):
                    pop_soma_pos[i, j] = self.pop_soma_pos[i][keys[j]]
            fname = os.path.join(self.populations_path,
                                 self.output_file.format(self.y, 'somapos.gdf'))
            np.savetxt(fname, pop_soma_pos)
            assert(os.path.isfile(fname))
            print('save somapos ok')

            #save rotations using hdf5
            fname = os.path.join(self.populations_path,
                                    self.output_file.format(self.y, 'rotations.h5'))
            f = h5py.File(fname, 'w')
            f.create_dataset('x', (len(self.rotations),))
            f.create_dataset('y', (len(self.rotations),))
            f.create_dataset('z', (len(self.rotations),))

            for i, rot in enumerate(self.rotations):
                for key, value in list(rot.items()):
                    f[key][i] = value
            f.close()
            assert(os.path.isfile(fname))
            print('save rotations ok')


        #resync threads
        COMM.Barrier()
1029213
def plotted_data(self):
        """"""
        return InteractiveList(
            [arr for arr, val in zip(self.iter_data,
                                     cycle(slist(self.value)))
             if val is not None])
1361887
def _collect_data(directory, input_ext, target_ext):
  """"""
  # Directory from string to tuple pair of strings
  # key: the filepath to a datafile including the datafile's basename. Example,
  #   if the datafile was "/path/to/datafile.wav" then the key would be
  #   "/path/to/datafile"
  # value: a pair of strings (input_filepath, target_filepath)
  data_files = {}
  for root, _, filenames in os.walk(directory):
    input_files = [filename for filename in filenames if input_ext in filename]
    for input_filename in input_files:
      basename = input_filename.strip(input_ext)
      input_file = os.path.join(root, input_filename)
      target_file = os.path.join(root, basename + target_ext)
      key = os.path.join(root, basename)
      assert os.path.exists(target_file)
      assert key not in data_files
      data_files[key] = (input_file, target_file)
  return data_files
738178
def files_log_graph(self, stream):
        '''  '''
        for row in list(stream):

            # dataframes['files_log'][['md5','mime_type','missing_bytes','rx_hosts','source','tx_hosts']]
            
            # If the mime-type is interesting add the uri and the host->uri->host relationships
            if row['mime_type'] not in self.exclude_mime_types:

                # Check for weird conditions
                if (row['total_bytes'] == '-'):
                    continue
                if ('-' in row['md5']):
                    continue

                # Check for missing bytes and small file
                if row['missing_bytes']:
                    labels = ['missing', 'file']
                elif row['total_bytes'] < 50*1024:
                    labels = ['small','file']
                else:
                    labels = ['file']

                # Make the file node name kewl
                name = '%6s  %s  %.0f-KB' % (row['md5'][:6], row['mime_type'], row['total_bytes']/1024.0)
                if row['missing_bytes']:
                    name += '*'
                name = name.replace('application/','')
                
                # Add the file node
                self.add_node(row['md5'], name, labels)

                # Add the tx_host
                self.add_node(row['tx_hosts'], row['tx_hosts'], ['host'])

                # Add the file->tx_host relationship
                self.add_rel(row['tx_hosts'], row['md5'], 'file')
45269
def parse_wyckoff_csv(wyckoff_file):
    

    rowdata = []
    points = []
    hP_nums = [433, 436, 444, 450, 452, 458, 460]
    for i, line in enumerate(wyckoff_file):
        if line.strip() == 'end of data':
            break
        rowdata.append(line.strip().split(':'))

        # 2:P -1  ::::::: <-- store line number if first element is number
        if rowdata[-1][0].isdigit():
            points.append(i)
    points.append(i)

    wyckoff = []
    for i in range(len(points) - 1):  # 0 to 529
        symbol = rowdata[points[i]][1]  # e.g. "C 1 2 1"
        if i + 1 in hP_nums:
            symbol = symbol.replace('R', 'H', 1)
        wyckoff.append({'symbol': symbol.strip()})

    # When the number of positions is larger than 4,
    # the positions are written in the next line.
    # So those positions are connected.
    for i in range(len(points) - 1):
        count = 0
        wyckoff[i]['wyckoff'] = []
        for j in range(points[i] + 1, points[i + 1]):
            # Hook if the third element is a number (multiplicity), e.g.,
            #
            # 232:P 2/b 2/m 2/b:::::::  <- ignored
            # ::8:r:1:(x,y,z):(-x,y,-z):(x,-y+1/2,-z):(-x,-y+1/2,z)
            # :::::(-x,-y,-z):(x,-y,z):(-x,y+1/2,z):(x,y+1/2,-z)  <- ignored
            # ::4:q:..m:(x,0,z):(-x,0,-z):(x,1/2,-z):(-x,1/2,z)
            # ::4:p:..2:(0,y,1/2):(0,-y+1/2,1/2):(0,-y,1/2):(0,y+1/2,1/2)
            # ::4:o:..2:(1/2,y,0):(1/2,-y+1/2,0):(1/2,-y,0):(1/2,y+1/2,0)
            # ...
            if rowdata[j][2].isdigit():
                pos = []
                w = {'letter': rowdata[j][3].strip(),
                     'multiplicity': int(rowdata[j][2]),
                     'site_symmetry': rowdata[j][4].strip(),
                     'positions': pos}
                wyckoff[i]['wyckoff'].append(w)

                for k in range(4):
                    if rowdata[j][k + 5]:  # check if '(x,y,z)' or ''
                        count += 1
                        pos.append(rowdata[j][k + 5])
            else:
                for k in range(4):
                    if rowdata[j][k + 5]:
                        count += 1
                        pos.append(rowdata[j][k + 5])

        # assertion
        for w in wyckoff[i]['wyckoff']:
            n_pos = len(w['positions'])
            n_pos *= len(lattice_symbols[wyckoff[i]['symbol'][0]])
            assert n_pos == w['multiplicity']

    return wyckoff
1064830
def _fileToMatrix(file_name):
    """"""
    # TODO: np.loadtxt() might be an alternative
    #     try:
    if 1 < 3:
        lres = []
        for line in open(file_name, 'r').readlines():
            if len(line) > 0 and line[0] not in ('%', '#'):
                lres.append(list(map(float, line.split())))
        res = lres
    while res != [] and res[0] == []:  # remove further leading empty lines
        del res[0]
    return res
    #     except:
    print('could not read file ' + file_name)
177424
def file_rows(self, fo):
        """"""

        rows = []
        for i in range(NUMROWS):
            line = fo.readline()
            if not line:
                break
            rows += [line]

        return rows
1343396
def input_fn(data_file, num_epochs, shuffle, batch_size):
  """"""
  assert tf.gfile.Exists(data_file), (
      '%s not found. Please make sure you have run census_dataset.py and '
      'set the --data_dir argument to the correct path.' % data_file)

  def parse_csv(value):
    tf.logging.info('Parsing {}'.format(data_file))
    columns = tf.decode_csv(value, record_defaults=_CSV_COLUMN_DEFAULTS)
    features = dict(zip(_CSV_COLUMNS, columns))
    labels = features.pop('income_bracket')
    classes = tf.equal(labels, '>50K')  # binary classification
    return features, classes

  # Extract lines from input files using the Dataset API.
  dataset = tf.data.TextLineDataset(data_file)

  if shuffle:
    dataset = dataset.shuffle(buffer_size=_NUM_EXAMPLES['train'])

  dataset = dataset.map(parse_csv, num_parallel_calls=5)

  # We call repeat after shuffling, rather than before, to prevent separate
  # epochs from blending together.
  dataset = dataset.repeat(num_epochs)
  dataset = dataset.batch(batch_size)
  return dataset
535751
def num_open_files(self):
        """"""
        return reduce(lambda x, y: x + y, (1 for rlist in self._fdict.values() if len(rlist) > 0), 0)
40778
def _load(self, filename):
        """"""
        filename = pathlib.Path(filename)
        with filename.open() as fd:
            data = fd.readlines()

        # Get the strings that correspond to self.fileid
        bool_head = [l.strip().startswith("[") for l in data]

        int_head = np.squeeze(np.where(bool_head))
        int_head = np.atleast_1d(int_head)

        start = int_head[self.fileid]+1

        if len(int_head) > self.fileid+1:
            end = int_head[self.fileid+1]
        else:
            end = len(data)

        subdata = data[start:end]

        # separate all elements and strip them
        subdata = [[it.strip() for it in l.split("=")] for l in subdata]

        points = []

        for var, val in subdata:
            if var.lower() == "x axis":
                xaxis = val.lower()
            elif var.lower() == "y axis":
                yaxis = val.lower()
            elif var.lower() == "name":
                self.name = val
            elif var.lower() == "inverted":
                if val == "True":
                    self.inverted = True
            elif var.lower().startswith("point"):
                val = np.array(val.strip("[]").split(), dtype=float)
                points.append([int(var[5:]), val])
            else:
                raise KeyError("Unknown variable: {} = {}".
                               format(var, val))
        self.axes = (xaxis, yaxis)
        # sort points
        points.sort()
        # get only coordinates from points
        self.points = np.array([p[1] for p in points])

        # overwrite unique id
        unique_id = int(data[start-1].strip().strip("Polygon []"))
        self._set_unique_id(unique_id)