work out gps lock times for a log file
195798
def lock_time(logfile):
    ''''''
    print("Processing log %s" % filename)
    mlog = mavutil.mavlink_connection(filename)

    locked = False
    start_time = 0.0
    total_time = 0.0
    t = None
    m = mlog.recv_match(type=['GPS_RAW_INT','GPS_RAW'], condition=args.condition)
    if m is None:
        return 0

    unlock_time = time.mktime(time.localtime(m._timestamp))

    while True:
        m = mlog.recv_match(type=['GPS_RAW_INT','GPS_RAW'], condition=args.condition)
        if m is None:
            if locked:
                total_time += time.mktime(t) - start_time
            if total_time > 0:
                print("Lock time : %u:%02u" % (int(total_time)/60, int(total_time)%60))
            return total_time
        t = time.localtime(m._timestamp)
        if m.fix_type >= 2 and not locked:
            print("Locked at %s after %u seconds" % (time.asctime(t),
                                                     time.mktime(t) - unlock_time))
            locked = True
            start_time = time.mktime(t)
        elif m.fix_type == 1 and locked:
            print("Lost GPS lock at %s" % time.asctime(t))
            locked = False
            total_time += time.mktime(t) - start_time
            unlock_time = time.mktime(t)
        elif m.fix_type == 0 and locked:
            print("Lost protocol lock at %s" % time.asctime(t))
            locked = False
            total_time += time.mktime(t) - start_time
            unlock_time = time.mktime(t)
    return total_time
654620
def parse_lock(self):
        
        try:
            with open(self.lock_file, "r") as reader:
                data = json.loads(reader.read())
                self.last_update = datetime.datetime.strptime(
                    data["last_update"],
                    AppCronLock.DATETIME_FORMAT
                )
        except:  # malformed lock file
            self.write_lock(last_update=datetime.datetime.fromtimestamp(0))
            self.parse_lock()
666549
public void logTimes(String src, long mtime, long atime) {
    TimesOp op = TimesOp.getInstance();
    op.set(src, mtime, atime);
    logEdit(op);
  }
716482
def get_stream_duration(stream):
    
    epoch = lal.LIGOTimeGPS(stream.epoch.gpsSeconds,
                            stream.epoch.gpsNanoSeconds)
    # loop over each file in the stream cache and query its duration
    nfile = stream.cache.length
    duration = 0
    for dummy_i in range(nfile):
        for dummy_j in range(lalframe.FrFileQueryNFrame(stream.file)):
            duration += lalframe.FrFileQueryDt(stream.file, 0)
            lalframe.FrStreamNext(stream)
    # rewind stream and return
    lalframe.FrStreamSeek(stream, epoch)
    return duration
334133
def main():
    
    # Detector positions in ENU relative to the station GPS
    x = [-6.34, -2.23, -3.6, 3.46]
    y = [6.34, 2.23, -3.6, 3.46]

    # Scale mips to fit the graph
    n = [35.0, 51.9, 35.8, 78.9]

    # Make times relative to first detection
    t = [15., 17.5, 20., 27.5]
    dt = [ti - min(t) for ti in t]

    plot = Plot()
    plot.scatter([0], [0], mark='triangle')
    plot.add_pin_at_xy(0, 0, 'Station 503', use_arrow=False, location='below')
    plot.scatter_table(x, y, dt, n)

    plot.set_scalebar(location="lower right")
    plot.set_colorbar('$\Delta$t [ns]')
    plot.set_axis_equal()
    plot.set_mlimits(max=16.)
    plot.set_slimits(min=10., max=100.)

    plot.set_xlabel('x [m]')
    plot.set_ylabel('y [m]')

    plot.save('event_display')


    # Add event by Station 508
    # Detector positions in ENU relative to the station GPS
    x508 = [6.12, 0.00, -3.54, 3.54]
    y508 = [-6.12, -13.23, -3.54, 3.54]

    # Event GPS timestamp: 1371498167.016412100
    # MIPS
    n508 = [5.6, 16.7, 36.6, 9.0]
    # Arrival Times
    t508 = [15., 22.5, 22.5, 30.]
    dt508 = [ti - min(t508) for ti in t508]

    plot = MultiPlot(1, 2, width=r'.33\linewidth')
    plot.set_xlimits_for_all(min=-10, max=15)
    plot.set_ylimits_for_all(min=-15, max=10)
    plot.set_mlimits_for_all(min=0., max=16.)
    plot.set_colorbar('$\Delta$t [ns]', False)
    plot.set_colormap('blackwhite')
    plot.set_scalebar_for_all(location="upper right")

    p = plot.get_subplot_at(0, 0)
    p.scatter([0], [0], mark='triangle')
    p.add_pin_at_xy(0, 0, 'Station 503', use_arrow=False, location='below')
    p.scatter_table(x, y, dt, n)
    p.set_axis_equal()

    p = plot.get_subplot_at(0, 1)
    p.scatter([0], [0], mark='triangle')
    p.add_pin_at_xy(0, 0, 'Station 508', use_arrow=False, location='below')
    p.scatter_table(x508, y508, dt508, n508)
    p.set_axis_equal()

    plot.show_yticklabels_for_all([(0, 0)])
    plot.show_xticklabels_for_all([(0, 0), (0, 1)])

    plot.set_xlabel('x [m]')
    plot.set_ylabel('y [m]')

    plot.save('multi_event_display')
748212
def lock_pid(self):
        
        
        if os.path.exists(self.lock_filename):
            return int(open(self.lock_filename).read())
        else:
            return None
522
def main():
    
    parser = argparse.ArgumentParser(
        description='Extract the raw gps communication from an ULog file')
    parser.add_argument('filename', metavar='file.ulg', help='ULog input file')

    def is_valid_directory(parser, arg):
        """Check if valid directory"""
        if not os.path.isdir(arg):
            parser.error('The directory {} does not exist'.format(arg))
        # File exists so return the directory
        return arg
    parser.add_argument('-o', '--output', dest='output', action='store',
                        help='Output directory (default is CWD)',
                        metavar='DIR', type=lambda x: is_valid_directory(parser, x))

    args = parser.parse_args()
    ulog_file_name = args.filename

    msg_filter = ['gps_dump']
    ulog = ULog(ulog_file_name, msg_filter)
    data = ulog.data_list

    output_file_prefix = os.path.basename(ulog_file_name)
    # strip '.ulg'
    if output_file_prefix.lower().endswith('.ulg'):
        output_file_prefix = output_file_prefix[:-4]

    # write to different output path?
    if args.output is not None:
        output_file_prefix = os.path.join(args.output, output_file_prefix)

    to_dev_filename = output_file_prefix+'_to_device.dat'
    from_dev_filename = output_file_prefix+'_from_device.dat'


    if len(data) == 0:
        print("File {0} does not contain gps_dump messages!".format(ulog_file_name))
        exit(0)

    gps_dump_data = data[0]

    # message format check
    field_names = [f.field_name for f in gps_dump_data.field_data]
    if not 'len' in field_names or not 'data[0]' in field_names:
        print('Error: gps_dump message has wrong format')
        exit(-1)

    if len(ulog.dropouts) > 0:
        print("Warning: file contains {0} dropouts".format(len(ulog.dropouts)))

    print("Creating files {0} and {1}".format(to_dev_filename, from_dev_filename))

    with open(to_dev_filename, 'wb') as to_dev_file:
        with open(from_dev_filename, 'wb') as from_dev_file:
            msg_lens = gps_dump_data.data['len']
            for i in range(len(gps_dump_data.data['timestamp'])):
                msg_len = msg_lens[i]
                if msg_len & (1<<7):
                    msg_len = msg_len & ~(1<<7)
                    file_handle = to_dev_file
                else:
                    file_handle = from_dev_file
                for k in range(msg_len):
                    file_handle.write(gps_dump_data.data['data['+str(k)+']'][i])
22187
def analyze_log_file(logfile, pattern, reverse_paths=True, progress=True):
    ""
    if progress:
        lines = count_lines_in(logfile)
        pbar = ProgressBar(widgets=[Percentage(), Bar()], maxval=lines+1).start()
        counter = 0
    
    data = {}
    
    compiled_pattern = compile(pattern)
    for line in fileinput.input([logfile]):
        
        if progress:
            counter = counter + 1
        
        parsed = compiled_pattern.findall(line)[0]
        date = parsed[0]
        method = parsed[1]
        path = parsed[2]
        status = parsed[3]
        time = parsed[4]
        sql = parsed[5]
        sqltime = parsed[6]

        try:
            ignore = False
            for ignored_path in IGNORE_PATHS:
                compiled_path = compile(ignored_path)
                if compiled_path.match(path):
                    ignore = True
            if not ignore:
                if reverse_paths:
                    view = view_name_from(path)
                else:
                    view = path
                key = "%s-%s-%s" % (view, status, method)
                try:
                    data[key]['count'] = data[key]['count'] + 1
                    data[key]['times'].append(float(time))
                    data[key]['sql'].append(int(sql))
                    data[key]['sqltime'].append(float(sqltime))
                except KeyError:
                    data[key] = {
                        'count': 1,
                        'status': status,
                        'view': view,
                        'method': method,
                        'times': [float(time)],
                        'sql': [int(sql)],
                        'sqltime': [float(sqltime)],
                    }
        except Resolver404:
            pass
        
        if progress:
            pbar.update(counter)
    
    if progress:
        pbar.finish()
    
    return data
1314470
def analyze_logfile_object(self, file_object):
        """"""
        log_parser = LogParser()

        if self._start_time is None:
            self._start_time = datetime.now()
            if self._timeout != 0:
                self._end_time = self._start_time + timedelta(minutes=self._timeout)
            else:
                self._end_time = None

        # For each line in the logfile ...
        for line in file_object:
            if self._end_time is not None and datetime.now() > self._end_time:
                self._run_stats['timedOut'] = True
                self._run_stats['timeoutInMinutes'] = self._timeout
                break
            self._process_query(line, log_parser)

        return 0
1046734
def tail(ctx):
    """"""
    click.echo('tailing logs')
    for e in ctx.tail()[-10:]:
        ts = datetime.utcfromtimestamp(e['timestamp'] // 1000).isoformat()
        click.echo("{}: {}".format(ts, e['message']))
    click.echo('done')
171819
def _log_to_file(self, etype, source, msg='', target='', hostmask='', params=''):
        
        today = datetime.utcnow()
        # TODO: Use self.locale['timezone'] for changing time
        date = today.strftime(self.locale['date_fmt'])
        time = today.strftime(self.locale['time_fmt'])
        datetime_s = today.strftime(self.locale['short_datetime_fmt'])
        datetime_l = today.strftime(self.locale['long_datetime_fmt'])
        
        if target == self.nickname and etype in ('privmsg', 'action'):
            target = source

        if etype in self.log_formats:
            file_path = os.path.expanduser(self.log_file.format(**locals()))
            file_dir = os.path.dirname(file_path)
            if not os.path.isdir(file_dir):
                os.makedirs(file_dir)

            line = self.log_formats[etype].format(**locals())
            with open(file_path, 'a') as log:
                log.write(line+'\n')
1319043
def getmtime(self, filepath):
        """"""
        if self.is_ssh(filepath):
            self._check_ftp()
            source = self._get_remote(filepath)
            mtime = self.ftp.stat(source).st_mtime
        else:
            mtime = os.path.getmtime(filepath)

        return mtime
687669
def _obtain_lock(self):
        """"""
        starttime = time.time()
        maxtime = starttime + float(self._max_block_time)
        while True:
            try:
                super(BlockingLockFile, self)._obtain_lock()
            except IOError:
                # synity check: if the directory leading to the lockfile is not
                # readable anymore, raise an exception
                curtime = time.time()
                if not osp.isdir(osp.dirname(self._lock_file_path())):
                    msg = "Directory containing the lockfile %r was not readable anymore after waiting %g seconds" % (
                        self._lock_file_path(), curtime - starttime)
                    raise IOError(msg)
                # END handle missing directory

                if curtime >= maxtime:
                    msg = "Waited %g seconds for lock at %r" % (maxtime - starttime, self._lock_file_path())
                    raise IOError(msg)
                # END abort if we wait too long
                time.sleep(self._check_interval)
            else:
                break
1217028
def _calculate_bounds(self):
        """"""
        # get start datetime
        first = self.coll_handle.find_one(None, sort=[("ts", ASCENDING)])
        last = self.coll_handle.find_one(None, sort=[("ts", DESCENDING)])

        self._start = first['ts']
        if self._start.tzinfo is None:
            self._start = self._start.replace(tzinfo=tzutc())

        self._end = last['ts']
        if self._end.tzinfo is None:
            self._end = self._end.replace(tzinfo=tzutc())

        return True
420067
def PrintSummary(logfile):
    ''''''
    # Open the log file
    mlog = mavutil.mavlink_connection(filename, notimestamps=args.notimestamps, dialect=args.dialect)

    autonomous_sections = 0 # How many different autonomous sections there are
    autonomous = False # Whether the vehicle is currently autonomous at this point in the logfile
    auto_time = 0.0 # The total time the vehicle was autonomous/guided (seconds)
    start_time = None # The datetime of the first received message (seconds since epoch)
    total_dist = 0.0 # The total ground distance travelled (meters)
    last_gps_msg = None # The first GPS message received
    first_gps_msg = None # The last GPS message received
    true_time = None # Track the first timestamp found that corresponds to a UNIX timestamp

    while True:
        m = mlog.recv_match(condition=args.condition)

        # If there's no match, it means we're done processing the log.
        if m is None:
            break

        # Ignore any failed messages
        if m.get_type() == 'BAD_DATA':
            continue

        # Keep track of the latest timestamp for various calculations
        timestamp = getattr(m, '_timestamp', 0.0)

        # Log the first message time
        if start_time is None:
            start_time = timestamp

        # Log the first timestamp found that is a true timestamp. We first try
        # to get the groundstation timestamp from the log directly. If that fails
        # we try to find a message that outputs a Unix time-since-epoch.
        if true_time is None:
            if not args.notimestamps and timestamp >= 1230768000:
                true_time = timestamp
            elif 'time_unix_usec' in m.__dict__ and m.time_unix_usec >= 1230768000:
                true_time = m.time_unix_usec * 1.0e-6
            elif 'time_usec' in m.__dict__ and m.time_usec >= 1230768000:
                true_time = m.time_usec * 1.0e-6

        # Track the vehicle's speed and status
        if m.get_type() == 'GPS_RAW_INT':

            # Ignore GPS messages without a proper fix
            if m.fix_type < 3 or m.lat == 0 or m.lon == 0:
                continue

            # Log the first GPS location found, being sure to skip GPS fixes
            # that are bad (at lat/lon of 0,0)
            if first_gps_msg is None:
                first_gps_msg = m

            # Track the distance travelled, being sure to skip GPS fixes
            # that are bad (at lat/lon of 0,0)
            if last_gps_msg is None or m.time_usec > last_gps_msg.time_usec or m.time_usec+30e6 < last_gps_msg.time_usec:
                if last_gps_msg is not None:
                    total_dist += distance_two(last_gps_msg, m)

                # Save this GPS message to do simple distance calculations with
                last_gps_msg = m

        elif m.get_type() == 'HEARTBEAT':
            if m.type == mavutil.mavlink.MAV_TYPE_GCS:
                continue
            if (m.base_mode & mavutil.mavlink.MAV_MODE_FLAG_GUIDED_ENABLED or
                m.base_mode & mavutil.mavlink.MAV_MODE_FLAG_AUTO_ENABLED) and autonomous == False:
                autonomous = True
                autonomous_sections += 1
                start_auto_time = timestamp
            elif (not m.base_mode & mavutil.mavlink.MAV_MODE_FLAG_GUIDED_ENABLED and
                not m.base_mode & mavutil.mavlink.MAV_MODE_FLAG_AUTO_ENABLED) and autonomous == True:
                autonomous = False
                auto_time += timestamp - start_auto_time

    # If there were no messages processed, say so
    if start_time is None:
        print("ERROR: No messages found.")
        return

    # If the vehicle ends in autonomous mode, make sure we log the total time
    if autonomous == True:
        auto_time += timestamp - start_auto_time

    # Compute the total logtime, checking that timestamps are 2009 or newer for validity
    # (MAVLink didn't exist until 2009)
    if true_time:
        start_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(true_time))
        print("Log started at about {}".format(start_time_str))
    else:
        print("Warning: No absolute timestamp found in datastream. No starting time can be provided for this log.")

    # Print location data
    if last_gps_msg is not None:
        first_gps_position = (first_gps_msg.lat / 1e7, first_gps_msg.lon / 1e7)
        last_gps_position = (last_gps_msg.lat / 1e7, last_gps_msg.lon / 1e7)
        print("Travelled from ({0[0]}, {0[1]}) to ({1[0]}, {1[1]})".format(first_gps_position, last_gps_position))
        print("Total distance : {:0.2f}m".format(total_dist))
    else:
        print("Warning: No GPS data found, can't give position summary.")

    # Print out the rest of the results.
    total_time = timestamp - start_time
    print("Total time (mm:ss): {:3.0f}:{:02.0f}".format(total_time / 60, total_time % 60))
    # The autonomous time should be good, as a steady HEARTBEAT is required for MAVLink operation
    print("Autonomous sections: {}".format(autonomous_sections))
    if autonomous_sections > 0:
        print("Autonomous time (mm:ss): {:3.0f}:{:02.0f}".format(auto_time / 60, auto_time % 60))

    totals.time += total_time
    totals.distance += total_dist
    totals.flights += 1
1334995
def lock_file(filename, interval=.5, timeout=15):
    '''
    
    '''
    log.trace('Attempting to obtain lock for %s', filename)
    lock = filename + '.lock'
    start = time.time()
    while True:
        if os.path.exists(lock):
            if time.time() - start >= timeout:
                log.warning('Unable to obtain lock for %s', filename)
                return False
            time.sleep(interval)
        else:
            break

    with salt.utils.files.fopen(lock, 'a'):
        pass
377345
def _margtime_loglr(self, mf_snr, opt_snr):
        
        return special.logsumexp(mf_snr, b=self._deltat) - 0.5*opt_snr