Convert data into an edge data sframe. Using src_field and dst_field to
    identify the source and target id column. The returned sframe will have id
    column name '__src_id', '__dst_id'
184766
def _edge_data_to_sframe(data, src_field, dst_field):
    
    if isinstance(data, SFrame):
        # '__src_vid' and '__dst_vid' already in the sframe, and
        # it is ok to not specify src_field and dst_field
        if src_field is None and dst_field is None and \
           _SRC_VID_COLUMN in data.column_names() and \
           _DST_VID_COLUMN in data.column_names():
            return data
        if src_field is None:
            raise ValueError("src_field must be specified for SFrame input")
        if dst_field is None:
            raise ValueError("dst_field must be specified for SFrame input")
        data_copy = copy.copy(data)
        if src_field == _DST_VID_COLUMN and dst_field == _SRC_VID_COLUMN:
            # special case when src_field = "__dst_id" and dst_field = "__src_id"
            # directly renaming will cause name collision
            dst_id_column = data_copy[_DST_VID_COLUMN]
            del data_copy[_DST_VID_COLUMN]
            data_copy.rename({_SRC_VID_COLUMN: _DST_VID_COLUMN}, inplace=True)
            data_copy[_SRC_VID_COLUMN] = dst_id_column
        else:
            data_copy.rename({src_field: _SRC_VID_COLUMN, dst_field: _DST_VID_COLUMN}, inplace=True)
        return data_copy

    elif HAS_PANDAS and type(data) == pd.DataFrame:
        if src_field is None:
            raise ValueError("src_field must be specified for Pandas input")
        if dst_field is None:
            raise ValueError("dst_field must be specified for Pandas input")
        sf = SFrame(data)
        if src_field == _DST_VID_COLUMN and dst_field == _SRC_VID_COLUMN:
            # special case when src_field = "__dst_id" and dst_field = "__src_id"
            # directly renaming will cause name collision
            dst_id_column = data_copy[_DST_VID_COLUMN]
            del sf[_DST_VID_COLUMN]
            sf.rename({_SRC_VID_COLUMN: _DST_VID_COLUMN}, inplace=True)
            sf[_SRC_VID_COLUMN] = dst_id_column
        else:
            sf.rename({src_field: _SRC_VID_COLUMN, dst_field: _DST_VID_COLUMN}, inplace=True)
        return sf

    elif type(data) == Edge:
        return _edge_list_to_sframe([data], _SRC_VID_COLUMN, _DST_VID_COLUMN)

    elif type(data) == list:
        return _edge_list_to_sframe(data, _SRC_VID_COLUMN, _DST_VID_COLUMN)

    else:
        raise TypeError('Edges type %s is Not supported.' % str(type(data)))
878315
def as_iframe(self, html_data):
        """"""

        srcdoc = html_data.replace('"', "'")
        return ('<iframe id="{div_id}", srcdoc="{srcdoc}" style="width: {width}; '
                'height: {height};"></iframe>'.format(
                    div_id=self.div_id,
                    srcdoc=srcdoc,
                    width=self.width,
                    height=self.height))
733679
def i2s_frameid(x):
    
    try:
        return PNIO_FRAME_IDS[x]
    except KeyError:
        pass
    if 0x0100 <= x < 0x1000:
        return "RT_CLASS_3 (%4x)" % x
    if 0x8000 <= x < 0xC000:
        return "RT_CLASS_1 (%4x)" % x
    if 0xC000 <= x < 0xFC00:
        return "RT_CLASS_UDP (%4x)" % x
    if 0xFF80 <= x < 0xFF90:
        return "FragmentationFrameID (%4x)" % x
    return x
770865
def to_string(self):
        

        src = self.parse_node_ref( self.get_source() )
        dst = self.parse_node_ref( self.get_destination() )
        
        if isinstance(src, frozendict):
            edge = [ Subgraph(obj_dict=src).to_string() ]
        elif isinstance(src, (int, long)):
            edge = [ str(src) ]
        else:
            edge = [ src ]
        
        if	(self.get_parent_graph() and
            self.get_parent_graph().get_top_graph_type() and
            self.get_parent_graph().get_top_graph_type() == 'digraph' ):

            edge.append( '->' )
            
        else:
            edge.append( '--' )
            
        if isinstance(dst, frozendict):
            edge.append( Subgraph(obj_dict=dst).to_string() )
        elif isinstance(dst, (int, long)):
            edge.append( str(dst) )
        else:
            edge.append( dst )


        edge_attr = list()
        
        for attr, value in self.obj_dict['attributes'].iteritems():
        
            if value is not None:
                edge_attr.append( '%s=%s' % (attr, quote_if_necessary(value) ) )
            else:
                edge_attr.append( attr )

        edge_attr = ', '.join(edge_attr)
        
        if edge_attr:
            edge.append( ' [' + edge_attr + ']' )

        return ' '.join(edge) + ';'
1316826
def _vertex_list_to_dataframe(ls, id_column_name):
    
    assert HAS_PANDAS, 'Cannot use dataframe because Pandas is not available or version is too low.'
    cols = reduce(set.union, (set(v.attr.keys()) for v in ls))
    df = pd.DataFrame({id_column_name: [v.vid for v in ls]})
    for c in cols:
        df[c] = [v.attr.get(c) for v in ls]
    return df
1121871
def _edge_list_to_dataframe(ls, src_column_name, dst_column_name):
    
    assert HAS_PANDAS, 'Cannot use dataframe because Pandas is not available or version is too low.'
    cols = reduce(set.union, (set(e.attr.keys()) for e in ls))
    df = pd.DataFrame({
        src_column_name: [e.src_vid for e in ls],
        dst_column_name: [e.dst_vid for e in ls]})
    for c in cols:
        df[c] = [e.attr.get(c) for e in ls]
    return df
553091
def _target_id_from_htm(self, line):
        
        # Detecting modern mbed.htm file format
        m = re.search("\\?code=([a-fA-F0-9]+)", line)
        if m:
            result = m.groups()[0]
            return result
        # Last resort, we can try to see if old mbed.htm format is there
        m = re.search("\\?auth=([a-fA-F0-9]+)", line)
        if m:
            result = m.groups()[0]
            return result

        return None
313458
function(src, dst) {
    dst = dst || {};

    // Remove any properties in destination that were not
    // returned from the source
    forEach(dst, function (value, key) {
        if(!src.hasOwnProperty(key) && key.charAt(0) !== '$') {
            delete dst[key];
        }
    });

    for(var key in src) {

        if(src.hasOwnProperty(key) && key.charAt(0) !== '$')  {
            // For properties common to both source and destination,
            // check for object references and recurse as needed. Route around
            // arrays to prevent value/order inconsistencies
            if(angular.isObject(src[key]) && !angular.isArray(src[key])) {
                dst[key] = shallowClearAndCopy(src[key], dst[key]);
            } else {
                // Not an object, so just overwrite with value from source
                dst[key] = src[key];
            }
        }
    }

    return dst;
}
902487
function convertRawDate(rawColumnValue, column, context)
{
  return new SfTimestamp(
      Number(rawColumnValue) * 86400, // convert to seconds
      0,                              // no nano seconds
      0,                              // no scale required
      'UTC',                          // use utc as the timezone
      context.format).toSfDate();
}
498283
def _get_mstd(cls, df, column, windows):
        
        window = cls.get_only_one_positive_int(windows)
        column_name = '{}_{}_mstd'.format(column, window)
        df[column_name] = df[column].rolling(min_periods=1, window=window,
                                             center=False).std()
833860
def process_dns_frame(self,
                          id=None,
                          msg=None):
        

        # normalize into a dataframe
        df = json_normalize(msg)
        # convert to a flattened dictionary
        dt = json.loads(df.to_json())

        flat_msg = {}

        for k in dt:
            new_key = "dns_{}".format(k)
            flat_msg[new_key] = dt[k]["0"]
            if new_key not in self.dns_keys:
                self.dns_keys[new_key] = k
        # end of capturing all unique keys

        dt["dns_id"] = id
        self.all_dns.append(dt)

        log.debug("DNS data updated:")
        log.debug(self.dns_keys)
        log.debug(self.all_dns)
        log.debug("")

        return flat_msg
15812
def _init_edges(self, dst_srcs_list):
        """"""
        from goatools.gosubdag.go_paths import get_paths_goobjs, paths2edges
        edges_all = set()
        goid_all = set()
        go2obj = self.go2obj
        for dst, srcs in dst_srcs_list:
            go2obj_srcs = {}
            for goid in srcs:
                go2obj_srcs[goid] = go2obj[goid]
            go_paths, go_all = get_paths_goobjs(go2obj_srcs.values(), go_top=dst, go2obj=go2obj)
            edges_all |= paths2edges(go_paths)
            goid_all |= go_all
        self.edges = [(a.id, b.id) for a, b in edges_all]
        self.goid_all = goid_all
630361
def _translate_src_mode_to_dst_mode(self, src_mode):
        # type: (SyncCopy, blobxfer.models.azure.StorageModes) -> bool
        
        if (self._spec.options.dest_mode ==
                blobxfer.models.azure.StorageModes.Auto):
            return src_mode
        else:
            return self._spec.options.dest_mode
106363
def getDMCframe(f, iFrm: int, finf: dict, verbose: bool=False):
    
    # on windows, "int" is int32 and overflows at 2.1GB!  We need np.int64
    currByte = iFrm * finf['bytesperframe']
# %% advance to start of frame in bytes
    if verbose:
        print(f'seeking to byte {currByte}')

    assert isinstance(iFrm, (int, int64)), 'int32 will fail on files > 2GB'

    try:
        f.seek(currByte, 0)
    except IOError as e:
        raise IOError(f'I couldnt seek to byte {currByte:d}. try using a 64-bit integer for iFrm \n'
                      'is {f.name} a DMCdata file?  {e}')
# %% read data ***LABVIEW USES ROW-MAJOR C ORDERING!!
    try:
        currFrame = fromfile(f, uint16,
                             finf['pixelsperimage']).reshape((finf['supery'], finf['superx']),
                                                             order='C')
    except ValueError as e:
        raise ValueError(f'read past end of file?  {e}')

    rawFrameInd = meta2rawInd(f, finf['nmetadata'])

    if rawFrameInd is None:  # 2011 no metadata file
        rawFrameInd = iFrm + 1  # fallback

    return currFrame, rawFrameInd
587300
def format_tsv_line(source, edge, target, value=None, metadata=None):
    
    return '{source}\t{edge}\t{target}\t{value}\t{metadata}'.format(
        source=source,
        edge=edge,
        target=target,
        value='{:.4f}'.format(value) if value is not None else '',
        metadata=metadata or ''
    ).rstrip(' \t')
1025090
def _append_dataframe(self, df, source_info="", units=None):
        

        units = units or {}

        t = df.index
        index_name = df.index.name
        time_name = index_name or "time"

        version = self.version

        timestamps = t

        # if self.version < '3.10':
        #     if timestamps.dtype.byteorder == '>':
        #         timestamps = timestamps.byteswap().newbyteorder()
        #     for signal in signals:
        #         if signal.samples.dtype.byteorder == '>':
        #             signal.samples = signal.samples.byteswap().newbyteorder()

        if self.version >= "3.00":
            channel_size = v23c.CN_DISPLAYNAME_BLOCK_SIZE
        elif self.version >= "2.10":
            channel_size = v23c.CN_LONGNAME_BLOCK_SIZE
        else:
            channel_size = v23c.CN_SHORT_BLOCK_SIZE

        file = self._tempfile
        tell = file.tell

        kargs = {
            "module_nr": 0,
            "module_address": 0,
            "type": v23c.SOURCE_ECU,
            "description": b"Channel inserted by Python Script",
        }
        ce_block = ChannelExtension(**kargs)

        dg_cntr = len(self.groups)

        gp = {}
        gp.channels = gp_channels = []
        gp.channel_dependencies = gp_dep = []
        gp.signal_types = gp_sig_types = []
        gp.string_dtypes = []

        self.groups.append(gp)

        cycles_nr = len(timestamps)
        fields = []
        types = []
        parents = {}
        ch_cntr = 0
        offset = 0
        field_names = UniqueDB()

        if df.shape[0]:
            # conversion for time channel
            kargs = {
                "conversion_type": v23c.CONVERSION_TYPE_NONE,
                "unit": b"s",
                "min_phy_value": timestamps[0] if cycles_nr else 0,
                "max_phy_value": timestamps[-1] if cycles_nr else 0,
            }
            conversion = ChannelConversion(**kargs)
            conversion.unit = "s"
            source = ce_block

            # time channel
            t_type, t_size = fmt_to_datatype_v3(timestamps.dtype, timestamps.shape)
            kargs = {
                "short_name": time_name.encode("latin-1"),
                "channel_type": v23c.CHANNEL_TYPE_MASTER,
                "data_type": t_type,
                "start_offset": 0,
                "min_raw_value": timestamps[0] if cycles_nr else 0,
                "max_raw_value": timestamps[-1] if cycles_nr else 0,
                "bit_count": t_size,
                "block_len": channel_size,
                "version": version,
            }
            channel = Channel(**kargs)
            channel.name = name = time_name
            channel.conversion = conversion
            channel.source = source

            gp_channels.append(channel)

            self.channels_db.add(name, (dg_cntr, ch_cntr))
            self.masters_db[dg_cntr] = 0
            # data group record parents
            parents[ch_cntr] = name, 0

            # time channel doesn't have channel dependencies
            gp_dep.append(None)

            fields.append(timestamps)
            types.append((name, timestamps.dtype))
            field_names.add(name)

            offset += t_size
            ch_cntr += 1

            gp_sig_types.append(0)

        for signal in df:

            sig = df[signal]
            name = signal

            sig_type = v23c.SIGNAL_TYPE_SCALAR

            gp_sig_types.append(sig_type)

            new_source = ce_block

            # compute additional byte offset for large records size
            if offset > v23c.MAX_UINT16:
                additional_byte_offset = ceil((offset - v23c.MAX_UINT16) / 8)
                start_bit_offset = offset - additional_byte_offset * 8
            else:
                start_bit_offset = offset
                additional_byte_offset = 0

            s_type, s_size = fmt_to_datatype_v3(sig.dtype, sig.shape)

            kargs = {
                "channel_type": v23c.CHANNEL_TYPE_VALUE,
                "data_type": s_type,
                "min_raw_value": 0,
                "max_raw_value": 0,
                "start_offset": start_bit_offset,
                "bit_count": s_size,
                "additional_byte_offset": additional_byte_offset,
                "block_len": channel_size,
                "version": version,
            }

            if s_size < 8:
                s_size = 8

            channel = Channel(**kargs)
            channel.name = name
            channel.source = new_source

            unit = units.get(name, b"")
            if unit:
                if hasattr(unit, "encode"):
                    unit = unit.encode("latin-1")
                # conversion for time channel
                kargs = {
                    "conversion_type": v23c.CONVERSION_TYPE_NONE,
                    "unit": unit,
                    "min_phy_value": 0,
                    "max_phy_value": 0,
                }
                conversion = ChannelConversion(**kargs)
                conversion.unit = unit

            gp_channels.append(channel)

            offset += s_size

            self.channels_db.add(name, (dg_cntr, ch_cntr))

            # update the parents as well
            field_name = field_names.get_unique_name(name)
            parents[ch_cntr] = field_name, 0

            if sig.dtype.kind == "S":
                gp.string_dtypes.append(sig.dtype)

            fields.append(sig)
            types.append((field_name, sig.dtype))

            ch_cntr += 1

            # simple channels don't have channel dependencies
            gp_dep.append(None)

        # channel group
        kargs = {
            "cycles_nr": cycles_nr,
            "samples_byte_nr": offset // 8,
            "ch_nr": ch_cntr,
        }
        if self.version >= "3.30":
            kargs["block_len"] = v23c.CG_POST_330_BLOCK_SIZE
        else:
            kargs["block_len"] = v23c.CG_PRE_330_BLOCK_SIZE
        gp.channel_group = ChannelGroup(**kargs)
        gp.channel_group.comment = source_info

        # data group
        if self.version >= "3.20":
            block_len = v23c.DG_POST_320_BLOCK_SIZE
        else:
            block_len = v23c.DG_PRE_320_BLOCK_SIZE
        gp.data_group = DataGroup(block_len=block_len)

        # data block
        types = dtype(types)

        gp.types = types
        gp.parents = parents
        gp.sorted = True

        if df.shape[0]:
            samples = fromarrays(fields, dtype=types)
        else:
            samples = array([])

        block = samples.tostring()

        gp.data_location = v23c.LOCATION_TEMPORARY_FILE
        if cycles_nr:
            data_address = tell()
            gp.data_group.data_block_addr = data_address
            size = len(block)
            self._tempfile.write(block)

            gp.data_blocks.append(
                DataBlockInfo(
                    address=data_address,
                    block_type=0,
                    raw_size=size,
                    size=size,
                    param=0,
                )
            )
        else:
            gp.data_location = v23c.LOCATION_TEMPORARY_FILE
        # data group trigger
        gp.trigger = None
1329221
def to_data_frame(sc, features, labels, categorical=False):
    
    lp_rdd = to_labeled_point(sc, features, labels, categorical)
    sql_context = SQLContext(sc)
    df = sql_context.createDataFrame(lp_rdd)
    return df