Create an RDD for DataFrame from an existing RDD, returns the RDD and schema.
182546
def _createFromRDD(self, rdd, schema, samplingRatio):
        
        if schema is None or isinstance(schema, (list, tuple)):
            struct = self._inferSchema(rdd, samplingRatio, names=schema)
            converter = _create_converter(struct)
            rdd = rdd.map(converter)
            if isinstance(schema, (list, tuple)):
                for i, name in enumerate(schema):
                    struct.fields[i].name = name
                    struct.names[i] = name
            schema = struct

        elif not isinstance(schema, StructType):
            raise TypeError("schema should be StructType or list or None, but got: %s" % schema)

        # convert python objects to sql data
        rdd = rdd.map(schema.toInternal)
        return rdd, schema
153806
def process_schema(doc, resource, df):
    """"""
    from rowgenerators import SourceError
    from requests.exceptions import ConnectionError

    from metapack.cli.core import extract_path_name, alt_col_name, type_map
    from tableintuit import TypeIntuiter
    from rowgenerators.generator.python import PandasDataframeSource
    from appurl import parse_app_url

    try:
        doc['Schema']
    except KeyError:
        doc.new_section('Schema', ['DataType', 'Altname', 'Description'])

    schema_name = resource.get_value('schema', resource.get_value('name'))

    schema_term = doc.find_first(term='Table', value=schema_name, section='Schema')

    if schema_term:
        logger.info("Found table for '{}'; skipping".format(schema_name))
        return

    path, name = extract_path_name(resource.url)

    logger.info("Processing {}".format(resource.url))

    si = PandasDataframeSource(parse_app_url(resource.url), df, cache=doc._cache, )

    try:
        ti = TypeIntuiter().run(si)
    except SourceError as e:
        logger.warn("Failed to process '{}'; {}".format(path, e))
        return
    except ConnectionError as e:
        logger.warn("Failed to download '{}'; {}".format(path, e))
        return

    table = doc['Schema'].new_term('Table', schema_name)

    logger.info("Adding table '{}' to metatab schema".format(schema_name))

    for i, c in enumerate(ti.to_rows()):
        raw_alt_name = alt_col_name(c['header'], i)
        alt_name = raw_alt_name if raw_alt_name != c['header'] else ''

        t = table.new_child('Column', c['header'],
                            datatype=type_map.get(c['resolved_type'], c['resolved_type']),
                            altname=alt_name,
                            description=df[c['header']].description \
                                        if hasattr(df, 'description') and df[c['header']].description else ''
                            )

    return table
1176670
def create(cls, bucket, key, value):
        """"""
        with db.session.begin_nested():
            obj = cls(
                bucket_id=as_bucket_id(bucket),
                key=key,
                value=value
            )
            db.session.add(obj)
        return obj
551395
def prepare(self, data_source):
        
        dataframe = self.__get_dataframe(data_source, use_target=True)
        self.__config.get_data_model().set_features_types_from_dataframe(dataframe)
        dataframe = self.__cleaner.prepare(dataframe)
        return self.__transformer.prepare(dataframe)
444253
def create(cls, train_ds, valid_ds, test_ds=None, path:PathOrStr='.', bs:int=32, val_bs:int=None, pad_idx=1,
               pad_first=True, device:torch.device=None, no_check:bool=False, backwards:bool=False, **dl_kwargs) -> DataBunch:
        ""
        datasets = cls._init_ds(train_ds, valid_ds, test_ds)
        val_bs = ifnone(val_bs, bs)
        collate_fn = partial(pad_collate, pad_idx=pad_idx, pad_first=pad_first, backwards=backwards)
        train_sampler = SortishSampler(datasets[0].x, key=lambda t: len(datasets[0][t][0].data), bs=bs)
        train_dl = DataLoader(datasets[0], batch_size=bs, sampler=train_sampler, drop_last=True, **dl_kwargs)
        dataloaders = [train_dl]
        for ds in datasets[1:]:
            lengths = [len(t) for t in ds.x.items]
            sampler = SortSampler(ds.x, key=lengths.__getitem__)
            dataloaders.append(DataLoader(ds, batch_size=val_bs, sampler=sampler, **dl_kwargs))
        return cls(*dataloaders, path=path, device=device, collate_fn=collate_fn, no_check=no_check)
1214612
def from_schema(cls, schema, handlers={}, **kwargs):
        
        return cls(
            schema.get('$id', schema.get('id', '')) if isinstance(schema, dict) else '',
            schema,
            handlers=handlers,
            **kwargs
        )
421486
def to_instants_dataframe(self, sql_ctx):
        
        ssql_ctx = sql_ctx._ssql_ctx
        jdf = self._jtsrdd.toInstantsDataFrame(ssql_ctx, -1)
        return DataFrame(jdf, sql_ctx)
441269
public CassandraJavaRDD<R> withDescOrder() {
        CassandraRDD<R> newRDD = rdd().withDescOrder();
        return wrap(newRDD);
    }
380380
def _createFromLocal(self, data, schema):
        
        # make sure data could consumed multiple times
        if not isinstance(data, list):
            data = list(data)

        if schema is None or isinstance(schema, (list, tuple)):
            struct = self._inferSchemaFromList(data, names=schema)
            converter = _create_converter(struct)
            data = map(converter, data)
            if isinstance(schema, (list, tuple)):
                for i, name in enumerate(schema):
                    struct.fields[i].name = name
                    struct.names[i] = name
            schema = struct

        elif not isinstance(schema, StructType):
            raise TypeError("schema should be StructType or list or None, but got: %s" % schema)

        # convert python objects to sql data
        data = [schema.toInternal(row) for row in data]
        return self._sc.parallelize(data), schema
1272591
def fromSchema(self, schema):
        ''' 
        '''  
        ns = schema.getTargetNamespace()
        assert self.targetNamespace is None or self.targetNamespace == ns,\
            'SchemaDescription instance represents %s, not %s'\
            %(self.targetNamespace, ns)

        if self.targetNamespace is None:
            self.targetNamespace = ns
 
        self.classHead.ns = self.classFoot.ns = ns
        for item in [t for t in schema.types if t.getAttributeName() not in self.__types]:
            self.__types.append(item.getAttributeName())
            self.items.append(TypeWriter(do_extended=self.do_extended, extPyClasses=self.extPyClasses))
            self.items[-1].fromSchemaItem(item)

        for item in [e for e in schema.elements if e.getAttributeName() not in self.__elements]:
            self.__elements.append(item.getAttributeName())
            self.items.append(ElementWriter(do_extended=self.do_extended))
            self.items[-1].fromSchemaItem(item)
1093238
def db_from_dataframe(
        db_filename,
        table_name,
        df,
        primary_key=None,
        subdir=None,
        overwrite=False,
        indices=(),
        version=1):
    
    return db_from_dataframes(
        db_filename=db_filename,
        dataframes={table_name: df},
        primary_keys={table_name: primary_key},
        indices={table_name: indices},
        subdir=subdir,
        overwrite=overwrite,
        version=version)
172692
def _make_tuple(self, env):
    """"""
    t = runtime.Tuple(self, env, dict2tuple)
    # A tuple also provides its own schema spec
    schema = schema_spec_from_tuple(t)
    t.attach_schema(schema)
    return t
118228
def data2schemacls(_data, **kwargs):
    
    content = {}

    for key in list(kwargs):  # fill kwargs
        kwargs[key] = data2schema(kwargs[key])

    if isinstance(_data, dict):
        datacontent = iteritems(_data)

    else:
        datacontent = getmembers(_data)

    for name, value in datacontent:

        if name[0] == '_':
            continue

        if isinstance(value, dict):
            schema = data2schemacls(value)()

        else:
            schema = data2schema(value)

        content[name] = schema

    content.update(kwargs)  # update content

    result = type('GeneratedSchema', (Schema,), content)

    return result
1329232
def create(
            self,
            overwrite=False):
        
        logger.info("Creating database: %s", self.local_db_path)
        df = self._load_gtf_as_dataframe(
            usecols=self.restrict_gtf_columns,
            features=self.restrict_gtf_features)
        all_index_groups = self._all_possible_indices(df.columns)

        if self.restrict_gtf_features:
            feature_names = self.restrict_gtf_features
        else:
            # split single DataFrame into dictionary mapping each unique
            # feature name onto that subset of the data
            feature_names = df['feature'].unique()
        dataframes = {}
        # every table gets the same set of indices
        indices_dict = {}
        # if a feature has an ID then make it that table's primary key
        primary_keys = {}

        for feature in feature_names:
            df_subset = df[df.feature == feature]
            if len(df_subset) == 0:
                continue
            dataframes[feature] = df_subset

            primary_key = self._get_primary_key(feature, df_subset)
            if primary_key:
                primary_keys[feature] = primary_key

            indices_dict[feature] = self._feature_indices(
                all_index_groups,
                primary_key,
                df_subset)

        self._connection = datacache.db_from_dataframes_with_absolute_path(
            db_path=self.local_db_path,
            table_names_to_dataframes=dataframes,
            table_names_to_primary_keys=primary_keys,
            table_names_to_indices=indices_dict,
            overwrite=overwrite,
            version=DATABASE_SCHEMA_VERSION)
        return self._connection
189300
def to_schema(self):
        
        schema = self._base_schema()
        schema.update(self._root_node.to_schema())
        return schema
965126
def _to_java_object_rdd(rdd):
    
    rdd = rdd._reserialize(AutoBatchedSerializer(PickleSerializer()))
    return rdd.ctx._jvm.org.apache.spark.ml.python.MLSerDe.pythonToJava(rdd._jrdd, True)
366190
def new_db_from_pandas(self, frame, table=None, data=None, load=True, **kwargs):
        

        from ..orm import Column
        # from dbexceptions import ConfigurationError

        # Create the table from the information in the data frame.
        with self.bundle.session:
            sch = self.bundle.schema
            t = sch.new_table(table)

            if frame.index.name:
                id_name = frame.index.name
            else:
                id_name = 'id'

            sch.add_column(t, id_name,
                           datatype=Column.convert_numpy_type(frame.index.dtype),
                           is_primary_key=True)

            for name, type_ in zip([row for row in frame.columns],
                                   [row for row in frame.convert_objects(convert_numeric=True,
                                                                         convert_dates=True).dtypes]):
                sch.add_column(t, name, datatype=Column.convert_numpy_type(type_))
                sch.write_schema()

        p = self.new_partition(table=table, data=data, **kwargs)

        if load:
            pk_name = frame.index.name
            with p.inserter(table) as ins:
                for i, row in frame.iterrows():
                    d = dict(row)
                    d[pk_name] = i

                    ins.insert(d)
        return p