Converts given partition to the document indexed by FTS backend.

        Args:
            partition (orm.Partition): partition to convert.

        Returns:
            dict with structure matches to BasePartitionIndex._schema.
422930
def _as_document(self, partition):
        

        schema = ' '.join(
            u'{} {} {} {} {}'.format(
                c.id,
                c.vid,
                c.name,
                c.altname,
                c.description) for c in partition.table.columns)

        values = ''

        for stat in partition.stats:
            if stat.uvalues :
                # SOme geometry vlaues are super long. They should not be in uvbalues, but when they are,
                # need to cut them down.
                values += ' '.join(e[:200] for e in stat.uvalues) + '\n'

        # Re-calculate the summarization of grains, since the geoid 0.0.7 package had a bug where state level
        # summaries had the same value as state-level allvals
        def resum(g):
            try:
                return str(GVid.parse(g).summarize())
            except KeyError:
                return g
            except ValueError:
                logger.debug("Failed to parse gvid '{}' from partition '{}' grain coverage"
                             .format(g, partition.identity.vname))
                return g

        keywords = (
            ' '.join(partition.space_coverage) + ' ' +
            ' '.join([resum(g) for g in partition.grain_coverage if resum(g)]) + ' ' +
            ' '.join(str(x) for x in partition.time_coverage)
        )

        doc_field = u('{} {} {} {} {} {}').format(
            values,
            schema,
            ' '.join([
                u('{}').format(partition.identity.vid),
                u('{}').format(partition.identity.id_),
                u('{}').format(partition.identity.name),
                u('{}').format(partition.identity.vname)]),
            partition.display.title,
            partition.display.description,
            partition.display.sub_description,
            partition.display.time_description,
            partition.display.geo_description
        )

        document = dict(
            vid=u('{}').format(partition.identity.vid),
            dataset_vid=u('{}').format(partition.identity.as_dataset().vid),
            title=u('{}').format(partition.table.description),
            keywords=u('{}').format(keywords),
            doc=doc_field)

        return document
1194936
def schema_to_json(self, schema_list, destination):
        
        json_schema_list = [f.to_api_repr() for f in schema_list]

        if isinstance(destination, io.IOBase):
            return self._schema_to_json_file_object(json_schema_list, destination)

        with open(destination, mode="w") as file_obj:
            return self._schema_to_json_file_object(json_schema_list, file_obj)
1077290
def build_input_partitions(cls, name='inputTablePartitions', input_name='input'):
        
        obj = cls(name)
        obj.exporter = 'get_input_partitions'
        obj.input_name = input_name
        return obj
841826
def partition(self, ref=None, **kwargs):
        """"""
        from ambry.orm.exc import NotFoundError
        from sqlalchemy.orm.exc import NoResultFound

        if not ref and not kwargs:
            return None

        if ref:
            for p in self.partitions:
                if ref == p.name or ref == p.vname or ref == p.vid or ref == p.id:
                  p._bundle = self
                  return p

            raise NotFoundError("No partition found for '{}' (a)".format(ref))

        elif kwargs:
            from ..identity import PartitionNameQuery
            pnq = PartitionNameQuery(**kwargs)
            try:
                p = self.partitions._find_orm(pnq).one()
                if p:
                    p._bundle = self
                    return p
            except NoResultFound:
                raise NotFoundError("No partition found for '{}' (b)".format(kwargs))
247519
def search(self, search_phrase, limit=None):
        

        query_string = self._make_query_from_terms(search_phrase)
        self._parsed_query = query_string
        schema = self._get_generic_schema()
        parser = QueryParser('doc', schema=schema)
        query = parser.parse(query_string)
        logger.debug('Searching partitions using `{}` query.'.format(query))
        with self.index.searcher() as searcher:
            results = searcher.search(query, limit=limit)
            for hit in results:
                yield PartitionSearchResult(
                    vid=hit['vid'], dataset_vid=hit['dataset_vid'], score=hit.score)
486391
def to_document(self):
        
        return dict(
            info=thaw(self.info),
            paths=thaw(self.paths),
            definitions=thaw(self.definitions),
            securityDefinitions=thaw(self.securityDefinitions),
            security=thaw(self.security),
            swagger=thaw(self.swagger),
        )
71356
def partition(self, mapped_values):
        
        partitioned_data = collections.defaultdict(list)
        for key, value in mapped_values:
            partitioned_data[key].append(value)
        return partitioned_data.items()
614118
def merge_partition(self, partition, path, value):
        
        dct = self.partitions[partition]
        *heads, tail = path
        for part in heads:
            dct = dct.setdefault(part, dict())
        dct[tail] = value
1061649
def mapPartitionsWithIndex(self, f, preservesPartitioning=False):
        
        return self.transform(lambda rdd: rdd.mapPartitionsWithIndex(f, preservesPartitioning))
1049279
def _partition(hours, partition = 3600.0):
		
		partition = float(partition)
		relative = hours - hours[0]
		total_partitions = np.ceil(relative[-1] / partition + 10*np.finfo(np.float).eps).astype('int')
		return [hours[np.floor(np.divide(relative, partition)) == i] for i in range(total_partitions)]
427343
private static List<TopicPartition> convertKafkaPartitions(List<KafkaTopicPartitionState<TopicPartition>> partitions) {
		ArrayList<TopicPartition> result = new ArrayList<>(partitions.size());
		for (KafkaTopicPartitionState<TopicPartition> p : partitions) {
			result.add(p.getKafkaPartitionHandle());
		}
		return result;
	}
292419
def position(self, partition):
        
        if not isinstance(partition, TopicPartition):
            raise TypeError('partition must be a TopicPartition namedtuple')
        assert self._subscription.is_assigned(partition), 'Partition is not assigned'
        offset = self._subscription.assignment[partition].position
        if offset is None:
            self._update_fetch_positions([partition])
            offset = self._subscription.assignment[partition].position
        return offset
77652
def export_schema_to_dict(back_references):
    """"""
    databases = [Database.export_schema(recursive=True,
                 include_parent_ref=back_references)]
    clusters = [DruidCluster.export_schema(recursive=True,
                include_parent_ref=back_references)]
    data = dict()
    if databases:
        data[DATABASES_KEY] = databases
    if clusters:
        data[DRUID_CLUSTERS_KEY] = clusters
    return data
1034098
def from_dict(self, document):
        
        # Get object properties from Json document
        identifier = str(document['_id'])
        active = document['active']
        timestamp = datetime.datetime.strptime(document['timestamp'], '%Y-%m-%dT%H:%M:%S.%f')
        properties = document['properties']
        # The directory is not materilaized in database to allow moving the
        # base directory without having to update the database.
        directory = self.get_directory(identifier)
        # Cretae image handle
        return ImageHandle(identifier, properties, directory, timestamp=timestamp, is_active=active)
749087
private DBObject createQueryPartition(MongoPartition partition) {

        QueryBuilder queryBuilderMin = QueryBuilder.start(partition.getKey());
        DBObject bsonObjectMin = queryBuilderMin.greaterThanEquals(partition.splitWrapper().getStartToken()).get();

        QueryBuilder queryBuilderMax = QueryBuilder.start(partition.getKey());
        DBObject bsonObjectMax = queryBuilderMax.lessThan(partition.splitWrapper().getEndToken()).get();

        QueryBuilder queryBuilder = QueryBuilder.start();
        if (partition.splitWrapper().getStartToken() != null) {
            queryBuilder.and(bsonObjectMin);
        }

        if (partition.splitWrapper().getEndToken() != null) {
            queryBuilder.and(bsonObjectMax);
        }

        LOG.debug("mongodb query "+queryBuilder.get());

        return queryBuilder.get();
    }
780412
def schema(self):
        
        if self.status == "DELETING":
            return ""
        parts = ["GLOBAL", self.index_type, "INDEX"]
        parts.append("('%s', %s," % (self.name, self.hash_key.name))
        if self.range_key:
            parts.append("%s," % self.range_key.name)
        if self.includes:
            parts.append("[%s]," % ", ".join(("'%s'" % i for i in self.includes)))

        parts.append(
            "THROUGHPUT (%d, %d))" % (self.read_throughput, self.write_throughput)
        )
        return " ".join(parts)
1351268
def _get_disorder_mappings(composition, partitions):
        

        def _get_replacement_dict_from_partition(partition):
            d = {}  # to be passed to Structure.replace_species()
            for sp_list in partition:
                if len(sp_list) > 1:
                    total_occ = sum([composition[sp] for sp in sp_list])
                    merged_comp = {sp: composition[sp] / total_occ for sp in sp_list}
                    for sp in sp_list:
                        d[sp] = merged_comp
            return d

        disorder_mapping = [_get_replacement_dict_from_partition(p)
                            for p in partitions]

        return disorder_mapping