Convert descriptor to BigQuery
188030
def convert_descriptor(self, descriptor):
        

        # Fields
        fields = []
        fallbacks = []
        schema = tableschema.Schema(descriptor)
        for index, field in enumerate(schema.fields):
            converted_type = self.convert_type(field.type)
            if not converted_type:
                converted_type = 'STRING'
                fallbacks.append(index)
            mode = 'NULLABLE'
            if field.required:
                mode = 'REQUIRED'
            fields.append({
                'name': _slugify_field_name(field.name),
                'type': converted_type,
                'mode': mode,
            })

        # Descriptor
        converted_descriptor = {
            'fields': fields,
        }

        return (converted_descriptor, fallbacks)
37758
def get_descriptor(ctx):
    
    d_net = gluon.nn.Sequential()
    with d_net.name_scope():

        d_net.add(SNConv2D(num_filter=64, kernel_size=4, strides=2, padding=1, in_channels=3, ctx=ctx))
        d_net.add(gluon.nn.LeakyReLU(0.2))

        d_net.add(SNConv2D(num_filter=128, kernel_size=4, strides=2, padding=1, in_channels=64, ctx=ctx))
        d_net.add(gluon.nn.LeakyReLU(0.2))

        d_net.add(SNConv2D(num_filter=256, kernel_size=4, strides=2, padding=1, in_channels=128, ctx=ctx))
        d_net.add(gluon.nn.LeakyReLU(0.2))

        d_net.add(SNConv2D(num_filter=512, kernel_size=4, strides=2, padding=1, in_channels=256, ctx=ctx))
        d_net.add(gluon.nn.LeakyReLU(0.2))

        d_net.add(SNConv2D(num_filter=1, kernel_size=4, strides=1, padding=0, in_channels=512, ctx=ctx))

    return d_net
902628
function loadPresetDescriptor(descriptor) {
  return {
    type: "preset",
    options: loadDescriptor(descriptor).value,
    alias: descriptor.alias,
    loc: descriptor.loc,
    dirname: descriptor.dirname
  };
}
731259
def to_bigquery_field(self, name_case=DdlParseBase.NAME_CASE.original):
        """"""

        col_name = self.get_name(name_case)
        mode = self.bigquery_mode

        if self.array_dimensional <= 1:
            # no or one dimensional array data type
            type = self.bigquery_legacy_data_type

        else:
            # multiple dimensional array data type
            type = "RECORD"

            fields = OrderedDict()
            fields_cur = fields

            for i in range(1, self.array_dimensional):
                is_last = True if i == self.array_dimensional - 1 else False

                fields_cur['fields'] = [OrderedDict()]
                fields_cur = fields_cur['fields'][0]

                fields_cur['name'] = "dimension_{}".format(i)
                fields_cur['type'] = self.bigquery_legacy_data_type if is_last else "RECORD"
                fields_cur['mode'] = self.bigquery_mode if is_last else "REPEATED"

        col = OrderedDict()
        col['name'] = col_name
        col['type'] = type
        col['mode'] = mode
        if self.array_dimensional > 1:
            col['fields'] = fields['fields']

        return json.dumps(col)
850291
def restore_descriptor(self, table_name, columns, constraints, autoincrement_column=None):
        

        # Fields
        fields = []
        for column in columns:
            if column.name == autoincrement_column:
                continue
            field_type = self.restore_type(column.type)
            field = {'name': column.name, 'type': field_type}
            if not column.nullable:
                field['constraints'] = {'required': True}
            fields.append(field)

        # Primary key
        pk = []
        for constraint in constraints:
            if isinstance(constraint, sa.PrimaryKeyConstraint):
                for column in constraint.columns:
                    if column.name == autoincrement_column:
                        continue
                    pk.append(column.name)

        # Foreign keys
        fks = []
        if self.__dialect == 'postgresql':
            for constraint in constraints:
                if isinstance(constraint, sa.ForeignKeyConstraint):
                    resource = ''
                    own_fields = []
                    foreign_fields = []
                    for element in constraint.elements:
                        own_fields.append(element.parent.name)
                        if element.column.table.name != table_name:
                            resource = self.restore_bucket(element.column.table.name)
                        foreign_fields.append(element.column.name)
                    if len(own_fields) == len(foreign_fields) == 1:
                        own_fields = own_fields.pop()
                        foreign_fields = foreign_fields.pop()
                    fks.append({
                        'fields': own_fields,
                        'reference': {'resource': resource, 'fields': foreign_fields},
                    })

        # Desscriptor
        descriptor = {}
        descriptor['fields'] = fields
        if len(pk) > 0:
            if len(pk) == 1:
                pk = pk.pop()
            descriptor['primaryKey'] = pk
        if len(fks) > 0:
            descriptor['foreignKeys'] = fks

        return descriptor
195775
def _property_descriptor_to_graphql_type(property_obj):
    """"""
    property_type = property_obj.type_id
    scalar_types = {
        PROPERTY_TYPE_BOOLEAN_ID: GraphQLBoolean,
        PROPERTY_TYPE_DATE_ID: GraphQLDate,
        PROPERTY_TYPE_DATETIME_ID: GraphQLDateTime,
        PROPERTY_TYPE_DECIMAL_ID: GraphQLDecimal,
        PROPERTY_TYPE_DOUBLE_ID: GraphQLFloat,
        PROPERTY_TYPE_FLOAT_ID: GraphQLFloat,
        PROPERTY_TYPE_INTEGER_ID: GraphQLInt,
        PROPERTY_TYPE_STRING_ID: GraphQLString,
    }

    result = scalar_types.get(property_type, None)
    if result:
        return result

    mapping_types = {
        PROPERTY_TYPE_EMBEDDED_SET_ID: GraphQLList,
        PROPERTY_TYPE_EMBEDDED_LIST_ID: GraphQLList,
    }
    wrapping_type = mapping_types.get(property_type, None)
    if wrapping_type:
        linked_property_obj = property_obj.qualifier
        # There are properties that are embedded collections of non-primitive types,
        # for example, ProxyEventSet.scalar_parameters.
        # The GraphQL compiler does not currently support these.
        if linked_property_obj in scalar_types:
            return wrapping_type(scalar_types[linked_property_obj])

    # We weren't able to represent this property in GraphQL, so we'll hide it instead.
    return None
1274775
private static int getColumnWidth(final PropertyDescriptor _property) {
        final Class type = _property.getPropertyType();
        if (Float.class.isAssignableFrom(type) || Double.class.isAssignableFrom(type)) {
            return 70;
        } else if (type == Boolean.class) {
            return 10;
        } else if (Number.class.isAssignableFrom(type)) {
            return 60;
        } else if (type == String.class) {
            return 100;
        } else if (Date.class.isAssignableFrom(type)) {
            return 50;
        } else {
            return 50;
        }
    }
893750
private QueryByCriteria getFKQuery1toN(Object obj, ClassDescriptor cld, CollectionDescriptor cod)
    {
        ValueContainer[] container = pb.serviceBrokerHelper().getKeyValues(cld, obj);
        ClassDescriptor refCld = pb.getClassDescriptor(cod.getItemClass());
        FieldDescriptor[] fields = cod.getForeignKeyFieldDescriptors(refCld);
        Criteria criteria = new Criteria();

        for (int i = 0; i < fields.length; i++)
        {
            FieldDescriptor fld = fields[i];
            criteria.addEqualTo(fld.getAttributeName(), container[i].getValue());
        }

        return QueryFactory.newQuery(refCld.getClassOfObject(), criteria);
    }
342703
def run_query(  # noqa / druid
            self,
            groupby, metrics,
            granularity,
            from_dttm, to_dttm,
            filter=None,  # noqa
            is_timeseries=True,
            timeseries_limit=None,
            timeseries_limit_metric=None,
            row_limit=None,
            inner_from_dttm=None, inner_to_dttm=None,
            orderby=None,
            extras=None,  # noqa
            columns=None, phase=2, client=None,
            order_desc=True,
            prequeries=None,
            is_prequery=False,
        ):
        
        # TODO refactor into using a TBD Query object
        client = client or self.cluster.get_pydruid_client()
        row_limit = row_limit or conf.get('ROW_LIMIT')

        if not is_timeseries:
            granularity = 'all'

        if granularity == 'all':
            phase = 1
        inner_from_dttm = inner_from_dttm or from_dttm
        inner_to_dttm = inner_to_dttm or to_dttm

        timezone = from_dttm.replace(tzinfo=DRUID_TZ).tzname() if from_dttm else None

        query_str = ''
        metrics_dict = {m.metric_name: m for m in self.metrics}
        columns_dict = {c.column_name: c for c in self.columns}

        if (
            self.cluster and
            LooseVersion(self.cluster.get_druid_version()) < LooseVersion('0.11.0')
        ):
            for metric in metrics:
                self.sanitize_metric_object(metric)
            self.sanitize_metric_object(timeseries_limit_metric)

        aggregations, post_aggs = DruidDatasource.metrics_and_post_aggs(
            metrics,
            metrics_dict)

        self.check_restricted_metrics(aggregations)

        # the dimensions list with dimensionSpecs expanded
        dimensions = self.get_dimensions(groupby, columns_dict)
        extras = extras or {}
        qry = dict(
            datasource=self.datasource_name,
            dimensions=dimensions,
            aggregations=aggregations,
            granularity=DruidDatasource.granularity(
                granularity,
                timezone=timezone,
                origin=extras.get('druid_time_origin'),
            ),
            post_aggregations=post_aggs,
            intervals=self.intervals_from_dttms(from_dttm, to_dttm),
        )

        filters = DruidDatasource.get_filters(filter, self.num_cols, columns_dict)
        if filters:
            qry['filter'] = filters

        having_filters = self.get_having_filters(extras.get('having_druid'))
        if having_filters:
            qry['having'] = having_filters

        order_direction = 'descending' if order_desc else 'ascending'

        if columns:
            columns.append('__time')
            del qry['post_aggregations']
            del qry['aggregations']
            qry['dimensions'] = columns
            qry['metrics'] = []
            qry['granularity'] = 'all'
            qry['limit'] = row_limit
            client.scan(**qry)
        elif len(groupby) == 0 and not having_filters:
            logging.info('Running timeseries query for no groupby values')
            del qry['dimensions']
            client.timeseries(**qry)
        elif (
                not having_filters and
                len(groupby) == 1 and
                order_desc
        ):
            dim = list(qry.get('dimensions'))[0]
            logging.info('Running two-phase topn query for dimension [{}]'.format(dim))
            pre_qry = deepcopy(qry)
            if timeseries_limit_metric:
                order_by = utils.get_metric_name(timeseries_limit_metric)
                aggs_dict, post_aggs_dict = DruidDatasource.metrics_and_post_aggs(
                    [timeseries_limit_metric],
                    metrics_dict)
                if phase == 1:
                    pre_qry['aggregations'].update(aggs_dict)
                    pre_qry['post_aggregations'].update(post_aggs_dict)
                else:
                    pre_qry['aggregations'] = aggs_dict
                    pre_qry['post_aggregations'] = post_aggs_dict
            else:
                agg_keys = qry['aggregations'].keys()
                order_by = list(agg_keys)[0] if agg_keys else None

            # Limit on the number of timeseries, doing a two-phases query
            pre_qry['granularity'] = 'all'
            pre_qry['threshold'] = min(row_limit,
                                       timeseries_limit or row_limit)
            pre_qry['metric'] = order_by
            pre_qry['dimension'] = self._dimensions_to_values(qry.get('dimensions'))[0]
            del pre_qry['dimensions']

            client.topn(**pre_qry)
            logging.info('Phase 1 Complete')
            if phase == 2:
                query_str += '// Two phase query\n// Phase 1\n'
            query_str += json.dumps(
                client.query_builder.last_query.query_dict, indent=2)
            query_str += '\n'
            if phase == 1:
                return query_str
            query_str += (
                "// Phase 2 (built based on phase one's results)\n")
            df = client.export_pandas()
            qry['filter'] = self._add_filter_from_pre_query_data(
                df,
                [pre_qry['dimension']],
                filters)
            qry['threshold'] = timeseries_limit or 1000
            if row_limit and granularity == 'all':
                qry['threshold'] = row_limit
            qry['dimension'] = dim
            del qry['dimensions']
            qry['metric'] = list(qry['aggregations'].keys())[0]
            client.topn(**qry)
            logging.info('Phase 2 Complete')
        elif len(groupby) > 0 or having_filters:
            # If grouping on multiple fields or using a having filter
            # we have to force a groupby query
            logging.info('Running groupby query for dimensions [{}]'.format(dimensions))
            if timeseries_limit and is_timeseries:
                logging.info('Running two-phase query for timeseries')

                pre_qry = deepcopy(qry)
                pre_qry_dims = self._dimensions_to_values(qry['dimensions'])

                # Can't use set on an array with dicts
                # Use set with non-dict items only
                non_dict_dims = list(
                    set([x for x in pre_qry_dims if not isinstance(x, dict)]),
                )
                dict_dims = [x for x in pre_qry_dims if isinstance(x, dict)]
                pre_qry['dimensions'] = non_dict_dims + dict_dims

                order_by = None
                if metrics:
                    order_by = utils.get_metric_name(metrics[0])
                else:
                    order_by = pre_qry_dims[0]

                if timeseries_limit_metric:
                    order_by = utils.get_metric_name(timeseries_limit_metric)
                    aggs_dict, post_aggs_dict = DruidDatasource.metrics_and_post_aggs(
                        [timeseries_limit_metric],
                        metrics_dict)
                    if phase == 1:
                        pre_qry['aggregations'].update(aggs_dict)
                        pre_qry['post_aggregations'].update(post_aggs_dict)
                    else:
                        pre_qry['aggregations'] = aggs_dict
                        pre_qry['post_aggregations'] = post_aggs_dict

                # Limit on the number of timeseries, doing a two-phases query
                pre_qry['granularity'] = 'all'
                pre_qry['limit_spec'] = {
                    'type': 'default',
                    'limit': min(timeseries_limit, row_limit),
                    'intervals': self.intervals_from_dttms(
                        inner_from_dttm, inner_to_dttm),
                    'columns': [{
                        'dimension': order_by,
                        'direction': order_direction,
                    }],
                }
                client.groupby(**pre_qry)
                logging.info('Phase 1 Complete')
                query_str += '// Two phase query\n// Phase 1\n'
                query_str += json.dumps(
                    client.query_builder.last_query.query_dict, indent=2)
                query_str += '\n'
                if phase == 1:
                    return query_str
                query_str += (
                    "// Phase 2 (built based on phase one's results)\n")
                df = client.export_pandas()
                qry['filter'] = self._add_filter_from_pre_query_data(
                    df,
                    pre_qry['dimensions'],
                    filters,
                )
                qry['limit_spec'] = None
            if row_limit:
                dimension_values = self._dimensions_to_values(dimensions)
                qry['limit_spec'] = {
                    'type': 'default',
                    'limit': row_limit,
                    'columns': [{
                        'dimension': (
                            utils.get_metric_name(
                                metrics[0],
                            ) if metrics else dimension_values[0]
                        ),
                        'direction': order_direction,
                    }],
                }
            client.groupby(**qry)
            logging.info('Query Complete')
        query_str += json.dumps(
            client.query_builder.last_query.query_dict, indent=2)
        return query_str
1245813
function updateObject(entities, descriptor) {

    if (!entities) {
        return entities;
    }

    let result = Array.isArray(entities) ? entities : [entities];

    if (!isUpdateDescriptor(descriptor)) {
        return descriptor && typeof descriptor === 'object'
            ? [util.copyMetaProperties(descriptor, result[0])]
            : result;
    }

    // get updater operations
    let operations = Object.keys(descriptor)
        .map(opType => ({name: opType, transformation: UPDATE_OPS[opType]}))
        .filter(op => op.transformation);

    for (let operator of operations) {
        result = mutate(result, descriptor[operator.name], operator.transformation);
    }

    return result;
}
376167
function convertExistingQueryProps(json, builder) {
  const keys = Object.keys(json);

  for (let i = 0, l = keys.length; i < l; ++i) {
    const key = keys[i];
    const value = json[key];

    if (isQueryProp(value)) {
      json[key] = queryPropToKnexRaw(value, builder);
    }
  }

  return json;
}
319820
def restore_descriptor(self, dataframe):
        

        # Prepare
        fields = []
        primary_key = None

        # Primary key
        if dataframe.index.name:
            field_type = self.restore_type(dataframe.index.dtype)
            field = {
                'name': dataframe.index.name,
                'type': field_type,
                'constraints': {'required': True},
            }
            fields.append(field)
            primary_key = dataframe.index.name

        # Fields
        for column, dtype in dataframe.dtypes.iteritems():
            sample = dataframe[column].iloc[0] if len(dataframe) else None
            field_type = self.restore_type(dtype, sample=sample)
            field = {'name': column, 'type': field_type}
            # TODO: provide better required indication
            # if dataframe[column].isnull().sum() == 0:
            #     field['constraints'] = {'required': True}
            fields.append(field)

        # Descriptor
        descriptor = {}
        descriptor['fields'] = fields
        if primary_key:
            descriptor['primaryKey'] = primary_key

        return descriptor
1329407
def from_description(cls, description, attrs):
        
        hash_key = None
        range_key = None
        index_type = description["Projection"]["ProjectionType"]
        includes = description["Projection"].get("NonKeyAttributes")
        for data in description["KeySchema"]:
            name = data["AttributeName"]
            if name not in attrs:
                continue
            key_type = data["KeyType"]
            if key_type == "HASH":
                hash_key = TableField(name, attrs[name].data_type, key_type)
            elif key_type == "RANGE":
                range_key = TableField(name, attrs[name].data_type, key_type)
        throughput = description["ProvisionedThroughput"]
        return cls(
            description["IndexName"],
            index_type,
            description["IndexStatus"],
            hash_key,
            range_key,
            throughput["ReadCapacityUnits"],
            throughput["WriteCapacityUnits"],
            description.get("IndexSizeBytes", 0),
            includes,
            description,
        )
139459
def to_python(self, value):
        """"""
        cache = None
        if value is None:
            return None

        if isinstance(value, DataDescriptor):
            return value
        elif isinstance(value, dict):
            # Allow pre-hydrated data objects.
            cache = value
            try:
                value = cache['__id']
            except KeyError:
                raise ValidationError("dictionary must contain an '__id' element")
        elif not isinstance(value, int):
            raise ValidationError("field must be a DataDescriptor, data object primary key or dict")

        return DataDescriptor(value, self, cache)
1308
public static MethodDesc forDescriptor(String desc) 
        throws IllegalArgumentException
    {
        try {
            int cursor = 0;
            char c;

            if ((c = desc.charAt(cursor++)) != '(') {
                throw invalidDescriptor(desc);
            }

            StringBuffer buf = new StringBuffer();
            List<TypeDesc> list = new ArrayList<TypeDesc>();

            while ((c = desc.charAt(cursor++)) != ')') {
                switch (c) {
                case 'V':
                case 'I':
                case 'C':
                case 'Z':
                case 'D':
                case 'F':
                case 'J':
                case 'B':
                case 'S':
                    buf.append(c);
                    break;
                case '[':
                    buf.append(c);
                    continue;
                case 'L':
                    while (true) {
                        buf.append(c);
                        if (c == ';') {
                            break;
                        }
                        c = desc.charAt(cursor++);
                    }
                    break;
                default:
                    throw invalidDescriptor(desc);
                }

                list.add(TypeDesc.forDescriptor(buf.toString()));
                buf.setLength(0);
            }

            TypeDesc ret = TypeDesc.forDescriptor(desc.substring(cursor));

            TypeDesc[] tds = list.toArray(new TypeDesc[list.size()]);

            return intern(new MethodDesc(desc, ret, tds));
        } catch (NullPointerException e) {
            throw invalidDescriptor(desc);
        } catch (IndexOutOfBoundsException e) {
            throw invalidDescriptor(desc);
        }
    }
546980
private double naiveQuery(V obj, WritableDoubleDataStore scores, HashSetModifiableDBIDs cands) {
    if(obj instanceof SparseNumberVector) {
      return naiveQuerySparse((SparseNumberVector) obj, scores, cands);
    }
    else {
      return naiveQueryDense(obj, scores, cands);
    }
  }
266161
def _gql(cls, query_string, *args, **kwds):
    """"""
    from .query import gql  # Import late to avoid circular imports.
    return gql('SELECT * FROM %s %s' % (cls._class_name(), query_string),
               *args, **kwds)