Infer a BigQuery table schema from a list of values.

    Args:
      data: The list of values.
    Returns:
      A list of dictionaries containing field 'name' and 'type' entries, suitable for use in a
          BigQuery Tables resource schema.
328411
def _from_list_record(data):
    
    return [Schema._get_field_entry('Column%d' % (i + 1), value) for i, value in enumerate(data)]
805495
def _get_mapping(self, schema):
        
        properties = {}
        for field, field_schema in schema.items():
            field_mapping = self._get_field_mapping(field_schema)
            if field_mapping:
                properties[field] = field_mapping
        return {'properties': properties}
784963
def fields_to_dtypes(schema):
    
    datetime_types = ['date', 'datetime']
    datetime_fields = {
        f['name']: _TABLE_SCHEMA_DTYPE_MAPPING.get(f['type'], 'object')
        for f in schema['fields']
        if f['type'] in datetime_types}

    other_fields = {
        f['name']: _TABLE_SCHEMA_DTYPE_MAPPING.get(f['type'], 'object')
        for f in schema['fields']
        if f['type'] not in datetime_types}

    return {'dates': datetime_fields, 'other': other_fields}
161211
def realize_input_schema(input_types,  # type: MutableSequence[Dict[Text, Any]]
                         schema_defs   # type: Dict[Text, Any]
                        ):  # type: (...) -> MutableSequence[Dict[Text, Any]]
    """"""
    for index, entry in enumerate(input_types):
        if isinstance(entry, string_types):
            if '#' in entry:
                _, input_type_name = entry.split('#')
            else:
                input_type_name = entry
            if input_type_name in schema_defs:
                entry = input_types[index] = schema_defs[input_type_name]
        if isinstance(entry, collections.Mapping):
            if isinstance(entry['type'], string_types) and '#' in entry['type']:
                _, input_type_name = entry['type'].split('#')
                if input_type_name in schema_defs:
                    input_types[index]['type'] = realize_input_schema(
                        schema_defs[input_type_name], schema_defs)
            if isinstance(entry['type'], collections.MutableSequence):
                input_types[index]['type'] = realize_input_schema(
                    entry['type'], schema_defs)
            if isinstance(entry['type'], collections.Mapping):
                input_types[index]['type'] = realize_input_schema(
                    [input_types[index]['type']], schema_defs)
            if entry['type'] == 'array':
                items = entry['items'] if \
                    not isinstance(entry['items'], string_types) else [entry['items']]
                input_types[index]['items'] = realize_input_schema(items, schema_defs)
            if entry['type'] == 'record':
                input_types[index]['fields'] = realize_input_schema(
                    entry['fields'], schema_defs)
    return input_types
926817
public static List<TableFieldSchema> getSchemaFromString(String fields) {
    logger.atFine().log("getSchemaFromString('%s')", fields);

    // Parse the output schema for Json from fields.
    JsonParser jsonParser = new JsonParser();
    JsonArray json = jsonParser.parse(fields).getAsJsonArray();
    List<TableFieldSchema> fieldsList = new ArrayList<>();

    // For each item in the list of fields.
    for (JsonElement jsonElement : json) {
      checkArgument(
          jsonElement.isJsonObject(), "Expected JsonObject for element, got '%s'.", jsonElement);
      JsonObject jsonObject = jsonElement.getAsJsonObject();

      // Set the name and type.
      checkArgument(
          jsonObject.get("name") != null,
          "Expected non-null entry for key 'name' in JsonObject '%s'", jsonObject);
      checkArgument(
          jsonObject.get("type") != null,
          "Expected non-null entry for key 'type' in JsonObject '%s'", jsonObject);
      TableFieldSchema fieldDef = new TableFieldSchema();
      fieldDef.setName(jsonObject.get("name").getAsString());
      fieldDef.setType(jsonObject.get("type").getAsString());

      // If mode is not null, set mode.
      if (jsonObject.get("mode") != null) {
        fieldDef.setMode(jsonObject.get("mode").getAsString());
      }

      // If the type is RECORD set the fields.
      if (jsonObject.get("type").getAsString().equals("RECORD")) {
        checkArgument(
            jsonObject.get("fields") != null,
            "Expected non-null entry for key 'fields' in JsonObject of type RECORD: '%s'",
            jsonObject);
        fieldDef.setFields(getSchemaFromString(jsonObject.get("fields").toString()));
      }

      fieldsList.add(fieldDef);
    }
    // Return list of TableFieldSchema.
    return fieldsList;
  }
328122
def convert_descriptor_and_rows(self, descriptor, rows):
        

        # Prepare
        primary_key = None
        schema = tableschema.Schema(descriptor)
        if len(schema.primary_key) == 1:
            primary_key = schema.primary_key[0]
        elif len(schema.primary_key) > 1:
            message = 'Multi-column primary keys are not supported'
            raise tableschema.exceptions.StorageError(message)

        # Get data/index
        data_rows = []
        index_rows = []
        jtstypes_map = {}
        for row in rows:
            values = []
            index = None
            for field, value in zip(schema.fields, row):
                try:
                    if isinstance(value, float) and np.isnan(value):
                        value = None
                    if value and field.type == 'integer':
                        value = int(value)
                    value = field.cast_value(value)
                except tableschema.exceptions.CastError:
                    value = json.loads(value)
                # http://pandas.pydata.org/pandas-docs/stable/gotchas.html#support-for-integer-na
                if value is None and field.type in ('number', 'integer'):
                    jtstypes_map[field.name] = 'number'
                    value = np.NaN
                if field.name == primary_key:
                    index = value
                else:
                    values.append(value)
            data_rows.append(tuple(values))
            index_rows.append(index)

        # Get dtypes
        dtypes = []
        for field in schema.fields:
            if field.name != primary_key:
                field_name = field.name
                if six.PY2:
                    field_name = field.name.encode('utf-8')
                dtype = self.convert_type(jtstypes_map.get(field.name, field.type))
                dtypes.append((field_name, dtype))

        # Create dataframe
        index = None
        columns = schema.headers
        array = np.array(data_rows, dtype=dtypes)
        if primary_key:
            index_field = schema.get_field(primary_key)
            index_dtype = self.convert_type(index_field.type)
            index_class = pd.Index
            if index_field.type in ['datetime', 'date']:
                index_class = pd.DatetimeIndex
            index = index_class(index_rows, name=primary_key, dtype=index_dtype)
            columns = filter(lambda column: column != primary_key, schema.headers)
        dataframe = pd.DataFrame(array, index=index, columns=columns)

        return dataframe
55052
def RDFValueToBigQuerySchema(self, value):
    """"""
    fields_array = []
    for type_info in value.__class__.type_infos:
      # Nested structures are indicated by setting type "RECORD"
      if type_info.__class__.__name__ == "ProtoEmbedded":
        fields_array.append({
            "name": type_info.name,
            "type": "RECORD",
            "description": type_info.description,
            "fields": self.RDFValueToBigQuerySchema(value.Get(type_info.name))
        })
      else:
        # If we don't have a specific map use string.
        bq_type = self.RDF_BIGQUERY_TYPE_MAP.get(type_info.proto_type_name,
                                                 None) or "STRING"

        # For protos with RDF types we need to do some more checking to properly
        # covert types.
        if hasattr(type_info, "original_proto_type_name"):
          if type_info.original_proto_type_name in [
              "RDFDatetime", "RDFDatetimeSeconds"
          ]:
            bq_type = "TIMESTAMP"
          elif type_info.proto_type_name == "uint64":
            # This is to catch fields like st_mode which are stored as ints but
            # exported as more useful strings. Things which are just plain ints
            # won't have an RDF type specified and so will be exported as
            # INTEGER
            bq_type = "STRING"

        fields_array.append({
            "name": type_info.name,
            "type": bq_type,
            "description": type_info.description
        })
    return fields_array
312500
def infer(self, rows, headers=1, confidence=0.75):
        

        # Get headers
        if isinstance(headers, int):
            headers_row = headers
            while True:
                headers_row -= 1
                headers = rows.pop(0)
                if not headers_row:
                    break
        elif not isinstance(headers, list):
            headers = []

        # Get descriptor
        guesser = _TypeGuesser()
        resolver = _TypeResolver()
        descriptor = {'fields': []}
        type_matches = {}
        for header in headers:
            descriptor['fields'].append({'name': header})
        for index, row in enumerate(rows):
            # Normalize rows with invalid dimensions for sanity
            row_length = len(row)
            headers_length = len(headers)
            if row_length > headers_length:
                row = row[:len(headers)]
            if row_length < headers_length:
                diff = headers_length - row_length
                fill = [''] * diff
                row = row + fill
            # build a column-wise lookup of type matches
            for index, value in enumerate(row):
                rv = guesser.cast(value)
                if type_matches.get(index):
                    type_matches[index].extend(rv)
                else:
                    type_matches[index] = list(rv)
        # choose a type/format for each column based on the matches
        for index, results in type_matches.items():
            rv = resolver.get(results, confidence)
            descriptor['fields'][index].update(**rv)

        # Save descriptor
        self.__current_descriptor = descriptor
        self.__build()

        return descriptor
403315
def schema():
    
    tables = orca.list_tables()
    cols = {t: orca.get_table(t).columns for t in tables}
    steps = orca.list_steps()
    injectables = orca.list_injectables()
    broadcasts = orca.list_broadcasts()

    return jsonify(
        tables=tables, columns=cols, steps=steps, injectables=injectables,
        broadcasts=broadcasts)
90824
def parse_results(fields, data):
    
    master = []

    for record in data['records']:  # for each 'record' in response
        row = [None] * len(fields)  # create null list the length of number of columns
        for obj, value in record.iteritems():  # for each obj in record
            if not isinstance(value, (dict, list, tuple)):  # if not data structure
                if obj in fields:
                    row[fields.index(obj)] = ensure_utf(value)

            elif isinstance(value, dict) and obj != 'attributes':  # traverse down into object
                path = obj
                _traverse_results(value, fields, row, path)

        master.append(row)
    return master
332402
def infer_schema(self, stream, namespace=None):
    
    return self._make_request(self._infer_schema_url,
                              data={'stream': stream,
                                    'namespace': namespace or self.namespace})
186152
def values(self, column_name, exclude_type=STRUCT):
        
        column_name = unnest_path(column_name)
        columns = self.columns
        output = []
        for path in self.query_path:
            full_path = untype_path(concat_field(path, column_name))
            for c in columns:
                if c.jx_type in exclude_type:
                    continue
                # if c.cardinality == 0:
                #     continue
                if untype_path(c.name) == full_path:
                    output.append(c)
            if output:
                return output
        return []
552273
def _get_data_from_dataframe(source, fields='*', first_row=0, count=-1, schema=None):
  
  if schema is None:
    schema = google.datalab.bigquery.Schema.from_data(source)
  fields = get_field_list(fields, schema)
  rows = []
  if count < 0:
    count = len(source.index)
  df_slice = source.reset_index(drop=True)[first_row:first_row + count]
  for index, data_frame_row in df_slice.iterrows():
    row = data_frame_row.to_dict()
    for key in list(row.keys()):
      val = row[key]
      if isinstance(val, pandas.Timestamp):
        row[key] = val.to_pydatetime()

    rows.append({'c': [{'v': row[c]} if c in row else {} for c in fields]})
  cols = _get_cols(fields, schema)
  return {'cols': cols, 'rows': rows}, len(source)
118228
def data2schemacls(_data, **kwargs):
    
    content = {}

    for key in list(kwargs):  # fill kwargs
        kwargs[key] = data2schema(kwargs[key])

    if isinstance(_data, dict):
        datacontent = iteritems(_data)

    else:
        datacontent = getmembers(_data)

    for name, value in datacontent:

        if name[0] == '_':
            continue

        if isinstance(value, dict):
            schema = data2schemacls(value)()

        else:
            schema = data2schema(value)

        content[name] = schema

    content.update(kwargs)  # update content

    result = type('GeneratedSchema', (Schema,), content)

    return result
207555
def table_schema(self):
        
        if self.__dict__.get('_table_schema') is None:
            self._table_schema = None
            table_schema = {}
            for row in self.query_schema():
                name, default, dtype = self.db().lexicon.column_info(row)
                if isinstance(default, str):
                    json_matches = re.findall(r"^\'(.*)\'::jsonb$", default)
                    if len(json_matches) > 0:
                        default = json.loads(json_matches[0])
                if name == self.primary_key:
                    default = None
                table_schema[name] = {'default': default, 'type': dtype}
            if len(table_schema):
                self._table_schema = table_schema
        return self._table_schema
903509
def create_from_name_and_dictionary(self, name, datas):
        
        parameter = ObjectParameter()
        self.set_common_datas(parameter, name, datas)

        if "optional" in datas:
            parameter.optional = to_boolean(datas["optional"])
        if "type" in datas:
            parameter.type = str(datas["type"])
        if "generic" in datas:
            parameter.generic = to_boolean(datas["generic"])

        return parameter
380380
def _createFromLocal(self, data, schema):
        
        # make sure data could consumed multiple times
        if not isinstance(data, list):
            data = list(data)

        if schema is None or isinstance(schema, (list, tuple)):
            struct = self._inferSchemaFromList(data, names=schema)
            converter = _create_converter(struct)
            data = map(converter, data)
            if isinstance(schema, (list, tuple)):
                for i, name in enumerate(schema):
                    struct.fields[i].name = name
                    struct.names[i] = name
            schema = struct

        elif not isinstance(schema, StructType):
            raise TypeError("schema should be StructType or list or None, but got: %s" % schema)

        # convert python objects to sql data
        data = [schema.toInternal(row) for row in data]
        return self._sc.parallelize(data), schema