Prepare and append a Record into its Table; flush to disk if necessary.
542013
def _add_record(table, data, buffer_size):
    
    fields = table.fields
    # remove any keys that aren't relation fields
    for invalid_key in set(data).difference([f.name for f in fields]):
        del data[invalid_key]
    table.append(Record.from_dict(fields, data))
    # write if requested and possible
    if buffer_size is not None and table.is_attached():
        # for now there isn't a public method to get the number of new
        # records, so use private members
        if (len(table) - 1) - table._last_synced_index > buffer_size:
            table.commit()
918267
def _prepare_record(record, index, doc_type):
        
        if current_app.config['INDEXER_REPLACE_REFS']:
            data = copy.deepcopy(record.replace_refs())
        else:
            data = record.dumps()

        data['_created'] = pytz.utc.localize(record.created).isoformat() \
            if record.created else None
        data['_updated'] = pytz.utc.localize(record.updated).isoformat() \
            if record.updated else None

        # Allow modification of data prior to sending to Elasticsearch.
        before_record_index.send(
            current_app._get_current_object(),
            json=data,
            record=record,
            index=index,
            doc_type=doc_type,
        )

        return data
921048
@Override
    public Put createPut(Record record, String tableName,
            String famName, String quaName) throws IOException {
        HTable hTable = getTable(tableName);
        Put put = null;
        if (hTable != null) {
            // Transforming data model record into an Avro record
            AvroSerializer serializer = getSerializer();
            final byte[] bytes = serializer.toBytes(record);
            // Resource's Key
            put = new Put(Bytes.toBytes(record.getID().toString()));
            // Resource's Value
            put.add(Bytes.toBytes(famName), Bytes.toBytes(quaName), bytes);
        }
        return put;
    }
935874
def append(self, request, response):
        """"""
        request = self._before_record_request(request)
        if not request:
            return
        # Deepcopy is here because mutation of `response` will corrupt the
        # real response.
        response = copy.deepcopy(response)
        response = self._before_record_response(response)
        if response is None:
            return
        self.data.append((request, response))
        self.dirty = True
125468
public void flushEndTable(final TableAppender appender) throws IOException {
        this.observer.update(new EndTableFlusher(appender,
                this.tableRows.subList(this.lastFlushedRowIndex, this.tableRows.usedSize())));
    }
126960
def _append(self, row):
		
		self.cursor.execute(self.append_statement, self.append_attrgetter(row))
112621
def flush(self):
        """"""
        self.log.info('Flushing tables and arrays to disk...')
        for tab in self._tables.values():
            tab.flush()
        self._write_ndarrays_cache_to_disk()
490391
public void processRecordInternally(Record record) {
        if (record instanceof FormatRecord) {
            FormatRecord fr = (FormatRecord) record;
            _customFormatRecords.put(Integer.valueOf(fr.getIndexCode()), fr);
        }
        if (record instanceof ExtendedFormatRecord) {
            ExtendedFormatRecord xr = (ExtendedFormatRecord) record;
            _xfRecords.add(xr);
        }
    }
666263
def __record(self):
        """"""
        f = self.__getFileObj(self.dbf)
        recFmt = self.__recordFmt()
        recordContents = unpack(recFmt[0], f.read(recFmt[1]))
        if recordContents[0] != b(' '):
            # deleted record
            return None
        record = []
        for (name, typ, size, deci), value in zip(self.fields,
                                                                                                recordContents):
            if name == 'DeletionFlag':
                continue
            elif not value.strip():
                record.append(value)
                continue
            elif typ == "N":
                value = value.replace(b('\0'), b('')).strip()
                if value == b(''):
                    value = 0
                elif deci:
                    try:
                        value = float(value)
                    except ValueError:
                        value = 0
                else:
                    value = int(value)
            elif typ == b('D'):
                try:
                    y, m, d = int(value[:4]), int(value[4:6]), int(value[6:8])
                    value = [y, m, d]
                except:
                    value = value.strip()
            elif typ == b('L'):
                value = (value in b('YyTt') and b('T')) or \
                                        (value in b('NnFf') and b('F')) or b('?')
            else:
                value = u(value)
                value = value.strip()
            record.append(value)
        return record
6973
def _AppendRecord(self):
    """"""

    # If no Values then don't output.
    if not self.values:
      return

    cur_record = []
    for value in self.values:
      try:
        value.OnSaveRecord()
      except SkipRecord:
        self._ClearRecord()
        return
      except SkipValue:
        continue

      # Build current record into a list.
      cur_record.append(value.value)

    # If no Values in template or whole record is empty then don't output.
    if len(cur_record) == (cur_record.count(None) + cur_record.count([])):
      return

    # Replace any 'None' entries with null string ''.
    while None in cur_record:
      cur_record[cur_record.index(None)] = ''

    self._result.append(cur_record)
    self._ClearRecord()
616616
function saveRecord(rType, idFun) {
  return function(records, callback) {
    GLOBAL.debug('inserting', records.length, rType);
    if (!_.isArray(records)) {
      records = [records];
    }
    var tc = [];
    records.forEach(function(d) {
      var meta = {_index : GLOBAL.config.ESEARCH._index, _type : rType};
      if (idFun) {
        meta._id = idFun(d);
      }
      tc.push({ index : meta});
      tc.push(d);
    });
    getEs().bulk(tc, function (err, data) {
      callback(err, data, records);
    });
  };
}
149223
def flush(self):
    """"""
    # Write data to in-memory buffer first.
    buf = cStringIO.StringIO()
    with records.RecordsWriter(buf) as w:
      for record in self._buffer:
        w.write(record)
      w._pad_block()
    str_buf = buf.getvalue()
    buf.close()

    if not self._exclusive and len(str_buf) > _FILE_POOL_MAX_SIZE:
      # Shouldn't really happen because of flush size.
      raise errors.Error(
          "Buffer too big. Can't write more than %s bytes in one request: "
          "risk of writes interleaving. Got: %s" %
          (_FILE_POOL_MAX_SIZE, len(str_buf)))

    # Write data to file.
    start_time = time.time()
    self._write(str_buf)
    if self._ctx:
      operation.counters.Increment(
          COUNTER_IO_WRITE_BYTES, len(str_buf))(self._ctx)
      operation.counters.Increment(
          COUNTER_IO_WRITE_MSEC,
          int((time.time() - start_time) * 1000))(self._ctx)

    # reset buffer
    self._buffer = []
    self._size = 0
    gc.collect()
222723
def create(self, columns=None, type_map=None, overwrite=False):
        """"""
        # Check already exists #
        if self.count_bytes > 0:
            if overwrite: self.remove()
            else: raise Exception("File exists already at '%s'" % self)
        # If we want it empty #
        if columns is None:
            self.touch()
        # Make the table #
        else:
            self.add_table(self.main_table, columns=columns, type_map=type_map)
756963
def upsert(self, table, value, update_columns=None, commit=True):
        
        if not isinstance(value, dict):
            raise TypeError('Input value should be a dictionary')

        if not update_columns:
            update_columns = value.keys()

        value_q, _args = self._value_parser(value, columnname=False)

        _sql = ''.join(['INSERT INTO ', self._backtick(table), ' (', self._backtick_columns(value), ') VALUES ',
                        '(', value_q, ') ',
                        'ON DUPLICATE KEY UPDATE ',
                        ', '.join(['='.join([k, 'VALUES('+k+')']) for k in update_columns]), ';'])

        if self.debug:
            return self.cur.mogrify(_sql, _args)

        self.cur.execute(_sql, _args)
        if commit:
            self.conn.commit()
        return self.cur.lastrowid
1083149
def prepareInsert(self, oself, store):
        
        if self.relative:
            fspath = self.__get__(oself)
            oself.__dirty__[self.attrname] = self, self.infilter(fspath, oself, store)
856923
def _update_record(self, identifier=None, rtype=None, name=None, content=None):
        """"""
        return self._change_record_sets('UPSERT', rtype, name, content)
1271241
def append_to_table(self, table):
        
        table.add_row(dict(path=self.path,
                           key=self.key,
                           creator=self.creator,
                           timestamp=self.timestamp,
                           status=self.status,
                           flags=self.flags))