Batch upsert records into postgresql database.
183833
def batch_upsert_records(session, table, records):
    """"""
    if not records:
        return
    for record_batch in _batch_postgres_query(table, records):
        stmt = insert(table.__table__)
        stmt = stmt.on_conflict_do_update(
            constraint=table.__table__.primary_key,
            set_={
                "keys": stmt.excluded.get("keys"),
                "values": stmt.excluded.get("values"),
            },
        )
        session.execute(stmt, record_batch)
        session.commit()
756863
def bulk_create(self, objs, batch_size=None):
        
        self._insert(objs, batch_size=batch_size, return_id=False, force_insert=True)
        self.refresh()
1184761
def upsert_keys(session, key_table, keys):
    
    # Do nothing if empty
    if not keys:
        return

    for key_batch in _batch_postgres_query(
        key_table, [{"name": k[0], "candidate_classes": k[1]} for k in keys.items()]
    ):
        stmt = insert(key_table.__table__)
        stmt = stmt.on_conflict_do_update(
            constraint=key_table.__table__.primary_key,
            set_={
                "name": stmt.excluded.get("name"),
                "candidate_classes": stmt.excluded.get("candidate_classes"),
            },
        )
        while True:
            try:
                session.execute(stmt, key_batch)
                session.commit()
                break
            except Exception as e:
                logger.debug(e)
1951
private void batchImport() {
        m_jdbcTemplate.batchUpdate(createInsertQuery(),
                                   new BatchPreparedStatementSetter() {
                                       public void setValues(
                                               PreparedStatement ps, int i)
                                               throws SQLException {
                                           int j = 1;
                                           for (String property : m_propertyNames) {
                                               Map params = (Map) m_batch.get(
                                                       i);
                                               ps.setObject(j++, params.get(
                                                       property));
                                           }
                                       }

                                       public int getBatchSize() {
                                           return m_batch.size();
                                       }
                                   });
    }
1040629
def insert_request_to_batch(self, request):
        '''
        
        '''
        self.validate_request_table(request)
        self.validate_request_partition_key(request)
        self.validate_request_row_key(request)
        self.batch_requests.append(request)
297353
def _insert_data(self, name, value, timestamp, interval, config, pipe, ttl_batch=None):
    ''''''
    # Calculate the TTL and abort if inserting into the past
    expire, ttl = config['expire'], config['ttl'](timestamp)
    if expire and not ttl:
      return

    i_bucket, r_bucket, i_key, r_key = self._calc_keys(config, name, timestamp)

    if config['coarse']:
      self._type_insert(pipe, i_key, value)
    else:
      # Add the resolution bucket to the interval. This allows us to easily
      # discover the resolution intervals within the larger interval, and
      # if there is a cap on the number of steps, it will go out of scope
      # along with the rest of the data
      pipe.sadd(i_key, r_bucket)
      self._type_insert(pipe, r_key, value)

    if expire:
      ttl_args = (i_key, ttl)
      if ttl_batch is not None:
        ttl_batch.add(ttl_args)
      else:
        pipe.expire(*ttl_args)
      if not config['coarse']:
        ttl_args = (r_key, ttl)
        if ttl_batch is not None:
          ttl_batch.add(ttl_args)
        else:
          pipe.expire(*ttl_args)
753630
def update(self,records,**kw):
        """"""
        # ignore unknown fields
        kw = dict([(k,v) for (k,v) in kw.iteritems() if k in self.fields])
        if isinstance(records,dict):
            records = [ records ]
        # update indices
        for indx in set(self.indices.keys()) & set (kw.keys()):
            for record in records:
                if record[indx] == kw[indx]:
                    continue
                _id = record["__id__"]
                # remove id for the old value
                old_pos = bisect.bisect(self.indices[indx][record[indx]],_id)-1
                del self.indices[indx][record[indx]][old_pos]
                if not self.indices[indx][record[indx]]:
                    del self.indices[indx][record[indx]]
                # insert new value
                bisect.insort(self.indices[indx].setdefault(kw[indx],[]),_id)
        for record in records:
            # update record values
            record.update(kw)
            # increment version number
            record["__version__"] += 1
935415
def insert(self):
    """"""
    if not self.curs:
      raise LIGOLwDBError, "Database connection not initalized"
    if len(self.table) == 0:
      raise LIGOLwDBError, 'attempt to insert empty table'
    for tab in self.table.keys():
      # find and add any missing unique ids
      generate = []
      missingcols = [k for k in self.ldb.uniqueids[tab] 
        if k not in self.table[tab]['column']]
      for m in missingcols:
        generate.append(',BLOB(GENERATE_UNIQUE())')
        self.table[tab]['orderedcol'].append(m)
      # and construct the sql query
      self.table[tab]['query'] = ' '.join( 
        ['INSERT INTO', tab, '(', ','.join(self.table[tab]['orderedcol']), 
        ') VALUES (', ','.join(['?' for x in self.table[tab]['column']]) , 
        ''.join(generate), ')'])
    for tabtup in self.ldb.tables:
      tab = tabtup[0].lower()
      try:
        try: 
          self.curs.executemany(self.table[tab]['query'],
            self.table[tab]['stream'])
          rowcount = self.curs.rowcount
        except DB2.Error, e:
          self.curs.execute('rollback')
          msg = e[2] 
          msg += self.xml() + '\n' 
          msg += str(self.table[tab]['query']) + '\n' 
          msg += str(self.table[tab]['stream']) + '\n'
          raise LIGOLwDBError, msg
        except DB2.Warning, e:
          self.curs.execute('rollback')
          raise LIGOLwDBError, e[2]
        #except Exception, e:
        #  self.curs.execute('rollback')
        #  raise LIGOLwDBError, e[2]
      except KeyError:
        pass
    self.curs.execute('commit')
    return rowcount
72731
def upsert(cls, name, **fields):
        """"""
        instance = cls.get(name)
        if instance:
            instance._set_fields(fields)
        else:
            instance = cls(name=name, **fields)
            instance = failsafe_add(cls.query.session, instance, name=name)
        return instance
824850
def record_as_run!(filename)
      Preconditions.check_state(filename.match(/^\d\d\d\d\d\d+\-\d\d\d\d\d\d\.sql$/),
                                "Invalid filename[#{filename}]. Must be like: 20120503-173242.sql")
      command = "insert into %s.%s (filename) select '%s' where not exists (select 1 from %s.%s where filename = '%s')" % [Db.schema_name, @table_name, filename, Db.schema_name, @table_name, filename]
      @db.psql_command(command)
    end
1192774
def _recursive_upsert(context, params, data):
    """"""
    children = params.get("children", {})
    nested_calls = []
    for child_params in children:
        key = child_params.get("key")
        child_data_list = ensure_list(data.pop(key))
        if isinstance(child_data_list, dict):
            child_data_list = [child_data_list]
        if not (isinstance(child_data_list, list) and
                all(isinstance(i, dict) for i in child_data_list)):
            context.log.warn(
                "Expecting a dict or a lost of dicts as children for key", key
            )
            continue
        if child_data_list:
            table_suffix = child_params.get("table_suffix", key)
            child_params["table"] = params.get("table") + "_" + table_suffix
            # copy some properties over from parent to child
            inherit = child_params.get("inherit", {})
            for child_data in child_data_list:
                for dest, src in inherit.items():
                    child_data[dest] = data.get(src)
                nested_calls.append((child_params, child_data))
    # Insert or update data
    _upsert(context, params, data)
    for child_params, child_data in nested_calls:
        _recursive_upsert(context, child_params, child_data)
90814
def batchify(data, batch_size):
    """"""
    nbatch = data.shape[0] // batch_size
    data = data[:nbatch * batch_size]
    data = data.reshape((batch_size, nbatch)).T
    return data
783681
public static function batchInsert($table,$columnsArr,$valueArr,$db_name=''){
		$db = self::getDb($db_name);
		$db->createCommand()
					->batchInsert($table,$columnsArr,$valueArr)
					->execute();
	
	}
217009
def bulk_upsert(self, docs, namespace, timestamp):
        
        if self.auto_commit_interval is not None:
            add_kwargs = {
                "commit": (self.auto_commit_interval == 0),
                "commitWithin": str(self.auto_commit_interval)
            }
        else:
            add_kwargs = {"commit": False}

        cleaned = (self._clean_doc(d, namespace, timestamp) for d in docs)
        if self.chunk_size > 0:
            batch = list(next(cleaned) for i in range(self.chunk_size))
            while batch:
                self.solr.add(batch, **add_kwargs)
                batch = list(next(cleaned)
                             for i in range(self.chunk_size))
        else:
            self.solr.add(cleaned, **add_kwargs)
1090021
def upsert_into(self, table):
    
    return SessionContext.session.execute(
        insert(table).from_select(
            self.c,
            self,
        ).on_conflict_do_nothing(),
    ).rowcount
664172
public function upsert($table, $insertFieldValues, $updateFieldValues = [], $primaryKey = null)
    {
        if (!$primaryKey) {
            $primaryKey = (string)key($insertFieldValues);
        }

        if (!$updateFieldValues) {
            $updateFieldValues = $insertFieldValues;
        }

        $fields = array_keys($insertFieldValues);

        $bind = $insertFieldValues;
        $updates = [];
        foreach ($updateFieldValues as $k => $v) {
            $field = is_string($k) ? $k : $v;
            if ($primaryKey === $field) {
                continue;
            }

            if (is_int($k)) {
                $updates[] = "[$field]=:{$field}_dku";
                $bind["{$field}_dku"] = $insertFieldValues[$field];
            } elseif ($v instanceof AssignmentInterface) {
                $v->setFieldName($k);
                $updates[] = $v->getSql();
                /** @noinspection SlowArrayOperationsInLoopInspection */
                $bind = array_merge($bind, $v->getBind());
            } else {
                $updates[] = $v;
            }
        }

        $insertFieldsSql = '[' . implode('],[', $fields) . ']';
        $insertValuesSql = ':' . implode(',:', $fields);

        $updateFieldsSql = implode(',', $updates);

        $sql = 'INSERT' . " INTO {$this->_escapeIdentifier($table)}($insertFieldsSql) VALUES($insertValuesSql) ON DUPLICATE KEY UPDATE $updateFieldsSql";

        return $this->execute('insert', $sql, $bind);
    }
279084
def insert(self, model_instances, batch_size=1000):
        '''
        
        '''
        from six import next
        from io import BytesIO
        i = iter(model_instances)
        try:
            first_instance = next(i)
        except StopIteration:
            return  # model_instances is empty
        model_class = first_instance.__class__

        if first_instance.is_read_only() or first_instance.is_system_model():
            raise DatabaseException("You can't insert into read only and system tables")

        fields_list = ','.join(
            ['`%s`' % name for name in first_instance.fields(writable=True)])

        def gen():
            buf = BytesIO()
            query = 'INSERT INTO $table (%s) FORMAT TabSeparated\n' % fields_list
            buf.write(self._substitute(query, model_class).encode('utf-8'))
            first_instance.set_database(self)
            buf.write(first_instance.to_tsv(include_readonly=False).encode('utf-8'))
            buf.write('\n'.encode('utf-8'))
            # Collect lines in batches of batch_size
            lines = 2
            for instance in i:
                instance.set_database(self)
                buf.write(instance.to_tsv(include_readonly=False).encode('utf-8'))
                buf.write('\n'.encode('utf-8'))
                lines += 1
                if lines >= batch_size:
                    # Return the current batch of lines
                    yield buf.getvalue()
                    # Start a new batch
                    buf = BytesIO()
                    lines = 0
            # Return any remaining lines in partial batch
            if lines:
                yield buf.getvalue()
        self._send(gen())