Performs a readonly "compaction" of all sstables in order to validate complete rows,
but without writing the merge result
960116
private void doValidationCompaction(ColumnFamilyStore cfs, Validator validator) throws IOException
    {
        // this isn't meant to be race-proof, because it's not -- it won't cause bugs for a CFS to be dropped
        // mid-validation, or to attempt to validate a droped CFS.  this is just a best effort to avoid useless work,
        // particularly in the scenario where a validation is submitted before the drop, and there are compactions
        // started prior to the drop keeping some sstables alive.  Since validationCompaction can run
        // concurrently with other compactions, it would otherwise go ahead and scan those again.
        if (!cfs.isValid())
            return;

        Refs<SSTableReader> sstables = null;
        try
        {

            String snapshotName = validator.desc.sessionId.toString();
            int gcBefore;
            boolean isSnapshotValidation = cfs.snapshotExists(snapshotName);
            if (isSnapshotValidation)
            {
                // If there is a snapshot created for the session then read from there.
                sstables = cfs.getSnapshotSSTableReader(snapshotName);

                // Computing gcbefore based on the current time wouldn't be very good because we know each replica will execute
                // this at a different time (that's the whole purpose of repair with snaphsot). So instead we take the creation
                // time of the snapshot, which should give us roughtly the same time on each replica (roughtly being in that case
                // 'as good as in the non-snapshot' case)
                gcBefore = cfs.gcBefore(cfs.getSnapshotCreationTime(snapshotName));
            }
            else
            {
                // flush first so everyone is validating data that is as similar as possible
                StorageService.instance.forceKeyspaceFlush(cfs.keyspace.getName(), cfs.name);
                // we don't mark validating sstables as compacting in DataTracker, so we have to mark them referenced
                // instead so they won't be cleaned up if they do get compacted during the validation
                if (validator.desc.parentSessionId == null || ActiveRepairService.instance.getParentRepairSession(validator.desc.parentSessionId) == null)
                    sstables = cfs.selectAndReference(ColumnFamilyStore.CANONICAL_SSTABLES).refs;
                else
                {
                    ColumnFamilyStore.RefViewFragment refView = cfs.selectAndReference(ColumnFamilyStore.UNREPAIRED_SSTABLES);
                    sstables = refView.refs;
                    Set<SSTableReader> currentlyRepairing = ActiveRepairService.instance.currentlyRepairing(cfs.metadata.cfId, validator.desc.parentSessionId);

                    if (!Sets.intersection(currentlyRepairing, Sets.newHashSet(refView.sstables)).isEmpty())
                    {
                        logger.error("Cannot start multiple repair sessions over the same sstables");
                        throw new RuntimeException("Cannot start multiple repair sessions over the same sstables");
                    }

                    ActiveRepairService.instance.getParentRepairSession(validator.desc.parentSessionId).addSSTables(cfs.metadata.cfId, refView.sstables);
                }

                if (validator.gcBefore > 0)
                    gcBefore = validator.gcBefore;
                else
                    gcBefore = getDefaultGcBefore(cfs);
            }

            // Create Merkle tree suitable to hold estimated partitions for given range.
            // We blindly assume that partition is evenly distributed on all sstables for now.
            long numPartitions = 0;
            for (SSTableReader sstable : sstables)
            {
                numPartitions += sstable.estimatedKeysForRanges(Collections.singleton(validator.desc.range));
            }
            // determine tree depth from number of partitions, but cap at 20 to prevent large tree.
            int depth = numPartitions > 0 ? (int) Math.min(Math.floor(Math.log(numPartitions)), 20) : 0;
            MerkleTree tree = new MerkleTree(cfs.partitioner, validator.desc.range, MerkleTree.RECOMMENDED_DEPTH, (int) Math.pow(2, depth));

            long start = System.nanoTime();
            try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategy().getScanners(sstables, validator.desc.range))
            {
                CompactionIterable ci = new ValidationCompactionIterable(cfs, scanners.scanners, gcBefore);
                Iterator<AbstractCompactedRow> iter = ci.iterator();
                metrics.beginCompaction(ci);
                try
                {
                    // validate the CF as we iterate over it
                    validator.prepare(cfs, tree);
                    while (iter.hasNext())
                    {
                        if (ci.isStopRequested())
                            throw new CompactionInterruptedException(ci.getCompactionInfo());
                        AbstractCompactedRow row = iter.next();
                        validator.add(row);
                    }
                    validator.complete();
                }
                finally
                {
                    if (isSnapshotValidation)
                    {
                        cfs.clearSnapshot(snapshotName);
                    }

                    metrics.finishCompaction(ci);
                }
            }

            if (logger.isDebugEnabled())
            {
                // MT serialize may take time
                long duration = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
                logger.debug("Validation finished in {} msec, depth {} for {} keys, serialized size {} bytes for {}",
                             duration,
                             depth,
                             numPartitions,
                             MerkleTree.serializer.serializedSize(tree, 0),
                             validator.desc);
            }
        }
        finally
        {
            if (sstables != null)
                sstables.release();
        }
    }
213744
@SuppressWarnings({"rawtypes", "unchecked"})
    private void withMissingLeftJoin(Table destination, Table table1) {
        for (int c = 0; c < destination.columnCount(); c++) {
            if (c < table1.columnCount()) {
                Column t1Col = table1.column(c);
                destination.column(c).append(t1Col);
            } else {
                for (int r1 = 0; r1 < table1.rowCount(); r1++) {
                    destination.column(c).appendMissing();
                }
            }
        }
    }
528192
@VisibleForTesting
    static Pair<List<SSTableReader>, Double> trimToThresholdWithHotness(List<SSTableReader> bucket, int maxThreshold)
    {
        // Sort by sstable hotness (descending). We first build a map because the hotness may change during the sort.
        final Map<SSTableReader, Double> hotnessSnapshot = getHotnessMap(bucket);
        Collections.sort(bucket, new Comparator<SSTableReader>()
        {
            public int compare(SSTableReader o1, SSTableReader o2)
            {
                return -1 * Double.compare(hotnessSnapshot.get(o1), hotnessSnapshot.get(o2));
            }
        });

        // and then trim the coldest sstables off the end to meet the maxThreshold
        List<SSTableReader> prunedBucket = bucket.subList(0, Math.min(bucket.size(), maxThreshold));

        // bucket hotness is the sum of the hotness of all sstable members
        double bucketHotness = 0.0;
        for (SSTableReader sstr : prunedBucket)
            bucketHotness += hotness(sstr);

        return Pair.create(prunedBucket, bucketHotness);
    }
680919
function _storeMergedCells()
    {
        // if there are no merged cell ranges set, return
        if (count($this->_merged_ranges) == 0) {
            return;
        }
        $record   = 0x00E5;
        $length   = 2 + count($this->_merged_ranges) * 8;

        $header   = pack('vv', $record, $length);
        $data     = pack('v',  count($this->_merged_ranges));
        foreach ($this->_merged_ranges as $range) {
            $data .= pack('vvvv', $range[0], $range[2], $range[1], $range[3]);
        }
        $this->_append($header . $data);
    }
569063
private void validateTable(TableDefinition tableDef) {
        for (String optName : tableDef.getOptionNames()) {
            String optValue = tableDef.getOption(optName);
            switch (optName) {
            case CommonDefs.OPT_AGING_FIELD:
                validateTableOptionAgingField(tableDef, optValue);
                break;
            case CommonDefs.OPT_RETENTION_AGE:
                validateTableOptionRetentionAge(tableDef, optValue);
                break;
            case CommonDefs.OPT_SHARDING_FIELD:
                validateTableOptionShardingField(tableDef, optValue);
                break;
            case CommonDefs.OPT_SHARDING_GRANULARITY:
                validateTableOptionShardingGranularity(tableDef, optValue);
                break;
            case CommonDefs.OPT_SHARDING_START:
                validateTableOptionShardingStart(tableDef, optValue);
                break;
            case CommonDefs.OPT_AGING_CHECK_FREQ:
                validateTableOptionAgingCheckFrequency(tableDef, optValue);
                break;
            default:
                Utils.require(false, "Unknown option for SpiderService table: " + optName);
            }
        }
        
        for (FieldDefinition fieldDef : tableDef.getFieldDefinitions()) {
            validateField(fieldDef);
        }
        
        if (tableDef.getOption(CommonDefs.OPT_AGING_FIELD) != null &&
            tableDef.getOption(CommonDefs.OPT_AGING_CHECK_FREQ) == null) {
            String agingCheckFreq = tableDef.getAppDef().getOption(CommonDefs.OPT_AGING_CHECK_FREQ);
            if (Utils.isEmpty(agingCheckFreq)) {
                agingCheckFreq = "1 DAY";
            }
            tableDef.setOption(CommonDefs.OPT_AGING_CHECK_FREQ, agingCheckFreq);
        }
    }
153050
private void dble() {
            Assert.check(shared == 0);
            Entry[] oldtable = table;
            Entry[] newtable = new Entry[oldtable.length * 2];
            for (ScopeImpl s = this; s != null; s = s.next) {
                if (s.table == oldtable) {
                    Assert.check(s == this || s.shared != 0);
                    s.table = newtable;
                    s.hashMask = newtable.length - 1;
                }
            }
            int n = 0;
            for (int i = oldtable.length; --i >= 0; ) {
                Entry e = oldtable[i];
                if (e != null && e != sentinel) {
                    table[getIndex(e.sym.name)] = e;
                    n++;
                }
            }
            // We don't need to update nelems for shared inherited scopes,
            // since that gets handled by leave().
            nelems = n;
        }
833947
def flush(self):
        
        if len(self.rows) != 0:
            self.table.mutate_rows(self.rows)
            self.total_mutation_count = 0
            self.total_size = 0
            self.rows = []
495587
static boolean needsCleanup(SSTableReader sstable, Collection<Range<Token>> ownedRanges)
    {
        assert !ownedRanges.isEmpty(); // cleanup checks for this

        // unwrap and sort the ranges by LHS token
        List<Range<Token>> sortedRanges = Range.normalize(ownedRanges);

        // see if there are any keys LTE the token for the start of the first range
        // (token range ownership is exclusive on the LHS.)
        Range<Token> firstRange = sortedRanges.get(0);
        if (sstable.first.getToken().compareTo(firstRange.left) <= 0)
            return true;

        // then, iterate over all owned ranges and see if the next key beyond the end of the owned
        // range falls before the start of the next range
        for (int i = 0; i < sortedRanges.size(); i++)
        {
            Range<Token> range = sortedRanges.get(i);
            if (range.right.isMinimum())
            {
                // we split a wrapping range and this is the second half.
                // there can't be any keys beyond this (and this is the last range)
                return false;
            }

            DecoratedKey firstBeyondRange = sstable.firstKeyBeyond(range.right.maxKeyBound());
            if (firstBeyondRange == null)
            {
                // we ran off the end of the sstable looking for the next key; we don't need to check any more ranges
                return false;
            }

            if (i == (sortedRanges.size() - 1))
            {
                // we're at the last range and we found a key beyond the end of the range
                return true;
            }

            Range<Token> nextRange = sortedRanges.get(i + 1);
            if (!nextRange.contains(firstBeyondRange.getToken()))
            {
                // we found a key in between the owned ranges
                return true;
            }
        }

        return false;
    }
691379
static boolean mergeKeepUpdate(Session session, HashMappedList rowSet,
                                   int[] cols, Type[] colTypes, Row row,
                                   Object[] newData) {

        Object[] data = (Object[]) rowSet.get(row);

        if (data != null) {
            if (IndexAVL
                    .compareRows(row
                        .getData(), newData, cols, colTypes) != 0 && IndexAVL
                            .compareRows(newData, data, cols, colTypes) != 0) {
                return false;
            }

            for (int j = 0; j < cols.length; j++) {
                newData[cols[j]] = data[cols[j]];
            }

            rowSet.put(row, newData);
        } else {
            rowSet.add(row, newData);
        }

        return true;
    }
1333044
function _updateRowspan(tableData, startRowIndex, endRowIndex) {
  util.range(startRowIndex, endRowIndex + 1).forEach(rowIndex => {
    tableData[rowIndex].forEach((cell, cellIndex) => {
      if (util.isExisty(cell.rowMergeWith)) {
        const merger = tableData[cell.rowMergeWith][cellIndex];

        if (merger.rowspan) {
          merger.rowspan -= 1;
        }
      } else if (cell.rowspan > 1) {
        const lastMergedRowIndex = rowIndex + cell.rowspan - 1;

        cell.rowspan -= (endRowIndex - rowIndex + 1);

        if (lastMergedRowIndex > endRowIndex) {
          tableData[endRowIndex + 1][cellIndex] = util.extend({}, cell);
        }
      }
    });
  });
}
1168975
private boolean fromSameShard(ByteBuffer fromInclusive, ByteBuffer toExclusive) {
        return fromInclusive.remaining() >= 9 &&
                toExclusive.remaining() >= 9 &&
                RowKeyUtils.getShardId(fromInclusive) == RowKeyUtils.getShardId(toExclusive) &&
                RowKeyUtils.getTableUuid(fromInclusive) == RowKeyUtils.getTableUuid(toExclusive);
    }
68236
public final void ensureRowsComplete() throws SnowflakeSQLException
  {
    // Check that all the rows have been decoded, raise an error if not
    if (rowCount != currentRow)
    {
      throw
          new SnowflakeSQLException(
              SqlState.INTERNAL_ERROR,
              ErrorCode.INTERNAL_ERROR
                  .getMessageCode(),
              "Exception: expected " +
              rowCount +
              " rows and received " +
              currentRow);
    }
  }
323237
def discard(self, s):
        

        lines = s.splitlines(True)
        for line in lines:
            if line[-1] not in '\r\n':
                if not self.warn:
                    logger.warning(
                        'partial line discard UNSUPPORTED; source map '
                        'generated will not match at the column level'
                    )
                    self.warn = True
            else:
                # simply increment row
                self.row += 1
912802
private void emptyEndsCells(Row tmplRow, int tmplCol, int rowOffset, Row row, int fieldsNum) {
        if (rowOffset <= 0) return;

        emptyCells(tmplRow, rowOffset, row, tmplRow.getFirstCellNum(), tmplCol - 1);
        emptyCells(tmplRow, rowOffset, row, tmplCol + fieldsNum, tmplRow.getLastCellNum());
    }
250574
@SuppressWarnings({"rawtypes","unchecked"})
    public static void copyRowsToTable(Selection rows, Table oldTable, Table newTable) {
        for (int columnIndex = 0; columnIndex < oldTable.columnCount(); columnIndex++) {
            Column oldColumn = oldTable.column(columnIndex);
            int r = 0;
            for (int i : rows) {
                newTable.column(columnIndex).set(r, oldColumn, i);
                r++;
            }
        }

    }
1087469
protected void prepareDroppingTablesApproachScripts() throws DBCleanException
   {
      cleaningScripts.addAll(getTablesDroppingScripts());
      cleaningScripts.addAll(getDBInitializationScripts());
      cleaningScripts.addAll(getFKRemovingScripts());
      cleaningScripts.addAll(getIndexesDroppingScripts());

      committingScripts.addAll(getIndexesAddingScripts());
      committingScripts.addAll(getFKAddingScripts());
   }
1250719
def _sync_columns(self, row, ensure, types=None):
        
        columns = self.columns
        ensure = self._check_ensure(ensure)
        types = types or {}
        types = {normalize_column_name(k): v for (k, v) in types.items()}
        out = {}
        sync_columns = []
        for name, value in row.items():
            name = normalize_column_name(name)
            if ensure and name not in columns:
                _type = types.get(name)
                if _type is None:
                    _type = self.db.types.guess(value)
                sync_columns.append(Column(name, _type))
                columns.append(name)
            if name in columns:
                out[name] = value
        self._sync_table(sync_columns)
        return out