/*
We have to special case digest creation for counter column because
we don't want to include the information about which shard of the
context is a delta or not, since this information differs from node to
node.
496175
@Override
    public void updateDigest(MessageDigest digest)
    {
        digest.update(name().toByteBuffer().duplicate());
        // We don't take the deltas into account in a digest
        contextManager.updateDigest(digest, value());

        FBUtilities.updateWithLong(digest, timestamp);
        FBUtilities.updateWithByte(digest, serializationFlags());
        FBUtilities.updateWithLong(digest, timestampOfLastDelete);
    }
123817
private static void compareSets(Delta delta, Collection deltas, LinkedList stack, ID idFetcher)
    {
        Set srcSet = (Set) delta.srcValue;
        Set targetSet = (Set) delta.targetValue;

        // Create ID to Object map for target Set
        Map targetIdToValue = new HashMap();
        for (Object targetValue : targetSet)
        {
            if (targetValue != null && isIdObject(targetValue, idFetcher))
            {   // Only map non-null target array elements
                targetIdToValue.put(idFetcher.getId(targetValue), targetValue);
            }
        }

        Map srcIdToValue = new HashMap();
        String sysId = "(" + System.identityHashCode(srcSet) + ").remove(";
        for (Object srcValue : srcSet)
        {
            String srcPtr = sysId + System.identityHashCode(srcValue) + ')';
            if (isIdObject(srcValue, idFetcher))
            {   // Only map non-null source array elements
                Object srcId = idFetcher.getId(srcValue);
                srcIdToValue.put(srcId, srcValue);

                if (targetIdToValue.containsKey(srcId))
                {   // Queue item for deep, field level check as the object is still there (it's fields could have changed).
                    stack.push(new Delta(delta.id, delta.fieldName, srcPtr, srcValue, targetIdToValue.get(srcId), null));
                }
                else
                {
                    Delta removeDelta = new Delta(delta.id, delta.fieldName, srcPtr, srcValue, null, null);
                    removeDelta.setCmd(SET_REMOVE);
                    deltas.add(removeDelta);
                }
            }
            else
            {
                if (!targetSet.contains(srcValue))
                {
                    Delta removeDelta = new Delta(delta.id, delta.fieldName, srcPtr, srcValue, null, null);
                    removeDelta.setCmd(SET_REMOVE);
                    deltas.add(removeDelta);
                }
            }
        }

        sysId = "(" + System.identityHashCode(targetSet) + ").add(";
        for (Object targetValue : targetSet)
        {
            String srcPtr = sysId + System.identityHashCode(targetValue) + ')';
            if (isIdObject(targetValue, idFetcher))
            {
                Object targetId = idFetcher.getId(targetValue);
                if (!srcIdToValue.containsKey(targetId))
                {
                    Delta addDelta = new Delta(delta.id, delta.fieldName, srcPtr, null, targetValue, null);
                    addDelta.setCmd(SET_ADD);
                    deltas.add(addDelta);
                }
            }
            else
            {
                if (!srcSet.contains(targetValue))
                {
                    Delta addDelta = new Delta(delta.id, delta.fieldName, srcPtr, null, targetValue, null);
                    addDelta.setCmd(SET_ADD);
                    deltas.add(addDelta);
                }
            }
        }

        // TODO: If LinkedHashSet, may need to issue commands to reorder...
    }
372523
private Record newRecordFromCql(Key key, Iterable<Row> rows) {
        Iterator<Map.Entry<DeltaClusteringKey, Change>> changeIter = decodeChangesFromCql(rows.iterator());
        Iterator<Map.Entry<DeltaClusteringKey, Compaction>> compactionIter = decodeCompactionsFromCql(rows.iterator());
        Iterator<RecordEntryRawMetadata> rawMetadataIter = rawMetadataFromCql(rows.iterator());

        return new RecordImpl(key, compactionIter, changeIter, rawMetadataIter);
    }
371010
def can_perform_case_insensitive_comparison_for?(column)
      @case_insensitive_cache ||= {}
      @case_insensitive_cache[column.sql_type] ||= begin
        sql = <<~SQL
          SELECT exists(
            SELECT * FROM pg_proc
            WHERE proname = 'lower'
              AND proargtypes = ARRAY[#{quote column.sql_type}::regtype]::oidvector
          ) OR exists(
            SELECT * FROM pg_proc
            INNER JOIN pg_cast
              ON ARRAY[casttarget]::oidvector = proargtypes
            WHERE proname = 'lower'
              AND castsource = #{quote column.sql_type}::regtype
          )
        SQL
        select_value(sql, 'SCHEMA')
      end
    end
417856
public Statistics columnDistinctCount(String columnName, Long ndv) {
		this.columnStats
			.computeIfAbsent(columnName, column -> new HashMap<>())
			.put(DISTINCT_COUNT, String.valueOf(ndv));
		return this;
	}
537877
private void append(final KeyValue column, 
                      final boolean is_count, 
                      final boolean strip_string) {
    // for now assume we properly allocated our qualifiers
    if (is_count) {
      int offset = strip_string ? 
          Internal.getOffsetFromQualifier(column.qualifier(), 
          RollupQuery.COUNT.length + 1) :
            Internal.getOffsetFromQualifier(column.qualifier(), 
                1);
      if (last_count_offset > -1 && offset <= last_count_offset) {
        // only accept equivalent offsets. If somehow we get an earlier one, HBase is broke 
        if (offset == last_count_offset && tsdb.getConfig().fix_duplicates()) {
          if (column.timestamp() < last_count_ts) {
            if (LOG.isDebugEnabled()) {
              LOG.debug("Skipping older duplicate count value for " + column 
              + " of " + offset + " which is = the last offset " + last_count_offset
              + " for " + this);
            }
            return;
          } else { // if it's equal, just use the one we got first
            // roll back the indices
            indices[2] -= 2;
            indices[3] -= Internal.getValueLengthFromQualifier(
                count_qualifiers, indices[2]);
            if (LOG.isDebugEnabled()) {
              LOG.debug("Replacing older duplicate count with " + column 
              + " of " + offset + " which is = the last offset " + last_count_offset
              + " for " + this);
            }
          }
        } else {
          throw new IllegalArgumentException("The count offset for " + column 
              + " of " + offset + " is <= the last offset " + last_count_offset
              + " for " + this);
        }
      }
      last_count_offset = offset;
      last_count_ts = column.timestamp();
      if (strip_string) {
        System.arraycopy(column.qualifier(), RollupQuery.COUNT.length + 1, 
            count_qualifiers, indices[2], 2);
      } else {
        System.arraycopy(column.qualifier(), 1, count_qualifiers, indices[2], 2);
      }
      indices[2] += 2;
      
      if (indices[3] + column.value().length > count_values.length) {
        byte[] buf = new byte[count_values.length * 2];
        System.arraycopy(count_values, 0, buf, 0, count_values.length);
        count_values = buf;
      }
      System.arraycopy(column.value(), 0, count_values, indices[3], 
          column.value().length);
      indices[3] += column.value().length;
    } else {
      int offset = strip_string ? 
          Internal.getOffsetFromQualifier(column.qualifier(), 
              rollup_query.getRollupAggPrefix().length)
          : Internal.getOffsetFromQualifier(column.qualifier(), 1);
      if (last_offset > -1 && offset <= last_offset) {
        // only accept equivalent offsets. If somehow we get an earlier one, HBase is broke
        if (offset == last_offset && tsdb.getConfig().fix_duplicates()) {
          if (column.timestamp() < last_value_ts) {
            if (LOG.isDebugEnabled()) {
              LOG.debug("Skipping older duplicate value for " + column 
              + " of " + offset + " which is = the last offset " + last_count_offset
              + " for " + this);
            }
            return;
          } else { // if it's equal, just use the one we got first
            // roll back the indices
            indices[0] -= 2;
            indices[1] -= Internal.getValueLengthFromQualifier(qualifiers, indices[0]);
            if (LOG.isDebugEnabled()) {
              LOG.debug("Replacing older duplicate value with " + column 
              + " of " + offset + " is = the last offset " + last_count_offset
              + " for " + this);
            }
          }
        } else {
          throw new IllegalDataException("The offset for " + column 
              + " of " + offset + " is <= the last offset " + last_offset
              + " for " + this);
        }
      }
      last_offset = offset;
      last_value_ts = column.timestamp();
      if (strip_string) {
        System.arraycopy(column.qualifier(), 
            rollup_query.getRollupAggPrefix().length, qualifiers, indices[0], 2);
      } else {
        System.arraycopy(column.qualifier(), 1, qualifiers, indices[0], 2);
      }
      indices[0] += 2;
      
      if (indices[1] + column.value().length > values.length) {
        byte[] buf = new byte[values.length * 2];
        System.arraycopy(values, 0, buf, 0, values.length);
        values = buf;
      }
      System.arraycopy(column.value(), 0, values, indices[1], 
          column.value().length);
      indices[1] += column.value().length;
    }
  }
1344461
protected void updateExclusion(Exclusion value, String xmlTag, Counter counter, Element element)
   {
      Element root = element;
      Counter innerCount = new Counter(counter.getDepth() + 1);
      findAndReplaceSimpleElement(innerCount, root, "artifactId", value.getArtifactId(), null);
      findAndReplaceSimpleElement(innerCount, root, "groupId", value.getGroupId(), null);
   }
833367
void keepOnlyPositiveCounts() {
    // Starting from the back, find the first empty cell, which marks a boundary between clusters.
    int firstProbe = keys.length - 1;
    while (states[firstProbe] > 0) {
      firstProbe--;
    }

    //Work towards the front; delete any non-positive entries.
    for (int probe = firstProbe; probe-- > 0; ) {
      // When we find the next non-empty cell, we know we are at the high end of a cluster,
      //  which is tracked by firstProbe.
      if ((states[probe] > 0) && (values[probe] <= 0)) {
        hashDelete(probe); //does the work of deletion and moving higher items towards the front.
        numActive--;
      }
    }
    //now work on the first cluster that was skipped.
    for (int probe = keys.length; probe-- > firstProbe;) {
      if ((states[probe] > 0) && (values[probe] <= 0)) {
        hashDelete(probe);
        numActive--;
      }
    }
  }
863276
private void validateTableOptionShardingField(TableDefinition tableDef, String optValue) {
        // Verify that the sharding-field exists and is a timestamp field.
        FieldDefinition shardingFieldDef = tableDef.getFieldDef(optValue);
        Utils.require(shardingFieldDef != null,
                      "Sharding field has not been defined: " + optValue);
        assert shardingFieldDef != null;    // Make FindBugs happy
        Utils.require(shardingFieldDef.getType() == FieldType.TIMESTAMP,
                      "Sharding field must be a timestamp field: " + optValue);
        Utils.require(!shardingFieldDef.isCollection(),
                        "Sharding field cannot be a collection: " + optValue);
        
        // Default sharding-granularity to MONTH.
        if (tableDef.getOption(CommonDefs.OPT_SHARDING_GRANULARITY) == null) {
            tableDef.setOption(CommonDefs.OPT_SHARDING_GRANULARITY, "MONTH");
        }
        
        // Default sharding-start to "tomorrow".
        if (tableDef.getOption(CommonDefs.OPT_SHARDING_START) == null) {
            GregorianCalendar startDate = new GregorianCalendar(Utils.UTC_TIMEZONE);
            startDate.add(Calendar.DAY_OF_MONTH, 1);  // adds 1 day
            String startOpt = String.format("%04d-%02d-%02d",
                                            startDate.get(Calendar.YEAR),
                                            startDate.get(Calendar.MONTH)+1,    // 0-relative!
                                            startDate.get(Calendar.DAY_OF_MONTH));
            tableDef.setOption(CommonDefs.OPT_SHARDING_START, startOpt);
        }
    }
779140
public static Optional<String> findTableName(final PredicateSegment predicateSegment, final SQLStatement sqlStatement, final ShardingTableMetaData shardingTableMetaData) {
        if (!(sqlStatement instanceof SelectStatement)) {
            return Optional.of(sqlStatement.getTables().getSingleTableName());
        }
        SelectStatement currentSelectStatement = (SelectStatement) sqlStatement;
        while (null != currentSelectStatement.getParentStatement()) {
            currentSelectStatement = currentSelectStatement.getParentStatement();
            Optional<String> tableName = findTableName(predicateSegment, currentSelectStatement.getTables(), shardingTableMetaData);
            if (tableName.isPresent()) {
                return tableName;
            }
        }
        return findTableName(predicateSegment, currentSelectStatement.getTables(), shardingTableMetaData);
    }
392188
protected function counter2() {
    if($this->nodb) {
      return;
    }

    $this->query("select count(*) from information_schema.tables " .
                 "where (table_schema = '$this->masterdb') ".
                 "and (table_name = 'counter2')");

    list($ok) = $this->fetchrow('num');

    if($ok) {
      if($this->isBot) {
        // BLP 2017-11-01 -- add left to keep from getting too long errors
        $sql = "insert into $this->masterdb.counter2 (site, date, filename, count, bots, lasttime) ".
               "values('$this->siteName', now(), left('$this->requestUri', 254), 0, 1, now()) ".
               "on duplicate key update bots=bots+1, lasttime=now()";
      } else {
        $member = 0;
        $memberUpdate = '';

        if($this->id) {
          $member = 1;
          $memberUpdate = ", members=members+1";
        }
        // BLP 2017-11-01 -- add left to keep from getting too long errors
        $sql = "insert into $this->masterdb.counter2 (site, date, filename, count, members, lasttime) ".
               "values('$this->siteName', now(), left('$this->requestUri', 254), 1, $member, now()) ".
               "on duplicate key update count=count+1$memberUpdate, lasttime=now()";
      }
      $this->query($sql);
    } else {
      $this->debug("$this->siteName: $this->self: table bots does not exist in the $this->masterdb database");
    }
  }
246154
private static void nodesTable(Statement st,
                                   TableLocation nodesName,
                                   double tolerance, boolean isH2, int srid) throws SQLException {
        LOGGER.info("Creating the nodes table...");
        // Creating nodes table by removing copies from the pts table.
        if (tolerance > 0) {
            if(isH2){
            st.execute("CREATE TABLE " + nodesName + "(" +
                    "NODE_ID SERIAL PRIMARY KEY, " +
                    "THE_GEOM POINT, " +
                    "EXP POLYGON" +
                    ") AS " +
                    "SELECT NULL, A.THE_GEOM, A.AREA FROM "+ PTS_TABLE + " as A," + PTS_TABLE +" as B " +
                    "WHERE A.AREA && B.AREA " +
                    "GROUP BY A.ID " +
                    "HAVING A.ID=MIN(B.ID);");
            }
            else{
               st.execute("CREATE TABLE " + nodesName + "(" +
                    "NODE_ID SERIAL PRIMARY KEY, " +
                    "THE_GEOM GEOMETRY(POINT, " + srid+"), "+
                    "EXP GEOMETRY(POLYGON," +srid+")"+
                    ") " );
                st.execute( "INSERT INTO "+nodesName +" (SELECT (row_number() over())::int , c.the_geom, c.area FROM (SELECT  A.THE_GEOM, A.AREA FROM "+ PTS_TABLE +" as  A, "+ PTS_TABLE +" as B " +
                    "WHERE A.AREA && B.AREA " +
                    "GROUP BY A.ID " +
                    "HAVING A.ID=MIN(B.ID)) as c);"); 
            }
        } else {
            
            if(isH2){
               // If the tolerance is zero, we can create the NODES table
            // by using = rather than &&.
            st.execute("CREATE TABLE " + nodesName + "(" +
                    "NODE_ID SERIAL PRIMARY KEY, " +
                    "THE_GEOM POINT" +
                    ") AS " +
                    "SELECT NULL, A.THE_GEOM FROM "+ PTS_TABLE + " as A," + PTS_TABLE + " as B " +
                    "WHERE A.THE_GEOM && B.THE_GEOM AND A.THE_GEOM=B.THE_GEOM " +
                    "GROUP BY A.ID " +
                    "HAVING A.ID=MIN(B.ID);"); 
            }
            else{
            // If the tolerance is zero, we can create the NODES table
            // by using = rather than &&.
            st.execute("CREATE TABLE " + nodesName + "(" +
                    "NODE_ID SERIAL PRIMARY KEY, " +
                    "THE_GEOM GEOMETRY(POINT, "+srid+")" +
                    ") " );            
            st.execute("INSERT INTO "+nodesName +" (SELECT (row_number() over())::int , c.the_geom FROM (SELECT A.THE_GEOM FROM "+ PTS_TABLE + " as A," + PTS_TABLE + " as B " +
                    "WHERE A.THE_GEOM && B.THE_GEOM AND A.THE_GEOM=B.THE_GEOM " +
                    "GROUP BY A.ID " +
                    "HAVING A.ID=MIN(B.ID)) as c);");
            }
        }
    }
1110167
function sumShards(store, cb) {
  exports.lazy(store.metaDb.createValueStream(), cb)
    .map(function(stream) {
      return stream.Shards.filter(function(shard) {
        return shard.SequenceNumberRange.EndingSequenceNumber == null
      }).length
    })
    .sum(function(sum) { return cb(null, sum) })
}
1026614
private Map<String, Integer> getShardNumbers(String tableName, Set<String> targObjIDs) {
        TableDefinition tableDef = m_tableDef.getAppDef().getTableDef(tableName);
        FieldDefinition shardField = tableDef.getShardingField();
        Map<String, String> shardFieldMap =
           SpiderService.instance().getObjectScalar(tableDef, targObjIDs, shardField.getName());
        Map<String, Integer> shardNoMap = new HashMap<>();
        for (String objID : shardFieldMap.keySet()) {
            Date shardingFieldDate = Utils.dateFromString(shardFieldMap.get(objID));
            int shardNo = tableDef.computeShardNumber(shardingFieldDate);
            shardNoMap.put(objID, shardNo);
        }
        return shardNoMap;
    }
969773
static boolean fragmentReferencesPersistentTable(AbstractPlanNode node) {
        if (node == null)
            return false;

        // these nodes can read/modify persistent tables
        if (node instanceof AbstractScanPlanNode)
            return true;
        if (node instanceof InsertPlanNode)
            return true;
        if (node instanceof DeletePlanNode)
            return true;
        if (node instanceof UpdatePlanNode)
            return true;

        // recursively check out children
        for (int i = 0; i < node.getChildCount(); i++) {
            AbstractPlanNode child = node.getChild(i);
            if (fragmentReferencesPersistentTable(child))
                return true;
        }

        // if nothing found, return false
        return false;
    }
1093097
private void addShardStart(TableDefinition tableDef, int shardNumber, Date shardDate) {
        SpiderTransaction spiderTran = new SpiderTransaction();
        spiderTran.addShardStart(tableDef, shardNumber, shardDate);
        Tenant tenant = Tenant.getTenant(tableDef);
        DBTransaction dbTran = DBService.instance(tenant).startTransaction();
        spiderTran.applyUpdates(dbTran);
        DBService.instance(tenant).commit(dbTran);
        synchronized (this) {
            cacheShardValue(tableDef, shardNumber, shardDate);
        }
    }
1325227
public int compareTo(Shard other) {
    // compare version
    if (version < other.version) {
      return -1;
    } else if (version > other.version) {
      return 1;
    }
    // compare dir
    int result = dir.compareTo(other.dir);
    if (result != 0) {
      return result;
    }
    // compare gen
    if (gen < other.gen) {
      return -1;
    } else if (gen == other.gen) {
      return 0;
    } else {
      return 1;
    }
  }