Run an integrity check on the store files and fix them as necessary
@throws DataStoreException if the files could not be fixed
322246
protected void integrityCheck() throws DataStoreException
    {
        try
        {
            //========================
            // 1 - Check files sizes
            //========================
            // -- Allocation table
            long atFileSize = allocationTableRandomAccessFile.length();
            if (atFileSize < AT_HEADER_SIZE+AT_BLOCK_SIZE) /* Should have at least one entry */
                throw new DataStoreException("Allocation table is truncated : "+allocationTableFile.getAbsolutePath());
            
            // Read some header fields
            FileInputStream inFile = new FileInputStream(allocationTableFile);
            DataInputStream in = new DataInputStream(new BufferedInputStream(inFile,16384));
            int blockCount = in.readInt();
            int blockSize  = in.readInt();
            int firstBlock = in.readInt();
            // Fix AT size
            long expectedATFileSize = AT_HEADER_SIZE+AT_BLOCK_SIZE*(long)blockCount;
            if (atFileSize != expectedATFileSize)
            {
                log.error("["+descriptor.getName()+"] Allocation table has an invalid size (actual:"+atFileSize+",expected:"+expectedATFileSize+"), fixing.");
                allocationTableRandomAccessFile.setLength(expectedATFileSize);
            }
            // Fix data size
            long dataFileSize = dataRandomAccessFile.length();
            long expectedDataFileSize = (long)blockSize*blockCount;
            if (dataFileSize != expectedDataFileSize)
            {
                log.error("["+descriptor.getName()+"] Data file has an invalid size (actual:"+dataFileSize+",expected:"+expectedDataFileSize+"), fixing.");
                dataRandomAccessFile.setLength(expectedDataFileSize);
            }
            
            //============================
            // 2 - Check allocation table
            //============================
            // Read the AT into memory
            byte[] flags        = new byte[blockCount];
            int[] allocatedSize = new int[blockCount];
            int[] previousBlock = new int[blockCount];
            int[] nextBlock     = new int[blockCount];
            int blocksInUse = 0;
            int msgCount = 0;
            for (int n = 0 ; n < blockCount ; n++)
            {
                flags[n]         = in.readByte();
                allocatedSize[n] = in.readInt();
                previousBlock[n] = in.readInt();
                nextBlock[n]     = in.readInt();
                if (allocatedSize[n] != -1)
                {
                    blocksInUse++;
                    if ((flags[n] & FLAG_START_BLOCK) > 0)
                        msgCount++;
                }
            }
            in.close();
            log.debug("["+descriptor.getName()+"] Blocks in use before fix : "+blocksInUse);
            log.debug("["+descriptor.getName()+"] Messages count before fix : "+msgCount);
            
            // Fix first block index
            boolean changed = false;
            if (firstBlock < -1 || firstBlock >= blockCount)
            {
                log.error("["+descriptor.getName()+"] Invalid allocation table first block index ("+firstBlock+"), guessing new one ...");
                firstBlock = guessFirstBlockIndex(blockCount, allocatedSize, nextBlock);
                log.debug("["+descriptor.getName()+"] Guessed first block index : "+firstBlock);
                changed = true;
            }
            
            // Recover table
            if (msgCount == 0)
            {
                if (firstBlock == -1)
                {
                    // Table is empty, cleanup dirty entries
                    changed = changed || cleanupEmptyBlocks(blockCount, flags, allocatedSize, previousBlock, nextBlock);
                }
                else
                {
                    log.error("["+descriptor.getName()+"] First block index should be -1, clearing ...");
                    firstBlock = -1;
                    changed = true;
                }
            }
            else
            {
                if (firstBlock == -1)
                {
                    log.error("["+descriptor.getName()+"] Invalid first block index, guessing value ...");
                    firstBlock = guessFirstBlockIndex(blockCount, allocatedSize, nextBlock);
                    log.debug("["+descriptor.getName()+"] Guessed first block index : "+firstBlock);
                    changed = true;
                }
     
                changed = changed || fixBlocks(blockCount, blockSize, firstBlock, flags, allocatedSize, previousBlock, nextBlock);
                changed = changed || cleanupEmptyBlocks(blockCount, flags, allocatedSize, previousBlock, nextBlock);
            }
            
            // Update the allocation file table
            if (changed)
            {
                // Re-compute size
                msgCount = 0;
                blocksInUse = 0;
                for (int n = 0 ; n < blockCount ; n++)
                {
                    if (allocatedSize[n] != -1) 
                    {
                        blocksInUse++;
                        if ((flags[n] & FLAG_START_BLOCK) > 0)
                            msgCount++;
                    }
                }
                log.debug("["+descriptor.getName()+"] Blocks in use after fix : "+blocksInUse);
                log.debug("["+descriptor.getName()+"] Messages count after fix : "+msgCount);
                
                log.debug("["+descriptor.getName()+"] Allocation table was altered, saving ...");
                allocationTableRandomAccessFile.seek(AT_HEADER_FIRSTBLOCK_OFFSET);
                allocationTableRandomAccessFile.writeInt(firstBlock);
                for (int n = 0 ; n < blockCount ; n++)
                {
                    byte[] allocationBlock = new byte[AT_BLOCK_SIZE];

                    // Regroup I/O to improve performance
                    allocationBlock[AB_FLAGS_OFFSET]       = flags[n];
                    allocationBlock[AB_ALLOCSIZE_OFFSET]   = (byte)((allocatedSize[n] >>> 24) & 0xFF);
                    allocationBlock[AB_ALLOCSIZE_OFFSET+1] = (byte)((allocatedSize[n] >>> 16) & 0xFF);
                    allocationBlock[AB_ALLOCSIZE_OFFSET+2] = (byte)((allocatedSize[n] >>>  8) & 0xFF);
                    allocationBlock[AB_ALLOCSIZE_OFFSET+3] = (byte)((allocatedSize[n] >>>  0) & 0xFF);
                    allocationBlock[AB_PREVBLOCK_OFFSET]   = (byte)((previousBlock[n] >>> 24) & 0xFF);
                    allocationBlock[AB_PREVBLOCK_OFFSET+1] = (byte)((previousBlock[n] >>> 16) & 0xFF);
                    allocationBlock[AB_PREVBLOCK_OFFSET+2] = (byte)((previousBlock[n] >>>  8) & 0xFF);
                    allocationBlock[AB_PREVBLOCK_OFFSET+3] = (byte)((previousBlock[n] >>>  0) & 0xFF);
                    allocationBlock[AB_NEXTBLOCK_OFFSET]   = (byte)((nextBlock[n] >>> 24) & 0xFF);
                    allocationBlock[AB_NEXTBLOCK_OFFSET+1] = (byte)((nextBlock[n] >>> 16) & 0xFF);
                    allocationBlock[AB_NEXTBLOCK_OFFSET+2] = (byte)((nextBlock[n] >>>  8) & 0xFF);
                    allocationBlock[AB_NEXTBLOCK_OFFSET+3] = (byte)((nextBlock[n] >>>  0) & 0xFF);
                    
                    allocationTableRandomAccessFile.seek(AT_HEADER_SIZE+n*AT_BLOCK_SIZE);
                    allocationTableRandomAccessFile.write(allocationBlock);
                }
                allocationTableRandomAccessFile.getFD().sync();
            }
            else
                log.debug("["+descriptor.getName()+"] Allocation table was not altered");
        }
        catch (IOException e)
        {
            throw new DataStoreException("Cannot check/fix store integrity : "+e);
        }
    }
1321577
private Key insert(Entity entity) throws DatastoreException {
    CommitRequest req = CommitRequest.newBuilder()
	.addMutations(Mutation.newBuilder()
	    .setInsert(entity))
        .setMode(CommitRequest.Mode.NON_TRANSACTIONAL)
	.build();
    return datastore.commit(req).getMutationResults(0).getKey();
  }
223852
protected void store(SecurityEvent auditEvent) throws Exception {
		if (myDataStore == null)
			throw new InternalErrorException("No data store provided to persist audit events");
		myDataStore.store(auditEvent);
	}
1004647
protected function _storePrecheck() {
		if (empty($this->args[0])) {
			$this->abort('No file provided!');
		}

		if (!file_exists($this->args[0])) {
			$this->abort('The file does not exist!');
		}

		$adapterConfig = StorageManager::config($this->params['adapter']);
		if (empty($adapterConfig)) {
			$this->abort(sprintf('Invalid adapter config `%s` provided!', $this->params['adapter']));
		}
	}
20539
void reset(boolean deleteFile) {
        String path = this.path;
        this.path = null;
        this.projectId = -1;
        this.projectToken = null;
        this.deviceId = null;

        this.client = null;

        synchronized (dataStoreLock) {
            dataStore = null;
            dataBatches.clear();
        }

        if (deleteFile) {
            File f = new File(path, DEVICE_FILENAME);
            if (f.exists()) {
                f.delete();
            }
        }
    }
761639
@Override
	protected synchronized void checkDataMaps() throws IOException {
		if (lastUpdateCheck == 0 || lastUpdateCheck < System.currentTimeMillis() - updateInterval) {
			String versionOnServer = getVersionFromServer();
			if (currentVersion == null || versionOnServer.compareTo(currentVersion) > 0) {
				loadDataFromInternetAndSave();
				loadDataFromFile(getCacheFile());
				currentVersion = versionOnServer;
				prop.setProperty("currentVersion", currentVersion);
			}
			lastUpdateCheck = System.currentTimeMillis();
			prop.setProperty("lastUpdateCheck", Long.toString(lastUpdateCheck));
			saveProperties(prop);
		}
	}
1214120
static EntityManagerException wrap(DatastoreException exp) {
    switch (exp.getCode()) {
      case ERROR_CODE_ENTITY_NOT_FOUND:
        return new EntityNotFoundException(exp);
      case ERROR_CODE_ENTITY_ALREADY_EXISTS:
        return new EntityAlreadyExistsException(exp);
      default:
        return new EntityManagerException(exp);

    }
  }
1155485
def validate_integrity(self):
        
        self._validate_columns(self._columns)
        self._validate_index(self._index)
        self._validate_data()
563349
public void flushToDatastore() {
		if ( getAssociation().isEmpty() ) {
			gridDialect.removeAssociation( getAssociationKey(), getAssociationContext() );
			association = null;
			OgmEntityEntryState.getStateFor( session, hostingEntity ).setAssociation( associationKeyMetadata.getCollectionRole(), null );
		}
		else if ( !getAssociation().getOperations().isEmpty() ) {
			gridDialect.insertOrUpdateAssociation( getAssociationKey(), getAssociation(), getAssociationContext() );
		}

		updateHostingEntityIfRequired();
	}
378121
private void onCopyFailure(
      KeySetView<IOException, Boolean> innerExceptions,
      GoogleJsonError e,
      String srcBucketName, String srcObjectName) {
    if (errorExtractor.itemNotFound(e)) {
      FileNotFoundException fnfe =
          GoogleCloudStorageExceptions.getFileNotFoundException(srcBucketName, srcObjectName);
      innerExceptions.add((FileNotFoundException) fnfe.initCause(new IOException(e.toString())));
    } else {
      String srcString = StorageResourceId.createReadableString(srcBucketName, srcObjectName);
      innerExceptions.add(new IOException(String.format("Error copying %s:%n%s", srcString, e)));
    }
  }
1101633
@Override
	public void commit() {
		List<Key> deletes = new ArrayList<Key>();
		deletes.add(Translation.toEntity(lock, Keys.create(
				Translation.toKind(entity.getClass()), 
				Entities.keyValue(entity))).getKey());
		Key transactionKey = KeyFactory.stringToKey(lock.transaction());
		
		if (Keys.ancestor(transactionKey).equals(
				Keys.create(Translation.toKind(entity.getClass()), Entities.keyValue(entity)))) {
			deletes.add(transactionKey);
			
			for (Entity entity : datastore.query(
					new Query(transactionKey).setKeysOnly())) {
				deletes.add(entity.getKey());
			}
		}
		
		Key parent = Keys.create(Translation.toKind(entity.getClass()), Entities.keyValue(entity));
		apply(operation, entity, Translation.toEntity(lock, parent).getKey(), deletes);
	}
485295
public function checkIntegrityData()
    {
        $prevId = 0;
        $identifier = $this->getIdentifier();

        foreach ($this as $item) {
            $this->checkIdentifierType($item[$identifier]);

            if ($item[$identifier] < $prevId) {
                throw new DataStoreException("This storage type supports only a list ordered by id ASC");
            }

            $prevId = $item[$identifier];
        }

        return true;
    }
319165
function DataStore ( obj ) {
	this.autosave    = false;
	this.callback    = null;
	this.collections = [];
	this.credentials = null;
	this.datalists   = [];
	this.depth       = 0;
	this.events      = false;
	this.expires     = null;
	this.headers     = {Accept: "application/json"};
	this.ignore      = [];
	this.key         = null;
	this.keys        = {};
	this.leafs       = [];
	this.loaded      = false;
	this.maxDepth    = 0;
	this.mongodb     = "";
	this.parentNode  = obj;
	this.pointer     = null;
	this.records     = [];
	this.retrieve    = false;
	this.source      = null;
	this.total       = 0;
	this.views       = {};
	this.versions    = {};
	this.versioning  = true;
	this.uri         = null;
}
1349997
public static void doMetaCheck(AdminClient adminClient, List<String> metaKeys) {
            Map<String, Integer> roStoreToReplicationFactorMap = new HashMap<String, Integer>();
            Collection<Node> allNodes = adminClient.getAdminClientCluster().getNodes();

            for(String key: metaKeys) {
                Map<String, Map<Object, List<String>>> storeDefToNodeMap = new HashMap<String, Map<Object, List<String>>>(); 
                Map<Object, List<String>> metadataNodeValueMap = new HashMap<Object, List<String>>();
                
                Collection<String> allNodeNames = new ArrayList<String>();

                Boolean checkResult = true;
                for(Node node: allNodes) {
                    String nodeName = "Host '" + node.getHost() + "' : ID " + node.getId();
                    allNodeNames.add(nodeName);

                    Versioned<String> versioned = adminClient.metadataMgmtOps.getRemoteMetadata(node.getId(),
                                                                                                key);
                    if(versioned == null || versioned.getValue() == null) {
                        throw new VoldemortException("Value returned from node " + node.getId()
                                                     + " was null");
                    } else if(key.compareTo(MetadataStore.STORES_KEY) == 0) {
                        List<StoreDefinition> storeDefinitions = new StoreDefinitionsMapper().readStoreList(new StringReader(versioned.getValue()));
                        for(StoreDefinition storeDef: storeDefinitions) {
                            String storeName = storeDef.getName();

                            if(storeDef.getType().compareTo(ReadOnlyStorageConfiguration.TYPE_NAME) == 0) {
                                roStoreToReplicationFactorMap.put(storeName, storeDef.getReplicationFactor());
                            }

                            if(storeDefToNodeMap.containsKey(storeName) == false) {
                                storeDefToNodeMap.put(storeName,
                                                      new HashMap<Object, List<String>>());
                            }
                            Map<Object, List<String>> storeDefMap = storeDefToNodeMap.get(storeName);
                            addMetadataValue(storeDefMap, storeDef, nodeName);
                            
                            // Try a random Get against one key for store/Node
                            checkGetStore(adminClient, storeDef, node);
                        }
                    } else {
                        if(key.compareTo(MetadataStore.CLUSTER_KEY) == 0
                           || key.compareTo(MetadataStore.REBALANCING_SOURCE_CLUSTER_XML) == 0) {
                            Cluster cluster = new ClusterMapper().readCluster(new StringReader(versioned.getValue()));
                            addMetadataValue(metadataNodeValueMap, cluster, nodeName);
                        } else if(key.compareTo(MetadataStore.SERVER_STATE_KEY) == 0) {
                            VoldemortState voldemortStateValue = VoldemortState.valueOf(versioned.getValue());
                            addMetadataValue(metadataNodeValueMap, voldemortStateValue, nodeName);
                        } else {
                            throw new VoldemortException("Incorrect metadata key");
                        }

                    }
                }

                if(metadataNodeValueMap.size() > 0) {
                    checkResult &= checkDiagnostics("Key " + key,
                                                    metadataNodeValueMap,
                                                    allNodeNames);
                }

                if(storeDefToNodeMap.size() > 0) {
                    for(Map.Entry<String, Map<Object, List<String>>> storeNodeValueEntry: storeDefToNodeMap.entrySet()) {
                        String storeName = storeNodeValueEntry.getKey();
                        Map<Object, List<String>> storeDefMap = storeNodeValueEntry.getValue();
                        checkResult &= checkDiagnostics("Store " + storeName,
                                                        storeDefMap,
                                                        allNodeNames);
                        for(QuotaType type : QuotaType.values()) {
                          Map<Object, List<String>> quotaNodeValues = new HashMap<Object, List<String>>();
                          String keyName = "Store:" + storeName + "_QuotaType_"+ type;
                          for(Node node: allNodes) {                          
                            String nodeName = "Host '" + node.getHost() + "' : ID " + node.getId();
                            Versioned<String> quotaValue = adminClient.quotaMgmtOps.getQuotaForNode(storeName, type, node.getId());
                            if(quotaValue == null || quotaValue.getValue() == null) {
                              continue;
                            }
                            addMetadataValue( quotaNodeValues , quotaValue.getValue() , nodeName);
                          }

                          if(quotaNodeValues.size() > 0) {
                            boolean result = checkDiagnostics(keyName,
                                quotaNodeValues,
                                allNodeNames);
                            checkResult &= result;
                          }
                        }
                    }
                }

                System.out.println(key + " metadata check : " + (checkResult ? "PASSED" : "FAILED"));
            }

            validateROFiles(adminClient, allNodes, roStoreToReplicationFactorMap);
        }
233566
private void checkState() throws H2DBLockException {
        if (!lockFile.getParentFile().isDirectory() && !lockFile.mkdir()) {
            throw new H2DBLockException("Unable to create path to data directory.");
        }
        if (lockFile.isFile()) {
            //TODO - this 30 minute check needs to be configurable.
            if (getFileAge(lockFile) > 30) {
                LOGGER.debug("An old db update lock file was found: {}", lockFile.getAbsolutePath());
                if (!lockFile.delete()) {
                    LOGGER.warn("An old db update lock file was found but the system was unable to delete "
                            + "the file. Consider manually deleting {}", lockFile.getAbsolutePath());
                }
            } else {
                LOGGER.info("Lock file found `{}`", lockFile);
                LOGGER.info("Existing update in progress; waiting for update to complete");
            }
        }
    }
807909
public void resetStore() throws BlockStoreException {
        maybeConnect();
        try {
            deleteStore();
            createTables();
            initFromDatabase();
        } catch (SQLException ex) {
            throw new RuntimeException(ex);
        }
    }
105181
static void saveSchemas(MessageStore msgStore, JMFSchema[] schemas) throws MessageEncodeFailedException {
    if (TraceComponent.isAnyTracingEnabled() && tc.isEntryEnabled()) SibTr.entry(tc, "saveSchemas");
    try {
      // Get the SchemaStore reference, creating a new one if necessary
      SchemaStoreItemStream schemaStore = getSchemaStore(msgStore);

      // Check if any new schemas need to be persisted to the SchemaStore
      for (int i = 0; i < schemas.length; i++) {
        if (!schemaStore.containsSchema(schemas[i].getLongID())) {
          if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) SibTr.debug(tc, "Saving schema:" + schemas[i].getID());
          schemaStore.addSchema(schemas[i], msgStore.getTransactionFactory().createAutoCommitTransaction());
        }
      }
    } catch (MessageStoreException e) {
      FFDCFilter.processException(e, "saveSchemas", "83");
      throw new MessageEncodeFailedException(e);
    }
    if (TraceComponent.isAnyTracingEnabled() && tc.isEntryEnabled()) SibTr.exit(tc, "saveSchemas");
  }