拆分文件

@param filePath 文件路径
@param splitPoints 拆分点数组（函数将根据拆分点来分割文件）
@param storageFolder 保存路径

@throws IOException 异常
425959
public static void splitFile(String filePath, long[] splitPoints, String storageFolder) throws IOException {
        splitFile(new File(filePath), splitPoints, storageFolder);
    }
888675
public void loadFolder(String folderPath) {
		// Get the file path.
		File path = null;
		if(folderPath != null && folderPath.length() > 0) {
			path = new File(folderPath);
		}
		
		this.loadFolder(path);
	}
15866
public static void compress(String folderPath, String zipFilePath) throws IOException {
    File folder = new File(folderPath);
    if (!folder.exists() || !folder.isDirectory()) {
      throw new IOException("Folder " + folderPath + " does't exist or isn't a directory");
    }

    File zipFile = new File(zipFilePath);
    if (!zipFile.exists()) {
      File zipFolder = zipFile.getParentFile();
      if (!zipFolder.exists()) {
        if (!zipFolder.mkdirs()) {
          throw new IOException("Zip folder " + zipFolder.getAbsolutePath() + " not created");
        }
      }
      if (!zipFile.createNewFile()) {
        throw new IOException("Zip file " + zipFilePath + " not created");
      }
    }

    BufferedInputStream bis;
    ZipOutputStream zos = new ZipOutputStream(
        new BufferedOutputStream(new FileOutputStream(zipFile)));
    try {
      final int BUFFER_SIZE = 8 * 1024; // 8K
      byte buffer[] = new byte[BUFFER_SIZE];
      for (String fileName : folder.list()) {
        if (fileName.equals(".") || fileName.equals("..")) {
          continue;
        }

        File file = new File(folder, fileName);
        if (!file.isFile()) {
          continue;
        }

        FileInputStream fis = new FileInputStream(file);
        bis = new BufferedInputStream(fis, BUFFER_SIZE);
        try {
          ZipEntry entry = new ZipEntry(fileName);
          zos.putNextEntry(entry);
          int count;
          while ((count = bis.read(buffer, 0, BUFFER_SIZE)) != -1) {
            zos.write(buffer, 0, count);
          }
        } finally {
          try {
            bis.close();
          } catch (IOException e) {
            // Ignore
          }
        }
      }
    } finally {
      try {
        zos.close();
      } catch (IOException e) {
        // Ignore
      }
    }
  }
1024107
public static void main(final String[] args) throws IOException {
		if (args == null || args.length != 2) {
			throw new IllegalArgumentException(
					"Please give the javamelody storage directory and the merging directory as arguments");
		}
		final File storageDirectory = new File(args[0]);
		final File mergingDirectory = new File(args[1]);
		final DataMerge dataMerge = new DataMerge(storageDirectory, mergingDirectory);
		dataMerge.mergeData();
	}
746301
public static String getLocalFileGroup(String filePath) throws IOException {
    PosixFileAttributes attr =
        Files.readAttributes(Paths.get(filePath), PosixFileAttributes.class);
    return attr.group().getName();
  }
888203
public static void compressList(String srcFilePath, String destFilePath) throws Exception {
		File src = new File(srcFilePath);
		if (!src.exists() || !src.isDirectory()) {
			throw new RuntimeException(srcFilePath + "不存在");
		}
		File zipFile = new File(destFilePath);
		ZipOutputStream zos=null;
		try {
			FileOutputStream fos = new FileOutputStream(zipFile);
			CheckedOutputStream cos = new CheckedOutputStream(fos, new CRC32());
			zos = new ZipOutputStream(cos);
			String baseDir = "";
			File[] files = src.listFiles();
			for (File file : files) {
				compressbyType(file, zos, baseDir);
			}			
		} finally {
			if(zos!=null)
			{
				zos.close();
			}
		}
	}
57835
public static void copyDirectories(File[] directories, String storageFolder) throws IOException {
        storageFolder = checkFolder(storageFolder) + ValueConsts.SEPARATOR;
        for (File directory : directories) {
            copyDirectory(directory, new File(storageFolder + directory.getName()));
        }
    }
532843
public static void remove(String filePath) {
        File file = new File(filePath);
        if (file.exists() && file.isFile()) {
            file.delete();
        }
    }
485819
public static void createFile(String filePath) throws IOException {
    Path storagePath = Paths.get(filePath);
    Files.createDirectories(storagePath.getParent());
    Files.createFile(storagePath);
  }
414345
public EvaluationMetrics evaluate(final String splitPath, final String strategyModelPath) throws IOException {

      EvaluationMetrics results = new EvaluationMetrics();
      double rmseResult = 0.0;
      double maeResult = 0.0;

      for (int cutoff : this.cutoffs) {
         double ndcgRes = 0.0;
         double precisionRes = 0.0;
         double recallRes = 0.0;

         for (int i = 0; i < this.numFolds; i++) {
            File testFile = new File(Paths.get(splitPath, "test_" + i + FILE_EXT).toString());
            File strategyFile = new File(Paths.get(strategyModelPath, "strategymodel_" + i + FILE_EXT).toString());
            DataModelIF<Long, Long> testModel = new SimpleParser().parseData(testFile);
            DataModelIF<Long, Long> strategyModel = new SimpleParser().parseData(strategyFile);

            // Error metrics calculated only once per fold, using all predictions
            if (cutoff == this.cutoffs[0]) {
               RMSE<Long, Long> rmse = new RMSE<>(strategyModel, testModel);
               rmse.compute();
               rmseResult += rmse.getValue();

               MAE<Long, Long> mae = new MAE<>(strategyModel, testModel);
               mae.compute();
               maeResult += mae.getValue();
            }

            // Ranking metrics
            NDCG<Long, Long> ndcg = new NDCG<>(strategyModel, testModel, new int[]{cutoff});
            ndcg.compute();
            ndcgRes += ndcg.getValueAt(cutoff);

            Precision<Long, Long> precision = new Precision<>(strategyModel, testModel, this.relevanceThreshold, new int[]{cutoff});
            precision.compute();
            precisionRes += precision.getValueAt(cutoff);

            Recall<Long, Long> recall = new Recall<>(strategyModel, testModel, this.relevanceThreshold, new int[]{cutoff});
            recall.compute();
            recallRes += recall.getValueAt(cutoff);
         }

         results.setPrecisionAtK(cutoff, precisionRes / this.numFolds);
         results.setRecallAtK(cutoff, recallRes / this.numFolds);
         results.setNDCGAtK(cutoff, ndcgRes / this.numFolds);

         log.info("Ranking metrics at {} computed", cutoff);
         log.info("NDCG@" + cutoff + ": " + ndcgRes / this.numFolds + " / P@" + cutoff + ": " + precisionRes / this.numFolds
                 + " / R@" + cutoff + ": " + recallRes / this.numFolds);

      }

      results.setMAE(maeResult / this.numFolds);
      results.setRMSE(rmseResult / this.numFolds);
      log.info("Error metrics computed");
      log.info("RMSE: " + rmseResult / this.numFolds + " / MAE: " + maeResult / this.numFolds);

      return results;
   }
837268
public static List<String> readLinesWithFilePathOrClasspath(
            String filePathOrClasspath) {
        return JMOptional
                .getOptional(JMFiles.readLines(filePathOrClasspath))
                .orElseGet(() -> getStringListAsOptWithClasspath(
                        filePathOrClasspath)
                        .orElseGet(Collections::emptyList));
    }
1216097
public static void write2File(String filePath, byte[] byteData) {
        try {
            BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(new FileOutputStream(new File(filePath)));
            bufferedOutputStream.write(byteData);
            bufferedOutputStream.close();
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
280946
public static void ensureDirectoryExists(final Path path) throws IOException {
        if (Files.exists(path)) {
            if (!Files.isDirectory(path)) {
                throw new IOException("path is not a directory: " + path);
            }
            // probably should fsync parent here just to be sure, but that might slow stuff down
        } else {
            Files.createDirectories(path);
            fsyncLineage(path.getParent());
        }
    }
929135
private static void loadFile(String filePath) {
        final Path path = FileSystems.getDefault().getPath(filePath);

        try {
            data.clear();
            data.load(Files.newBufferedReader(path));
        } catch(IOException e) {
            LOG.warn("Exception while loading " + path.toString(), e);
        }
    }
599970
private static String getFilePath(String defaultPath, String filePath) {
		return defaultPath+File.separator+filePath;
	}
457267
public static void writeIndex(final FileSystem fileSystem, final Path sstablePath) throws IOException {

        final Configuration configuration = fileSystem.getConf();

        final long splitSize = configuration.getLong(HadoopSSTableConstants.HADOOP_SSTABLE_SPLIT_MB,
                HadoopSSTableConstants.DEFAULT_SPLIT_MB) * 1024 * 1024;

        final Closer closer = Closer.create();

        final Path outputPath = sstablePath.suffix(SSTABLE_INDEX_SUFFIX);
        final Path inProgressOutputPath = sstablePath.suffix(SSTABLE_INDEX_IN_PROGRESS_SUFFIX);

        boolean success = false;
        try {
            final FSDataOutputStream os = closer.register(fileSystem.create(inProgressOutputPath));

            final TLongArrayList splitOffsets = new TLongArrayList();
            long currentStart = 0;
            long currentEnd = 0;
            final IndexOffsetScanner index = closer.register(new IndexOffsetScanner(sstablePath, fileSystem));

            while (index.hasNext()) {
                // NOTE: This does not give an exact size of this split in bytes but a rough estimate.
                // This should be good enough since it's only used for sorting splits by size in hadoop land.
                while (currentEnd - currentStart < splitSize && index.hasNext()) {
                    currentEnd = index.next();
                    splitOffsets.add(currentEnd);
                }

                // Record the split
                final long[] offsets = splitOffsets.toArray();
                os.writeLong(offsets[0]); // Start
                os.writeLong(offsets[offsets.length - 1]); // End

                // Clear the offsets
                splitOffsets.clear();

                if (index.hasNext()) {
                    currentStart = index.next();
                    currentEnd = currentStart;
                    splitOffsets.add(currentStart);
                }
            }

            success = true;
        } finally {
            closer.close();

            if (!success) {
                fileSystem.delete(inProgressOutputPath, false);
            } else {
                fileSystem.rename(inProgressOutputPath, outputPath);
            }
        }
    }
950788
public static StoreConfig newInstance(File filePath) throws IOException {
        File homeDir;
        Properties p = new Properties();
        if(filePath.exists()) {
            if(filePath.isDirectory()) {
                homeDir = filePath;
                File propertiesFile = new File(filePath, CONFIG_PROPERTIES_FILE);
                if(propertiesFile.exists()) {
                    FileReader reader = new FileReader(propertiesFile);
                    p.load(reader);
                    reader.close();
                } else {
                    throw new FileNotFoundException(propertiesFile.toString());
                }
            } else {
                homeDir = filePath.getParentFile();
                FileReader reader = new FileReader(filePath);
                p.load(reader);
                reader.close();
            }
        } else {
            throw new FileNotFoundException(filePath.toString());
        }
        
        int initialCapacity = -1;
        String initialCapacityStr = p.getProperty(StoreParams.PARAM_INITIAL_CAPACITY);
        if(initialCapacityStr == null) {
            throw new InvalidStoreConfigException(StoreParams.PARAM_INITIAL_CAPACITY + " not found");
        } else if((initialCapacity = parseInt(StoreParams.PARAM_INITIAL_CAPACITY, initialCapacityStr, -1)) < 1) {
            throw new InvalidStoreConfigException(StoreParams.PARAM_INITIAL_CAPACITY + "=" + initialCapacityStr);
        }
        
        int partitionStart = -1;
        int partitionCount = -1;
        String partitionStartStr = p.getProperty(StoreParams.PARAM_PARTITION_START);
        String partitionCountStr = p.getProperty(StoreParams.PARAM_PARTITION_COUNT);
        if(partitionStartStr != null && partitionCountStr != null) {
            if((partitionStart = parseInt(StoreParams.PARAM_PARTITION_START, partitionStartStr, -1)) < 0) {
                throw new InvalidStoreConfigException(StoreParams.PARAM_PARTITION_START + "=" + partitionStartStr);
            }
            
            if((partitionCount = parseInt(StoreParams.PARAM_PARTITION_COUNT, partitionCountStr, -1)) < 1) {
                throw new InvalidStoreConfigException(StoreParams.PARAM_PARTITION_COUNT + "=" + partitionCountStr);
            }
        } else {
            return new StoreConfig(homeDir, initialCapacity);
        }
        
        return new StorePartitionConfig(homeDir, partitionStart, partitionCount);
    }