Read a file and process it for list information.

@param ref system path of the file to process
@throws DITAOTException if processing failed
421193
private void processFile(final Reference ref) throws DITAOTException {
        currentFile = ref.filename;
        assert currentFile.isAbsolute();
        logger.info("Processing " + currentFile);
        final String[] params = { currentFile.toString() };

        try {
            XMLReader xmlSource = getXmlReader(ref.format);
            for (final XMLFilter f: getProcessingPipe(currentFile)) {
                f.setParent(xmlSource);
                f.setEntityResolver(CatalogUtils.getCatalogResolver());
                xmlSource = f;
            }
            xmlSource.setContentHandler(nullHandler);

            xmlSource.parse(currentFile.toString());

            if (listFilter.isValidInput()) {
                processParseResult(currentFile);
                categorizeCurrentFile(ref);
            } else if (!currentFile.equals(rootFile)) {
                logger.warn(MessageUtils.getMessage("DOTJ021W", params).toString());
                failureList.add(currentFile);
            }
        } catch (final RuntimeException e) {
            throw e;
        } catch (final SAXParseException sax) {
            final Exception inner = sax.getException();
            if (inner != null && inner instanceof DITAOTException) {
                throw (DITAOTException) inner;
            }
            if (currentFile.equals(rootFile)) {
                throw new DITAOTException(MessageUtils.getMessage("DOTJ012F", params).toString() + ": " + sax.getMessage(), sax);
            } else if (processingMode == Mode.STRICT) {
                throw new DITAOTException(MessageUtils.getMessage("DOTJ013E", params).toString() + ": " + sax.getMessage(), sax);
            } else {
                logger.error(MessageUtils.getMessage("DOTJ013E", params).toString() + ": " + sax.getMessage(), sax);
            }
            failureList.add(currentFile);
        } catch (final FileNotFoundException e) {
            if (!exists(currentFile)) {
                if (currentFile.equals(rootFile)) {
                    throw new DITAOTException(MessageUtils.getMessage("DOTA069F", params).toString(), e);
                } else if (processingMode == Mode.STRICT) {
                    throw new DITAOTException(MessageUtils.getMessage("DOTX008E", params).toString(), e);
                } else {
                    logger.error(MessageUtils.getMessage("DOTX008E", params).toString());
                }
            } else if (currentFile.equals(rootFile)) {
                throw new DITAOTException(MessageUtils.getMessage("DOTJ078F", params).toString() + " Cannot load file: " + e.getMessage(), e);
            } else if (processingMode == Mode.STRICT) {
                throw new DITAOTException(MessageUtils.getMessage("DOTJ079E", params).toString() + " Cannot load file: " + e.getMessage(), e);
            } else {
                logger.error(MessageUtils.getMessage("DOTJ079E", params).toString() + " Cannot load file: " + e.getMessage());
            }
            failureList.add(currentFile);
        } catch (final Exception e) {
            if (currentFile.equals(rootFile)) {
                throw new DITAOTException(MessageUtils.getMessage("DOTJ012F", params).toString() + ": " + e.getMessage(),  e);
            } else if (processingMode == Mode.STRICT) {
                throw new DITAOTException(MessageUtils.getMessage("DOTJ013E", params).toString() + ": " + e.getMessage(), e);
            } else {
                logger.error(MessageUtils.getMessage("DOTJ013E", params).toString() + ": " + e.getMessage(), e);
            }
            failureList.add(currentFile);
        }

        if (!listFilter.isValidInput() && currentFile.equals(rootFile)) {
            if (validate) {
                // stop the build if all content in the input file was filtered out.
                throw new DITAOTException(MessageUtils.getMessage("DOTJ022F", params).toString());
            } else {
                // stop the build if the content of the file is not valid.
                throw new DITAOTException(MessageUtils.getMessage("DOTJ034F", params).toString());
            }
        }

        doneList.add(currentFile);
        listFilter.reset();
        keydefFilter.reset();

    }
665498
void processParseResult(final URI currentFile) {
        // Category non-copyto result
        for (final Reference file: listFilter.getNonCopytoResult()) {
            categorizeReferenceFile(file);
        }
        for (final Map.Entry<URI, URI> e : listFilter.getCopytoMap().entrySet()) {
            final URI source = e.getValue();
            final URI target = e.getKey();
            copyTo.put(target, source);
        }
        schemeSet.addAll(listFilter.getSchemeRefSet());

        hrefTargetSet.addAll(listFilter.getHrefTargets());
        conrefTargetSet.addAll(listFilter.getConrefTargets());
        nonConrefCopytoTargetSet.addAll(listFilter.getNonConrefCopytoTargets());
        coderefTargetSet.addAll(listFilter.getCoderefTargets());
        outDitaFilesSet.addAll(listFilter.getOutFilesSet());

        // Generate topic-scheme dictionary
        final Set<URI> schemeSet = listFilter.getSchemeSet();
        if (schemeSet != null && !schemeSet.isEmpty()) {
            Set<URI> children = schemeDictionary.get(currentFile);
            if (children == null) {
                children = new HashSet<>();
            }
            children.addAll(schemeSet);
            schemeDictionary.put(currentFile, children);
            final Set<URI> hrfSet = listFilter.getHrefTargets();
            for (final URI filename: hrfSet) {
                children = schemeDictionary.get(filename);
                if (children == null) {
                    children = new HashSet<>();
                }
                children.addAll(schemeSet);
                schemeDictionary.put(filename, children);
            }
        }
    }
241345
@Override
    public void setRef(int parameterIndex, Ref x) throws SQLException {
        internalStmt.setRef(parameterIndex, x);
    }
134111
private void generateScheme(final File filename, final Document root) throws DITAOTException {
        final File p = filename.getParentFile();
        if (!p.exists() && !p.mkdirs()) {
            throw new DITAOTException("Failed to make directory " + p.getAbsolutePath());
        }
        Result res = null;
        try {
            res = new StreamResult(new FileOutputStream(filename));
            final DOMSource ds = new DOMSource(root);
            final TransformerFactory tff = TransformerFactory.newInstance();
            final Transformer tf = tff.newTransformer();
            tf.transform(ds, res);
        } catch (final RuntimeException e) {
            throw e;
        } catch (final Exception e) {
            logger.error(e.getMessage(), e) ;
            throw new DITAOTException(e);
        } finally {
            try {
                close(res);
            } catch (IOException e) {
                throw new DITAOTException(e);
            }
        }
    }
603448
@Override
    public synchronized File read()
        throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException {

        if (walker == null) {
            this.walker = new SimpleDirectoryWalker(rootDirectory);
            this.walker.start();
            //skip ahead if items have already been processed.
            long itemsRead = getItemsRead();
            if (itemsRead > 0) {
                for (int i = 0; i < itemsRead; i++) {
                    this.walker.next();
                }
            }
        }

        return this.walker.next();
    }
299768
@Override
    public AbstractPipelineOutput execute(final AbstractPipelineInput input)
            throws DITAOTException {
        if (fileInfoFilter == null) {
            fileInfoFilter = f -> f.format == null || f.format.equals(ATTR_FORMAT_VALUE_DITA) || f.format.equals(ATTR_FORMAT_VALUE_DITAMAP);
        }
        final Collection<FileInfo> fis = job.getFileInfo(fileInfoFilter).stream()
                .filter(f -> f.hasKeyref)
                .collect(Collectors.toSet());
        if (!fis.isEmpty()) {
            try {
                final String cls = Optional
                        .ofNullable(job.getProperty("temp-file-name-scheme"))
                        .orElse(configuration.get("temp-file-name-scheme"));
                tempFileNameScheme = (GenMapAndTopicListModule.TempFileNameScheme) Class.forName(cls).newInstance();
            } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
                throw new RuntimeException(e);
            }
            tempFileNameScheme.setBaseDir(job.getInputDir());
            initFilters();

            final Document doc = readMap();

            final KeyrefReader reader = new KeyrefReader();
            reader.setLogger(logger);
            final Job.FileInfo in = job.getFileInfo(fi -> fi.isInput).iterator().next();
            final URI mapFile = in.uri;
            logger.info("Reading " + job.tempDirURI.resolve(mapFile).toString());
            reader.read(job.tempDirURI.resolve(mapFile), doc);

            final KeyScope rootScope = reader.getKeyDefinition();
            final List<ResolveTask> jobs = collectProcessingTopics(fis, rootScope, doc);
            writeMap(doc);

            transtype = input.getAttribute(ANT_INVOKER_EXT_PARAM_TRANSTYPE);
            delayConrefUtils = transtype.equals(INDEX_TYPE_ECLIPSEHELP) ? new DelayConrefUtils() : null;
            for (final ResolveTask r: jobs) {
                if (r.out != null) {
                    processFile(r);
                }
            }
            for (final ResolveTask r: jobs) {
                if (r.out == null) {
                    processFile(r);
                }
            }

            // Store job configuration updates
            for (final URI file: normalProcessingRole) {
                final FileInfo f = job.getFileInfo(file);
                if (f != null) {
                    f.isResourceOnly = false;
                    job.add(f);
                }
            }

            try {
                job.write();
            } catch (final IOException e) {
                throw new DITAOTException("Failed to store job state: " + e.getMessage(), e);
            }
        }
        return null;
    }
1167317
@Override public void process(MPPReader reader, ProjectFile file, DirectoryEntry root) throws MPXJException, IOException
   {
      try
      {
         populateMemberData(reader, file, root);
         processProjectProperties();

         if (!reader.getReadPropertiesOnly())
         {
            processCalendarData();
            processResourceData();
            processTaskData();
            processConstraintData();
            processAssignmentData();

            if (reader.getReadPresentationData())
            {
               processViewPropertyData();
               processViewData();
               processTableData();
            }
         }
      }

      finally
      {
         clearMemberData();
      }
   }
136260
function getReference( referenceFile ) {
  var text = '';

  // Read references file.
  var referencePath = path.join( process.cwd(), referenceFile );

  var stats = fs.statSync( referencePath );
  if (stats && stats.isFile()) {
    text = fs.readFileSync( referencePath, { encoding: 'utf8' } );

    // Find common root definitions (tilde references).
    var tildes = { };
    var re = /^\s*\[(~\d*)]\s*:/g;
    var source = text.split( '\n' );
    var target = [ ];
    for (var i = 0; i < source.length; i++) {
      var line = source[ i ].trim();
      // Search common root pattern.
      var result = re.exec( line );
      if (result)
      // Save common root value.
        tildes[ result[ 1 ] ] = line.substring( line.indexOf( ':' ) + 1 ).trim();
      else if (line[ 0 ] === '[')
      // Store reference link.
        target.push( line );
    }
    text = target.join( '\n' );

    // Replace tilde+number pairs with common roots.
    var noIndex = false;
    for (var attr in tildes) {
      if (tildes.hasOwnProperty( attr )) {
        if (attr === '~')
          noIndex = true;
        else {
          var re_n = new RegExp( ' ' + attr, 'g' );
          var re_value = ' ' + tildes[ attr ];
          text = text.replace( re_n, re_value );
        }
      }
    }
    // Finally replace tildes with its common root.
    if (noIndex)
      text = text.replace( / ~/g, ' ' + tildes[ '~' ] );
  }
  return text;
}
671836
public void process( String uniprotDatFileName,String fastaFileName ) throws Exception{

			FileReader fr = new FileReader(uniprotDatFileName);
			BufferedReader br = new BufferedReader(fr);
			String line = br.readLine();
			String id = "";
			StringBuffer sequence = new StringBuffer();
			ArrayList<ProteinSequence> seqCodingRegionsList = new ArrayList<ProteinSequence>();
			int count = 0;
			HashMap<String,String> uniqueGenes = new HashMap<String,String>();
			HashMap<String,String> uniqueSpecies = new HashMap<String,String>();
			while(line != null){
				if(line.startsWith("ID")){
					String[] data = line.split(" ");
					id = data[3];
				}else if(line.startsWith("SQ")){
					line = br.readLine();
					while(!line.startsWith("//")){

						for(int i = 0; i < line.length(); i++){
							char aa = line.charAt(i);
							if((aa >= 'A' && aa <= 'Z') || (aa >= 'a' && aa <= 'z' )){
								sequence.append(aa);
							}
						}
						line = br.readLine();
					}

				 //   System.out.println(">" + id);
				 //   System.out.println(sequence.toString());

					ProteinSequence seq = new ProteinSequence(sequence.toString() );
					seq.setAccession(new AccessionID(id));

					seqCodingRegionsList.add(seq);
					sequence = new StringBuffer();
					count++;
					if(count % 100 == 0)
						logger.info("Count: ", count);
					String[] parts = id.split("_");
					uniqueGenes.put(parts[0], "");
					uniqueSpecies.put(parts[1],"");
				}
				line = br.readLine();
			}
	   //     System.out.println("Unique Genes=" + uniqueGenes.size());
	   //     System.out.println("Unique Species=" + uniqueSpecies.size());
	   //     System.out.println("Total sequences=" + seqCodingRegionsList.size());
			FastaWriterHelper.writeProteinSequence(new File(fastaFileName), seqCodingRegionsList);

			br.close();
			fr.close();

	  //      System.out.println(uniqueGenes.keySet());
	  //      System.out.println("====================");
	  //      System.out.println(uniqueSpecies.keySet());


	}
823962
public void read() throws IOException
   {
      SecurityHelper.doPrivilegedIOExceptionAction(new PrivilegedExceptionAction<Object>()
      {
         public Object run() throws Exception
         {
            // Known issue for NFS bases on ext3. Need to refresh directory to read actual data.
            dir.listAll();

            names.clear();
            indexes.clear();
            if (dir.fileExists(name))
            {
               // clear current lists
               InputStream in = new IndexInputStream(dir.openInput(name));
               DataInputStream di = null;
               try
               {
                  di = new DataInputStream(in);
                  counter = di.readInt();
                  for (int i = di.readInt(); i > 0; i--)
                  {
                     String indexName = di.readUTF();
                     indexes.add(indexName);
                     names.add(indexName);
                  }
               }
               finally
               {
                  if (di != null)
                     di.close();
                  in.close();
               }
            }
            return null;
         }
      });
   }
1027944
public final static void readBytesRef(BytesRef src, BytesRef dst) {
    int length = Bytes.readVInt(src);

    if (length == 0) {
      dst.offset = dst.length = 0;
      return;
    }

    if (dst.bytes.length < length) {
      dst.bytes = new byte[length];
    }

    System.arraycopy(src.bytes, src.offset, dst.bytes, 0, length);

    src.offset += length;
    dst.offset = 0;
    dst.length = length;
  }
1184375
void process(byte ttfAfm[], boolean preload) throws DocumentException, IOException {
        tables = new HashMap();
        
        try {
            if (ttfAfm == null)
                rf = new RandomAccessFileOrArray(fileName, preload, Document.plainRandomAccess);
            else
                rf = new RandomAccessFileOrArray(ttfAfm);
            if (ttcIndex.length() > 0) {
                int dirIdx = Integer.parseInt(ttcIndex);
                if (dirIdx < 0)
                    throw new DocumentException("The font index for " + fileName + " must be positive.");
                String mainTag = readStandardString(4);
                if (!mainTag.equals("ttcf"))
                    throw new DocumentException(fileName + " is not a valid TTC file.");
                rf.skipBytes(4);
                int dirCount = rf.readInt();
                if (dirIdx >= dirCount)
                    throw new DocumentException("The font index for " + fileName + " must be between 0 and " + (dirCount - 1) + ". It was " + dirIdx + ".");
                rf.skipBytes(dirIdx * 4);
                directoryOffset = rf.readInt();
            }
            rf.seek(directoryOffset);
            int ttId = rf.readInt();
            if (ttId != 0x00010000 && ttId != 0x4F54544F)
                throw new DocumentException(fileName + " is not a valid TTF or OTF file.");
            int num_tables = rf.readUnsignedShort();
            rf.skipBytes(6);
            for (int k = 0; k < num_tables; ++k) {
                String tag = readStandardString(4);
                rf.skipBytes(4);
                int table_location[] = new int[2];
                table_location[0] = rf.readInt();
                table_location[1] = rf.readInt();
                tables.put(tag, table_location);
            }
            checkCff();
            fontName = getBaseFont();
            fullName = getNames(4); //full name
            familyName = getNames(1); //family name
            allNameEntries = getAllNames();
            if (!justNames) {
                fillTables();
                readGlyphWidths();
                readCMaps();
                readKerning();
                readBbox();
//                GlyphWidths = null;
            }
        }
        finally {
            if (rf != null) {
                rf.close();
                if (!embedded)
                    rf = null;
            }
        }
    }
479491
private void processDirectory(File directory) throws IOException, ClassNotFoundException {

        findFiles(directory);
        File[] files = directory.listFiles();
        if (files != null) {
            for (File file : files) {
                if (file.isDirectory()) {
                    processDirectory(file);
                }
            }
        }
    }
910198
public void process(DirectoryEntry projectDir, ProjectFile file, DocumentInputStreamFactory inputStreamFactory) throws IOException
   {
      DirectoryEntry consDir;
      try
      {
         consDir = (DirectoryEntry) projectDir.getEntry("TBkndCons");
      }

      catch (FileNotFoundException ex)
      {
         consDir = null;
      }

      if (consDir != null)
      {
         FixedMeta consFixedMeta = new FixedMeta(new DocumentInputStream(((DocumentEntry) consDir.getEntry("FixedMeta"))), 10);
         FixedData consFixedData = new FixedData(consFixedMeta, 20, inputStreamFactory.getInstance(consDir, "FixedData"));
         //         FixedMeta consFixed2Meta = new FixedMeta(new DocumentInputStream(((DocumentEntry) consDir.getEntry("Fixed2Meta"))), 9);
         //         FixedData consFixed2Data = new FixedData(consFixed2Meta, 48, getEncryptableInputStream(consDir, "Fixed2Data"));

         int count = consFixedMeta.getAdjustedItemCount();
         int lastConstraintID = -1;

         ProjectProperties properties = file.getProjectProperties();
         EventManager eventManager = file.getEventManager();

         boolean project15 = NumberHelper.getInt(properties.getMppFileType()) == 14 && NumberHelper.getInt(properties.getApplicationVersion()) > ApplicationVersion.PROJECT_2010;
         int durationUnitsOffset = project15 ? 18 : 14;
         int durationOffset = project15 ? 14 : 16;

         for (int loop = 0; loop < count; loop++)
         {
            byte[] metaData = consFixedMeta.getByteArrayValue(loop);

            //
            // SourceForge bug 2209477: we were reading an int here, but
            // it looks like the deleted flag is just a short.
            //
            if (MPPUtility.getShort(metaData, 0) != 0)
            {
               continue;
            }

            int index = consFixedData.getIndexFromOffset(MPPUtility.getInt(metaData, 4));
            if (index == -1)
            {
               continue;
            }

            //
            // Do we have enough data?
            //
            byte[] data = consFixedData.getByteArrayValue(index);
            if (data.length < 14)
            {
               continue;
            }

            int constraintID = MPPUtility.getInt(data, 0);
            if (constraintID <= lastConstraintID)
            {
               continue;
            }

            lastConstraintID = constraintID;
            int taskID1 = MPPUtility.getInt(data, 4);
            int taskID2 = MPPUtility.getInt(data, 8);

            if (taskID1 == taskID2)
            {
               continue;
            }

            // byte[] metaData2 = consFixed2Meta.getByteArrayValue(loop);
            // int index2 = consFixed2Data.getIndexFromOffset(MPPUtility.getInt(metaData2, 4));
            // byte[] data2 = consFixed2Data.getByteArrayValue(index2);

            Task task1 = file.getTaskByUniqueID(Integer.valueOf(taskID1));
            Task task2 = file.getTaskByUniqueID(Integer.valueOf(taskID2));
            if (task1 != null && task2 != null)
            {
               RelationType type = RelationType.getInstance(MPPUtility.getShort(data, 12));
               TimeUnit durationUnits = MPPUtility.getDurationTimeUnits(MPPUtility.getShort(data, durationUnitsOffset));
               Duration lag = MPPUtility.getAdjustedDuration(properties, MPPUtility.getInt(data, durationOffset), durationUnits);
               Relation relation = task2.addPredecessor(task1, type, lag);
               relation.setUniqueID(Integer.valueOf(constraintID));
               eventManager.fireRelationReadEvent(relation);
            }
         }
      }
   }
325231
public void restoreFromFile(File file) throws IOException
    {
        byte[] rawData = new byte[(int) file.length()];
        ByteBuffer bufData = null;
        FileInputStream fis = null;
        DataInputStream dis = null;
        try {
            fis = new FileInputStream(file);
            dis = new DataInputStream(fis);
            dis.readFully(rawData);
            bufData = ByteBuffer.wrap(rawData);
            restoreFromBuffer(bufData);
        }
        finally {
            if (dis != null) {
                dis.close();
            }
            if (fis != null) {
                fis.close();
            }
        }
    }
299544
public static String getTerm(IndexInput inTerm, Long ref) throws IOException {
    try {
      inTerm.seek(ref);
      return inTerm.readString();
    } catch (Exception e) {
      throw new IOException(e);
    }
  }
1275776
private void processParseResult(final URI currentFile) {
        // Category non-copyto result and update uplevels accordingly
        for (final Reference file: listFilter.getNonCopytoResult()) {
            categorizeReferenceFile(file);
            updateUplevels(file.filename);
        }
        for (final Map.Entry<URI, URI> e : listFilter.getCopytoMap().entrySet()) {
            final URI source = e.getValue();
            final URI target = e.getKey();
            copyTo.put(target, source);
            updateUplevels(target);

        }
        schemeSet.addAll(listFilter.getSchemeRefSet());

        // collect key definitions
        for (final Map.Entry<String, KeyDef> e: keydefFilter.getKeysDMap().entrySet()) {
            // key and value.keys will differ when keydef is a redirect to another keydef
            final String key = e.getKey();
            final KeyDef value = e.getValue();
            if (schemeSet.contains(currentFile)) {
                schemekeydefMap.put(key, new KeyDef(key, value.href, value.scope, value.format, currentFile, null));
            }
        }

        hrefTargetSet.addAll(listFilter.getHrefTargets());
        conrefTargetSet.addAll(listFilter.getConrefTargets());
        nonConrefCopytoTargetSet.addAll(listFilter.getNonConrefCopytoTargets());
        coderefTargetSet.addAll(listFilter.getCoderefTargets());
        outDitaFilesSet.addAll(listFilter.getOutFilesSet());

        // Generate topic-scheme dictionary
        final Set<URI> schemeSet = listFilter.getSchemeSet();
        if (schemeSet != null && !schemeSet.isEmpty()) {
            Set<URI> children = schemeDictionary.get(currentFile);
            if (children == null) {
                children = new HashSet<>();
            }
            children.addAll(schemeSet);
            schemeDictionary.put(currentFile, children);
            final Set<URI> hrfSet = listFilter.getHrefTargets();
            for (final URI filename: hrfSet) {
                children = schemeDictionary.get(filename);
                if (children == null) {
                    children = new HashSet<>();
                }
                children.addAll(schemeSet);
                schemeDictionary.put(filename, children);
            }
        }
    }