Process an item previously read by the reader

@param itemRead
the item read
@return the processed item
331714
private Object processItem(Object itemRead) {
		logger.entering(sourceClass, "processItem", itemRead);
		Object processedItem = null;

		// if no processor defined for this chunk
		if (processorProxy == null){
			return itemRead;
		}

		try {

			// call process listeners before and after the actual process call
			for (ItemProcessListenerProxy processListenerProxy : itemProcessListeners) {
				processListenerProxy.beforeProcess(itemRead);
			}

			processedItem = processorProxy.processItem(itemRead);

			if (processedItem == null) {
				currentItemStatus.setFiltered(true);
			}

			for (ItemProcessListenerProxy processListenerProxy : itemProcessListeners) {
				processListenerProxy.afterProcess(itemRead, processedItem);
			}
		} catch (Exception e) {
			for (ItemProcessListenerProxy processListenerProxy : itemProcessListeners) {
				processListenerProxy.onProcessError(itemRead, e);
			}
			if(!currentChunkStatus.isRetryingAfterRollback()) {
				if (retryProcessException(e, itemRead)) {
					if (!retryHandler.isRollbackException(e)) {
						processedItem = processItem(itemRead);
					} else {
						currentChunkStatus.markForRollbackWithRetry(e);
					}
				}
				else if (skipProcessException(e, itemRead)) {
					currentItemStatus.setSkipped(true);
					stepContext.getMetric(MetricImpl.MetricType.PROCESS_SKIP_COUNT).incValue();
				}
				else {
					throw new BatchContainerRuntimeException(e);
				}
			}
			else {
				if (skipProcessException(e, itemRead)) {
					currentItemStatus.setSkipped(true);
					stepContext.getMetric(MetricImpl.MetricType.PROCESS_SKIP_COUNT).incValue();
				} else if (retryProcessException(e, itemRead)) {

					if (!retryHandler.isRollbackException(e)) {
						// retry without rollback
						processedItem = processItem(itemRead);
					} else {
						// retry with rollback
						currentChunkStatus.markForRollbackWithRetry(e);
					}
				} else {
					throw new BatchContainerRuntimeException(e);
				}
			}

		} catch (Throwable e) {
			throw new BatchContainerRuntimeException(e);
		}

		logger.exiting(sourceClass, "processItem", processedItem==null ? "<null>" : processedItem);
		return processedItem;
	}
22491
public <T> T read(ReaderHandler<T> readerHandler) throws IORuntimeException {
		BufferedReader reader = null;
		T result = null;
		try {
			reader = FileUtil.getReader(this.file, charset);
			result = readerHandler.handle(reader);
		} catch (IOException e) {
			throw new IORuntimeException(e);
		} finally {
			IoUtil.close(reader);
		}
		return result;
	}
901222
public void process() throws IOException {
        int pos = 0;
        do {
            int meta = findNextMeta(text, pos);
            if (meta >= 0) {
                emitLiteral(text.substring(pos, meta));
                emitLiteral(map.getReplacement(text.substring(meta, meta + 1)));
                pos = meta + 1;
            } else {
                emitLiteral(text.substring(pos, text.length()));
                pos = text.length();
            }
        } while (pos < text.length());
    }
603448
@Override
    public synchronized File read()
        throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException {

        if (walker == null) {
            this.walker = new SimpleDirectoryWalker(rootDirectory);
            this.walker.start();
            //skip ahead if items have already been processed.
            long itemsRead = getItemsRead();
            if (itemsRead > 0) {
                for (int i = 0; i < itemsRead; i++) {
                    this.walker.next();
                }
            }
        }

        return this.walker.next();
    }
145805
def process_items(r, keys, timeout, limit=0, log_every=1000, wait=.1):
    
    limit = limit or float('inf')
    processed = 0
    while processed < limit:
        # Change ``blpop`` to ``brpop`` to process as LIFO.
        ret = r.blpop(keys, timeout)
        # If data is found before the timeout then we consider we are done.
        if ret is None:
            time.sleep(wait)
            continue

        source, data = ret
        try:
            item = json.loads(data)
        except Exception:
            logger.exception("Failed to load item:\n%r", pprint.pformat(data))
            continue

        try:
            name = item.get('name') or item.get('title')
            url = item.get('url') or item.get('link')
            logger.debug("[%s] Processing item: %s <%s>", source, name, url)
        except KeyError:
            logger.exception("[%s] Failed to process item:\n%r",
                             source, pprint.pformat(item))
            continue

        processed += 1
        if processed % log_every == 0:
            logger.info("Processed %s items", processed)
1080631
public ReadDocumentResponse readDocument(String documentId) {
        ReadDocumentRequest request = new ReadDocumentRequest();
        request.setDocumentId(documentId);
        return this.readDocument(request);
    }
1002413
@Override
    public synchronized SnapshotContentItem read()
        throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException {

        if (this.items == null) {
            this.items =
                new StreamingIterator<>(new JpaIteratorSource<SnapshotContentItemRepo, SnapshotContentItem>(repo) {
                    @Override
                    protected Page<SnapshotContentItem> getNextPage(Pageable pageable, SnapshotContentItemRepo repo) {
                        return repo.findBySnapshotName(snapshotName, pageable);
                    }
                });
            skipLinesAlreadyRead(this.items);
        }
        return this.items.hasNext() ? this.items.next() : null;
    }
1077815
@Override
    public Item process(Context context, CachingOptions cachingOptions, Item item) {
        for (String xpathQuery : metaDataNodesXPathQueries) {
            String metaDataValue = item.queryDescriptorValue(xpathQuery);
            if (StringUtils.isNotEmpty(metaDataValue)) {
                item.setProperty(xpathQuery, metaDataValue);
            }
        }

        return item;
    }
1255545
@Override
    public boolean checkRead()
    {
        //  Was the value prefetched already? If so, return.
        int h = queue.frontPos();
        if (h != r) {
            return true;
        }

        //  There's no prefetched value, so let us prefetch more values.
        //  Prefetching is to simply retrieve the
        //  pointer from c in atomic fashion. If there are no
        //  items to prefetch, set c to -1 (using compare-and-swap).
        if (c.compareAndSet(h, -1)) {
            // nothing to read, h == r must be the same
        }
        else {
            // something to have been written
            r = c.get();
        }

        //  If there are no elements prefetched, exit.
        //  During pipe's lifetime r should never be NULL, however,
        //  it can happen during pipe shutdown when items
        //  are being deallocated.
        if (h == r || r == -1) {
            return false;
        }

        //  There was at least one value prefetched.
        return true;
    }
353002
protected void skipLinesAlreadyRead(Iterator it) {
        long linesRead = getItemsRead();
        if (linesRead > 0) {
            for (long i = 0; i < linesRead; i++) {
                if (it.hasNext()) {
                    it.next();
                }
            }
        }
    }
603150
public void processDocument(BufferedReader document) throws IOException {
        Queue<String> prevWords = new ArrayDeque<String>(windowSize);
        Queue<String> nextWords = new ArrayDeque<String>(windowSize);

        Iterator<String> documentTokens = 
            IteratorFactory.tokenizeOrdered(document);

        String focusWord = null;

        // Prefetch the first windowSize words.  As soon as a word enters the
        // nextWords buffer increase its occurrence count.
        for (int i = 0; i < windowSize && documentTokens.hasNext(); ++i) 
            nextWords.offer(documentTokens.next());        
        
        while (!nextWords.isEmpty()) {
            
            focusWord = nextWords.remove();

            // shift over the window to the next word
            if (documentTokens.hasNext()) {
                String windowEdge = documentTokens.next(); 
                nextWords.offer(windowEdge);
            }    
                
            // Don't bother calculating the semantics for empty tokens
            // (i.e. words that were filtered out)
            if (!focusWord.equals(IteratorFactory.EMPTY_TOKEN)) {
                SemanticVector focusMeaning = getSemanticVector(focusWord);

                // Sum up the index vector for all the surrounding words.  If
                // permutations are enabled, permute the index vector based on
                // its relative position to the focus word.
                int permutations = -(prevWords.size());        
                for (String word : prevWords) {
                    // Skip the addition of any words that are excluded from the
                    // filter set.  Note that by doing the exclusion here, we
                    // ensure that the token stream maintains its existing
                    // ordering, which is necessary when permutations are taken
                    // into account.
                    if (word.equals(IteratorFactory.EMPTY_TOKEN)) {
                        ++permutations;
                        continue;
                    }
                    
                    TernaryVector iv = wordToIndexVector.get(word);
                    if (usePermutations) {
                        iv = permutationFunc.permute(iv, permutations);
                        ++permutations;
                    }
                            
                    updateSemantics(focusMeaning, word, iv);
                }
                
                // Repeat for the words in the forward window.
                permutations = 1;
                for (String word : nextWords) {
                    // Skip the addition of any words that are excluded from the
                    // filter set.  Note that by doing the exclusion here, we
                    // ensure that the token stream maintains its existing
                    // ordering, which is necessary when permutations are taken
                    // into account.
                    if (word.equals(IteratorFactory.EMPTY_TOKEN)) {
                        ++permutations;
                        continue;
                    }
                    
                    TernaryVector iv = wordToIndexVector.get(word);
                    if (usePermutations) {
                        iv = permutationFunc.permute(iv, permutations);
                        ++permutations;
                    }
                    
                    updateSemantics(focusMeaning, word, iv);
                }
            }

            // Last put this focus word in the prev words and shift off the
            // front of the previous word window if it now contains more words
            // than the maximum window size
            prevWords.offer(focusWord);

            // Increment the frequency count for the word now that it has been
            // seen and processed.
            Integer count = wordToOccurrences.get(focusWord);
            wordToOccurrences.put(focusWord, (count == null) ? 1 : count + 1);

            if (prevWords.size() > windowSize) {
                prevWords.remove();
            }
        }    

        document.close();
    }
77104
public static ReadStreamOld openRead(Reader reader)
  {
    if (reader instanceof ReadStreamOld.StreamReader)
      return ((ReadStreamOld.StreamReader) reader).getStream();
    
    ReaderWriterStream s = new ReaderWriterStream(reader, null);
    ReadStreamOld is = new ReadStreamOld(s);
    try {
      is.setEncoding("utf-8");
    } catch (Exception e) {
    }

    return is;
  }
772350
public void processDocument(BufferedReader document) throws IOException {
        int docIndex = documentCounter.getAndIncrement();

        Iterator<String> documentTokens = 
            IteratorFactory.tokenizeOrdered(document);

        // As we read in the document, generate a compressed version of it,
        // which we will use during the process space method to recompute all of
        // the word vectors' semantics
        ByteArrayOutputStream compressedDocument = 
            new ByteArrayOutputStream(4096);
        DataOutputStream dos = new DataOutputStream(compressedDocument);
        int tokens = 0; // count how many are in this document
        int unfilteredTokens = 0; // how many tokens remained after filtering

        IntegerVector docVector = createVector();
        docToVector.put(docIndex, docVector);

        while (documentTokens.hasNext()) {
            tokens++;
            String focusWord = documentTokens.next();

            // If we are filtering the semantic vectors, check whether this word
            // should have its semantics calculated.  In addition, if there is a
            // filter and it would have excluded the word, do not keep its
            // semantics around
            boolean calculateSemantics =
                semanticFilter.isEmpty() || semanticFilter.contains(focusWord)
                && !focusWord.equals(IteratorFactory.EMPTY_TOKEN);

	    // If the filter does not accept this word, skip the semantic
	    // processing, continue with the next word
            if (!calculateSemantics) {
                // Do not write out any removed tokens to save space
		continue;
	    }

            // Update the occurrences of this token
            unfilteredTokens++;
            add(docVector, getTermIndexVector(focusWord));

            // Update the compress version of the document with the token.
            //
            // NOTE: this call to termToIndex *must* come after the
            // getTermIndexVector() call, which is responsible for adding this
            // mapping if it doesn't already exist.
	    int focusIndex = termToIndex.get(focusWord);

            // write the term index into the compressed for the document for
            // later corpus reprocessing
            dos.writeInt(focusIndex);
        }

        document.close();
        
        dos.close();
        byte[] docAsBytes = compressedDocument.toByteArray();

        // Once the document is finished, write the compressed contents to the
        // corpus stream
        synchronized(compressedDocumentsWriter) {
            // Write how many terms were in this document after filtering
            compressedDocumentsWriter.writeInt(unfilteredTokens);
            compressedDocumentsWriter.write(docAsBytes, 0, docAsBytes.length);
        }
    }
1325379
@Override
	public void process(String xmlString, ITempelEntryProcessor entryProcessor) throws Exception {
		process(new ByteArrayInputStream(xmlString.getBytes()), entryProcessor);
	}
1140566
private boolean doRead(int amountToRead) throws IOException{
        if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()){
            Tr.debug(tc, "doRead, Current buffer, " + _buffer + ", reading from the TCP Channel, readLine : " + _isReadLine);
        }
        
        try {
            
            if(_tcpChannelCallback != null && !_isReadLine){
                //async read logic
                return immediateRead(amountToRead);
            } else {
                return syncRead(amountToRead);
            }
        } catch (IOException e) {
            if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()){
                Tr.debug(tc, "doRead, we encountered an exception during the read : " + e);
            }
            
            if(_error != null){
                return false;
            }
            _error = e;
            throw e;
        }
    }
266355
def process(self):
        '''
        '''
        self._item_session.request = request = Request(self._item_session.url_record.url)
        verdict = self._fetch_rule.check_ftp_request(self._item_session)[0]

        if not verdict:
            self._item_session.skip()
            return

        self._add_request_password(request)

        dir_name, filename = self._item_session.url_record.url_info.split_path()
        if self._processor.fetch_params.glob and frozenset(filename) & GLOB_CHARS:
            request = self._to_directory_request(request)
            is_file = False
            self._glob_pattern = urllib.parse.unquote(filename)
        else:
            is_file = yield from self._prepare_request_file_vs_dir(request)

            self._file_writer_session.process_request(request)

        wait_time = yield from self._fetch(request, is_file)

        if wait_time:
            _logger.debug('Sleeping {0}.', wait_time)
            yield from asyncio.sleep(wait_time)
192513
protected List<Item> doFilter(List<Item> items, ItemFilter filter, boolean runningBeforeProcessing) {
        List<Item> acceptedItems = new ArrayList<>();
        List<Item> rejectedItems = new ArrayList<>();

        for (Item item : items) {
            if (filter.accepts(item, acceptedItems, rejectedItems, runningBeforeProcessing)) {
                acceptedItems.add(item);
            } else {
                rejectedItems.add(item);
            }
        }

        if (logger.isDebugEnabled()) {
            logger.debug("Items filtered from " + items + " by " + filter + ": " + acceptedItems);
        }

        return acceptedItems;
    }