Process all the lines of a document string

@param string $document the string (content) of the document
327204
protected function parseLines($document)
    {
        // Including files
        $document = str_replace("\r\n", "\n", $document);
        $document = "\n$document\n";
        $document = $this->includeFiles($document);
        
        // Removing UTF-8 BOM
        $bom = "\xef\xbb\xbf";
        $document = str_replace($bom, '', $document);

        $lines = explode("\n", $document);
        $this->state = self::STATE_BEGIN;

        foreach ($lines as $n => $line) {
            $this->currentLine = $n;
            while (!$this->parseLine($line));
        }

        // Document is flushed twice to trigger the directives
        $this->flush();
        $this->flush();
    }
720248
public function init(Document $document)
	{
		$this->hashId = $this->getHashId($document, $this->serviceLocator);
		$this->title = $document->getTitle();
		$this->price = $this->getPrice($document);
		$this->priceFormatted = $this->getPriceFormatted($document);
		$this->meta = $this->getMeta($document);
		$this->date = $this->getDateFormatted($document);
		$this->image = $this->getImage($document, $this->serviceLocator); // rimuovere
		$this->images = $this->getImages($document, $this->serviceLocator); 
		$this->content = $document->getContent();
		$this->nl2brContent = nl2br($document->getContent());
		$this->dateInsert = $document->getDateInsert()->format('Y/m/d H:i:s');
		$this->visited = $document->getVisited();

		$this->state = $document->getState();
		$this->stateFormatted = $this->getStateFormatted($document);
		
		$state = $document->getState();
		$this->isActive = Document::DOCUMENT_STATE_ACTIVE == $state;
		$this->isDeactive = Document::DOCUMENT_STATE_DEACTIVE == $state;
		$this->isDeleted = Document::DOCUMENT_STATE_DELETED == $state;
		$this->isExpired = $this->getIsExpired($document);
		$this->cssClass = $this->isActive && !$this->isExpired ? '' : 'warning';
		$this->purpose = $this->getPurpose($document); 
		$this->mobile = $this->getMobile($document); 
		$this->hidemobile = $this->getHideMobile($document); 

		$this->location = $this->getLocation($document, $this->serviceLocator);
		$this->locationFormatted = $this->getLocationFormatted($document, $this->serviceLocator); 
		
		$this->categories = $this->getCategories($document); 
		$this->category = $this->getCategory($document); 
		$this->slug = $this->getSlug($document); 
		$this->address = $this->getAddress($document, $this->serviceLocator);
		$this->editaddress = $this->getEditAddress($document);
		$this->insert = $this->getInsertDate($document);
		
		// Some user data
		$user = $document->getUser();
		$this->userId = $this->getUserHashId($document, $this->serviceLocator);
		$this->fullname = $user->getName() . ' ' . $user->getSurname();
		
		// Some location data
		$geoname = $document->getGeoname();
		$this->longitude = null !== $geoname ? $geoname->getLongitude() : null;
		$this->latitude = null !== $geoname ? $geoname->getLatitude() : null;
		
		$this->breadcrumbs = $this->getBreadcrumbs($document);
		
		return $this;
	}
78538
private function indexDocument($document)
    {
        if (!$document instanceof StructureBehavior) {
            return;
        }

        if ($document instanceof SecurityBehavior && !empty($document->getPermissions())) {
            return;
        }

        $this->searchManager->index($document);
    }
89985
public function delete($document)
    {
        $qb         = $this->dm->createQueryBuilder($this->class->name);
        $identifier = $this->uow->getDocumentIdentifier($document);

        $qb->delete();
        $qb->setIdentifier($identifier);

        if ($this->class->isLockable) {
            $qb->attr($this->class->lockMetadata->name)->exists(false);
        }

        try {
            $qb->getQuery()->execute();
        } catch (\Aws\DynamoDb\Exception\DynamoDbException $e) {
            if (
                ($this->class->isVersioned || $this->class->isLockable) &&
                $e->getAwsErrorCode() === 'ConditionalCheckFailedException'
            ) {
                throw LockException::lockFailed($document);
            }

            throw $e;
        }
    }
515011
protected function processInline($text)
    {
        while ($text) {
            if (preg_match(self::MATCH_LINK, $text, $m)) {
                return $this->processLink($m[1], $m[2], $m[3], $m[4]);
            }

            $importantTextAhead = $this->lookAhead($text, self::STRONG_START_END);
            $emphasizedTextAhead = $this->lookAhead($text, self::EMPHASIZED_START_END);

            if ($importantTextAhead === false) {
                $importantTextAhead = $this->lookAhead($text, self::STRONG_START_END_TYPE_2);
            }

            if ($emphasizedTextAhead === false) {
                $emphasizedTextAhead = $this->lookAhead($text, self::EMPHASIZED_START_END_TYPE_2);
            }

            if ($importantTextAhead !== false && $emphasizedTextAhead !== false) {
                if ($importantTextAhead <= $emphasizedTextAhead) {
                    $unformattedText = mb_substr($text, 0, $importantTextAhead);
                    $importantText = mb_substr($text, $importantTextAhead);
                    $this->text($unformattedText);

                    return $this->processStrong($importantText);
                } else {
                    $unformattedText = mb_substr($text, 0, $emphasizedTextAhead);
                    $emphasizedText = mb_substr($text, $emphasizedTextAhead);
                    $this->text($unformattedText);

                    return $this->processEmphasized($emphasizedText);
                }
            } elseif ($importantTextAhead !== false) {
                $unformattedText = mb_substr($text, 0, $importantTextAhead);
                $importantText = mb_substr($text, $importantTextAhead);
                $this->text($unformattedText);

                return $this->processStrong($importantText);
            } elseif ($emphasizedTextAhead !== false) {
                $unformattedText = mb_substr($text, 0, $emphasizedTextAhead);
                $emphasizedText = mb_substr($text, $emphasizedTextAhead);
                $this->text($unformattedText);

                return $this->processEmphasized($emphasizedText);
            }

            $text = $this->processRestOfLine($text);
        };

        return ''; //All text is consumed
    }
623502
public function parseHtml( $string )
	{
		$this->document = new DOMDocument();
		ob_start();
		$this->document->loadHTML( $string );
		$content	= ob_get_clean();
		if( $content )
			$this->errors	= $content;
	}
772350
public void processDocument(BufferedReader document) throws IOException {
        int docIndex = documentCounter.getAndIncrement();

        Iterator<String> documentTokens = 
            IteratorFactory.tokenizeOrdered(document);

        // As we read in the document, generate a compressed version of it,
        // which we will use during the process space method to recompute all of
        // the word vectors' semantics
        ByteArrayOutputStream compressedDocument = 
            new ByteArrayOutputStream(4096);
        DataOutputStream dos = new DataOutputStream(compressedDocument);
        int tokens = 0; // count how many are in this document
        int unfilteredTokens = 0; // how many tokens remained after filtering

        IntegerVector docVector = createVector();
        docToVector.put(docIndex, docVector);

        while (documentTokens.hasNext()) {
            tokens++;
            String focusWord = documentTokens.next();

            // If we are filtering the semantic vectors, check whether this word
            // should have its semantics calculated.  In addition, if there is a
            // filter and it would have excluded the word, do not keep its
            // semantics around
            boolean calculateSemantics =
                semanticFilter.isEmpty() || semanticFilter.contains(focusWord)
                && !focusWord.equals(IteratorFactory.EMPTY_TOKEN);

	    // If the filter does not accept this word, skip the semantic
	    // processing, continue with the next word
            if (!calculateSemantics) {
                // Do not write out any removed tokens to save space
		continue;
	    }

            // Update the occurrences of this token
            unfilteredTokens++;
            add(docVector, getTermIndexVector(focusWord));

            // Update the compress version of the document with the token.
            //
            // NOTE: this call to termToIndex *must* come after the
            // getTermIndexVector() call, which is responsible for adding this
            // mapping if it doesn't already exist.
	    int focusIndex = termToIndex.get(focusWord);

            // write the term index into the compressed for the document for
            // later corpus reprocessing
            dos.writeInt(focusIndex);
        }

        document.close();
        
        dos.close();
        byte[] docAsBytes = compressedDocument.toByteArray();

        // Once the document is finished, write the compressed contents to the
        // corpus stream
        synchronized(compressedDocumentsWriter) {
            // Write how many terms were in this document after filtering
            compressedDocumentsWriter.writeInt(unfilteredTokens);
            compressedDocumentsWriter.write(docAsBytes, 0, docAsBytes.length);
        }
    }
1247901
protected function writeMultipartDocument( array $document )
    {
        $body = json_encode( $document );

        $boundary = $this->getBoundary();
        fwrite( $this->stream, "Content-ID: " . $document['_id'] . "\r\n" );
        fwrite( $this->stream, "Content-Type: multipart/mixed; boundary=\"" . $boundary . "\"\r\n\r\n" );
 
        $attachments = $document['_attachments'];
        unset( $document['_attachments'] );

        // Write document first
        fwrite( $this->stream, "--" . $boundary . "\r\n" );
        $body = json_encode( $document );

        fwrite( $this->stream, "Content-Length: " . strlen( $body ) . "\r\n" );
        fwrite( $this->stream, "Content-Type: application/json\r\n" );
        fwrite( $this->stream, "\r\n" );
        fwrite( $this->stream, "$body\r\n" );

        // Write all attachments
        foreach ( $attachments as $name => $attachment )
        {
            fwrite( $this->stream, "--" . $boundary . "\r\n" );
            $body = $attachment['data'];

            fwrite( $this->stream, "Content-ID: " . $name . "\r\n" );
            fwrite( $this->stream, "Content-Length: " . strlen( $body ) . "\r\n" );
            fwrite( $this->stream, "Content-Type: " . $attachment['content_type'] . "\r\n" );
            fwrite( $this->stream, "\r\n" );
            fwrite( $this->stream, "$body\r\n" );
        }

        // End of multipart/mixed data
        fwrite( $this->stream, "--" . $boundary . "--\r\n" );
    }
893076
protected function processBlocks(string &$content): void
    {
        $pattern = $this->getTagPattern('block\s+([_a-zA-Z][_a-zA-Z0-9]*)' . preg_quote($this->endTag, '/') . '\s*(.+?)\s*' . preg_quote($this->startTag . '/block', '/'));
        $blockCodes = [];
        $content = preg_replace_callback($pattern, function ($match) use (&$blockCodes) {
            $blockCodes[$match[1]] = $match[2];
            return '';
        }, $content);
        $displayPattern = $this->getTagPattern('display_block\s+([_a-zA-Z][_a-zA-Z0-9]*)');
        $content = preg_replace_callback($displayPattern, function ($match) use ($blockCodes) {
            if (isset($blockCodes[$match[1]])) {
                return $blockCodes[$match[1]];
            }
            return '';
        }, $content);
    }
1159253
private function processLine(string $line): void
    {
        $iterator = new \ArrayIterator(explode(Tokens::SPACE, $line));

        if ($this->isControlEntry($iterator)) {
            $this->processControlEntry($iterator);

            return;
        }

        $resourceRecord = new ResourceRecord();

        $this->processResourceName($iterator, $resourceRecord);
        $this->processTtl($iterator, $resourceRecord);
        $this->processClass($iterator, $resourceRecord);
        $resourceRecord->setRdata($this->extractRdata($iterator));

        $this->zone->addResourceRecord($resourceRecord);
    }
19194
public function persist($document)
    {
        $documentArray = $this->converter->convertToArray($document);
        $type = $this->getMetadataCollector()->getDocumentType(get_class($document));

        $this->bulk('index', $type, $documentArray);
    }
194006
private static function clean($doc)
    {
        if ($doc === null) {
            return null;
        }

        $result = null;
        $lines = preg_split('/\r/', $doc);
        foreach ($lines as $line) {

            $line = preg_replace('/^\s*\* /', '', trim($line, "\n/* \t\x0B\0"));
            if ($line === '') {
                continue;
            }

            if ($result != null) {
                $result .= ' ';
            }
            $result .= $line;
        }

        return $result;
    }
1031498
public function index($document, $overwrite = null, $commitWithin = null, $buffer = 1500)
	{
		$this->initBufferAdd($overwrite, $commitWithin, $buffer);
		$this->addDocument($document);
	}
1227575
public function printLine($line)
    {
        if (!$this->handle) {
            throw new \RuntimeException('can not write to file: invalid file handle');
        }
        fwrite($this->handle, $line . "\n");
        ++$this->lineNumber;
    }
924426
private function doRemoveAllTranslations($document, ClassMetadata $metadata)
    {
        if (!$this->isDocumentTranslatable($metadata)) {
            return;
        }

        $node = $this->session->getNode($this->getDocumentId($document));
        $strategy = $this->dm->getTranslationStrategy($metadata->translator);
        $strategy->removeAllTranslations($document, $node, $metadata);
    }
998718
protected function line($text)
    {
        if($this->console) {
            $this->console->line($text);
        }

        if($this->logger) {
            $this->logger->line($text);
        }
    }
1114613
public function remove($documents)
    {
        if (!is_array($documents)) {
            $documents = array($documents);
        }

        $remove =& $this->getArchive()->getByRef('remove', array());
        foreach ($documents as $document) {
            $remove[] = $document;
        }
    }