Analyse content just after it's saved. This is useful for creating
new pages etc where referenced by particular formatting
188028
public function analyseSavedContent($wikiPage) {
		$newPages = $this->parseNewPagesFrom($wikiPage->Content);

		foreach ($newPages as $pageTitle) {
			$trimmedTitle = trim($pageTitle);

			// need to trim the title before doing the search
			$page = DataObject::get_one('WikiPage', '"SiteTree"."Title" = \'' . Convert::raw2sql($trimmedTitle) . '\'');

			if (!$page) {
				// it's a new page, so create that
				$page = new WikiPage();
				$page->Title = $trimmedTitle;
				$page->MenuTitle = $trimmedTitle;
				$page->ParentID = $wikiPage->ID;
				$page->write();

				// publish if we're on autopublish
				if (WikiPage::$auto_publish) {
					$page->doPublish();
				}
			}

			$replacement = '<a href="[sitetree_link id=' . $page->ID . ']">' . $pageTitle . '</a>';
			$wikiPage->Content = str_replace('[[' . $pageTitle . ']]', $replacement, $wikiPage->Content);
		}
	}
829470
function(content) {
			content = this.sanitize(content);
			var stream = stringStream(content);
			var ranges = [], matchedRanges;
			var self = this;

			var saveRule = function(r) {
				var selRange = self.extractSelector(content, r.start);
				var rule = range.create2(selRange.start, r.end);
				rule._selectorEnd = selRange.end;
				rule._contentStart = r.start;
				ranges.push(rule);
			};

			var ch;
			while (ch = stream.next()) {
				if (isQuote(ch)) {
					if (!stream.skipString(ch)) {
						break; // unterminated string
					}

					continue;
				}

				if (ch == '{') {
					matchedRanges = this.matchBracesRanges(content, stream.pos - 1);
					matchedRanges.forEach(saveRule);

					if (matchedRanges.length) {
						stream.pos = utils.last(matchedRanges).end;
						continue;
					} 
				}
			}
			
			return ranges.sort(function(a, b) {
				return a.start - b.start;
			});
		}
611631
def _log(self, content):
        
        self._buffer += content
        if self._auto_flush:
            self.flush()
99971
def decode_content(self):
        
        self.content = pickle.loads(zlib.decompress(self.content))
1095774
def find_new_posts_and_pages(db):
    """"""

    Q = Query()
    for root, dirs, files in os.walk(CONFIG['content_root']):
        for filename in sorted([f for f in files if
                               f.endswith(('md', 'markdown'))]):
            fullpath = os.path.join(root, filename)
            _p = fullpath.split(CONFIG['content_root'])[-1].lstrip('/')
            new_mtime = int(os.path.getmtime(fullpath))
            e, item = None, None

            for collection in ['posts', 'pages']:
                item = db[collection].get(Q.filename == _p)
                if item:
                    if new_mtime > item['mtime']:
                        db[collection].update({'mtime': new_mtime},
                                              doc_ids=[item.doc_id])
                        e = Entry(fullpath, doc_id=item.doc_id)
                    break

            if not item:
                e = Entry(fullpath)
            if e:
                yield e, e.id
882605
def read_analysis(self, file_handle):
        
        start = self.annotation['__header__']['analysis start']
        end = self.annotation['__header__']['analysis end']
        if start != 0 and end != 0:
            file_handle.seek(start, 0)
            self._analysis = file_handle.read(end - start)
        else:
            self._analysis = None
1344370
def rebuild_page_content(locale)
      unless page_part && page_part.template
        raise "page_part with valid template not available"
      end
      # page_part = self.page_parts.find_by_page_part_key page_part_key

      if page_part.present?
        l_template = Liquid::Template.parse(page_part.template)
        new_fragment_html = l_template.render('page_part' => page_part.block_contents[locale]["blocks"] )
        # p "#{page_part_key} content for #{self.slug} page parsed."
        # save in content model associated with page

        page_fragment_content = find_or_create_content
        # container.contents.find_or_create_by(page_part_key: page_part_key)
        content_html_col = "raw_" + locale + "="
        # above is the col used by globalize gem to store localized data
        # page_fragment_content[content_html_col] = new_fragment_html
        page_fragment_content.send content_html_col, new_fragment_html
        page_fragment_content.save!

        # set page_part_key value on join model
        page_content_join_model = get_join_model
        # page_fragment_content.page_contents.find_by_page_id self.id
        page_content_join_model.page_part_key = page_part_key
        page_content_join_model.save!
      else
        new_fragment_html = ""
      end

      new_fragment_html
    end
564642
def analyze(self):
        
        from_date = self._out.latest_date()
        if from_date:
            logger.info("Reading items since " + from_date)
        else:
            logger.info("Reading items since the beginning of times")

        cont = 0
        total_processed = 0
        total_written = 0

        for item_block in self._in.read_block(size=self._block_size, from_date=from_date):
            cont = cont + len(item_block)

            process_results = self.process(item_block)
            total_processed += process_results.processed

            if len(process_results.out_items) > 0:
                self._out.write(process_results.out_items)
                total_written += len(process_results.out_items)
            else:
                logger.info("No new items to be written this time.")

            logger.info(
                "Items read/to be written/total read/total processed/total written: "
                "{0}/{1}/{2}/{3}/{4}".format(str(len(item_block)),
                                             str(len(process_results.out_items)),
                                             str(cont),
                                             str(total_processed),
                                             str(total_written)))

        logger.info("SUMMARY: Items total read/total processed/total written: "
                    "{0}/{1}/{2}".format(str(cont),
                                         str(total_processed),
                                         str(total_written)))

        logger.info("This is the end.")

        return total_written
332467
function(status) {

      // Page is now opened
      pageInformation.isOpened = true;
      pageInformation.status = status;

      // Failing
      if (status !== 'success') {
        parent.replyTo(callId, wrapFailure('fail'));
        return cleanup();
      }

      // Wrong status code
      if (!pageInformation.response.status || pageInformation.response.status >= 400) {
        parent.replyTo(callId, wrapFailure('status'));
        return cleanup();
      }

      // Waiting for body to load
      page.evaluateAsync(function() {
        var interval = setInterval(function() {
          if (document.readyState === 'complete' || document.readyState === 'interactive') {
            clearInterval(interval);
            window.callPhantom({
              head: 'documentReady',
              body: true,
              passphrase: 'detoo'
            });
          }
        }, 30);
      });
    }
558243
def raw_content(self, output=None, str_output=None):
        """"""
        return self._search_page(output, str_output, self.response.data, lambda regex, content: regex.search(content.decode()))
491674
def extract(self, content, output):
    """"""

    # First apply default options.
    plugin_settings = DEFAULT_OPTIONS.copy()
    plugin_settings.update(self['lines'])
    self['lines'] = plugin_settings

    # Validate settings
    assert 'start' in self['lines'], 'Lines start regex missing'
    assert 'end' in self['lines'], 'Lines end regex missing'
    assert 'line' in self['lines'], 'Line regex missing'

    start = re.search(self['lines']['start'], content)
    end = re.search(self['lines']['end'], content)
    if not start or not end:
        logger.warning('no lines found - start %s, end %s', start, end)
        return
    content = content[start.end(): end.start()]
    lines = []
    current_row = {}
    if 'first_line' not in self['lines'] and 'last_line' not in self['lines']:
        self['lines']['first_line'] = self['lines']['line']
    for line in re.split(self['lines']['line_separator'], content):
        # if the line has empty lines in it , skip them
        if not line.strip('').strip('\n') or not line:
            continue
        if 'first_line' in self['lines']:
            match = re.search(self['lines']['first_line'], line)
            if match:
                if 'last_line' not in self['lines']:
                    if current_row:
                        lines.append(current_row)
                    current_row = {}
                if current_row:
                    lines.append(current_row)
                current_row = {
                    field: value.strip() if value else ''
                    for field, value in match.groupdict().items()
                }
                continue
        if 'last_line' in self['lines']:
            match = re.search(self['lines']['last_line'], line)
            if match:
                for field, value in match.groupdict().items():
                    current_row[field] = '%s%s%s' % (
                        current_row.get(field, ''),
                        current_row.get(field, '') and '\n' or '',
                        value.strip() if value else '',
                    )
                if current_row:
                    lines.append(current_row)
                current_row = {}
                continue
        match = re.search(self['lines']['line'], line)
        if match:
            for field, value in match.groupdict().items():
                current_row[field] = '%s%s%s' % (
                    current_row.get(field, ''),
                    current_row.get(field, '') and '\n' or '',
                    value.strip() if value else '',
                )
            continue
        logger.debug('ignoring *%s* because it doesn\'t match anything', line)
    if current_row:
        lines.append(current_row)

    types = self['lines'].get('types', [])
    for row in lines:
        for name in row.keys():
            if name in types:
                row[name] = self.coerce_type(row[name], types[name])

    if lines:
        output['lines'] = lines
415531
function processContent(target, opt, tagsRegExp, listOfFiles) {
	let targetContent = String(target.contents);
	const targetPath = target.path;
	const files = extractFilePaths(targetContent, targetPath, opt, tagsRegExp);

	// recursively process files
	files.forEach(function (fileData) {
		if (listOfFiles.indexOf(fileData.file.path) !== -1) {
			throw error("Circular definition found. File: " + fileData.file.path + " referenced in a child file.");
		}
		listOfFiles.push(fileData.file.path);
		const content = processContent(fileData.file, opt, tagsRegExp, listOfFiles);
		listOfFiles.pop();

		targetContent = inject(targetContent, String(content), opt, fileData.tags);
	});
	if (listOfFiles.length === 1 && !opt.quiet && files.length) {
		log(
			colors.cyan(files.length.toString()) +
			' partials injected into ' +
			colors.magenta(targetPath) +
			'.');
	}
	return new Buffer.from(targetContent);
}
265804
public function definition_after_data() {
        $mform = $this->_form;
        $cohortfile = $mform->getElementValue('cohortfile');
        $allowsubmitform = false;
        if ($cohortfile && ($file = $this->get_cohort_file($cohortfile))) {
            // File was uploaded. Parse it.
            $encoding = $mform->getElementValue('encoding')[0];
            $delimiter = $mform->getElementValue('delimiter')[0];
            $contextid = $mform->getElementValue('contextid')[0];
            if (!empty($contextid) && ($context = context::instance_by_id($contextid, IGNORE_MISSING))) {
                $this->processeddata = $this->process_upload_file($file, $encoding, $delimiter, $context);
                if ($this->processeddata && count($this->processeddata) > 1 && !$this->processeddata[0]['errors']) {
                    $allowsubmitform = true;
                }
            }
        }
        if (!$allowsubmitform) {
            // Hide submit button.
            $el = $mform->getElement('buttonar')->getElements()[0];
            $el->setValue('');
            $el->freeze();
        } else {
            $mform->setExpanded('cohortfileuploadform', false);
        }

    }
206698
def finalize_content(self):
        
        self.write_closed = True
        body = self.raw_body.decode(self.encoding)
        self._init_xml(body)
        self._form_output()
1183986
function extractContent(desc) {
  /*
  // search all possible matches, returning first find
  var founds = sites.findMatch(desc.uri);

  if (founds.length) {
    var $ = cheerio.load(desc.content);
  // use first found extractor
    founds.forEach(function(found) {
      if ($(found.selector)) {
        var content = $(found.selector).html();
        return { selector: found.selector, content: content };
      }
    });
  }
  // ok, let's try unfluff
  var uf = unfluff(desc.content);
  return { selector: 'body', content: desc.content, text: uf.text, title: uf.title };
  */
  var title = getTitle(desc.content, '<title') || getTitle(desc.content, '<TITLE');

  return { selector: 'body', content: desc.content, text: 'NOTEXTRACTED', title: title };
}
1051369
function splitPagesPromise (content) {
    var pages = content.split(/\f/)
    if (!pages) {
      var ex = {
        message: 'pdf-text-extract failed',
        error: 'no text returned from the pdftotext command',
        filePath: this.filePath,
        stack: new Error().stack
      }
      throw ex
    }
    // sometimes there can be an extract blank page on the end
    var lastPage = pages[pages.length - 1]
    if (!lastPage) {
      pages.pop()
    }
    resolve(pages)
  }
810610
private function analyzeExtended()
    {
        $existObjectSeparator = pack('c*', 0x0B, 0x00, 0x08, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00);

        $pack = $this->rec->getResourcePack();

        list ($mapSizeX, $mapSizeY) = $this->analysis->mapSize;

        $version = $this->version;
        $players = $this->analysis->players;
        $playersByIndex = [];
        foreach ($players as $p) {
            $playersByIndex[$p->index] = $p;
        }

        // Add GAIA
        $numPlayers = $this->analysis->numPlayers + 1;
        $gaia = new Player($this->rec);
        $gaia->name = 'GAIA';

        // Player -1 is GAIA.
        $playersWithGaia = $players;
        array_unshift($playersWithGaia, $gaia);
        foreach ($playersWithGaia as $player) {
            // Co-op partners do not have an info block.
            if ($player->isCoopPartner()) {
                $coopMain = $player->getCoopMain();
                $player->civId = $coopMain->civId;
                $player->colorId = $coopMain->colorId;
                $player->team = $coopMain->team;
                continue;
            }

            if ($version->isTrial) {
                $this->position += 4;
            }
            $this->position += $numPlayers + 43;

            // skip playername
            $playerNameLen = $this->readHeader('v', 2);
            $this->position += $playerNameLen;

            $this->position += 1; // always 22?
            $numResources = $this->readHeader('l', 4);
            $this->position += 1; // always 33?
            $resourcesEnd = $this->position + 4 * $numResources;

            // Interesting resources
            $food = $this->readHeader('f', 4);
            $wood = $this->readHeader('f', 4);
            $stone = $this->readHeader('f', 4);
            $gold = $this->readHeader('f', 4);
            // headroom = (house capacity - population)
            $headroom = $this->readHeader('f', 4);
            $this->position += 4;
            // Post-Imperial Age = Imperial Age here
            $startingAge = $this->readHeader('f', 4);
            $this->position += 4 * 4;
            $population = $this->readHeader('f', 4);
            $this->position += 25 * 4;
            $civilianPop = $this->readHeader('f', 4);
            $this->position += 2 * 4;
            $militaryPop = $this->readHeader('f', 4);

            $this->position = $resourcesEnd;
            $this->position += 1; // 1 byte

            $initCameraX = $this->readHeader('f', 4);
            $initCameraY = $this->readHeader('f', 4);
            if ($version->isMgx) {
                $this->position += 9;
            } else {
                $this->position += 5;
            }
            $civilization = ord($this->header[$this->position++]);
            if (!$civilization) {
                $civilization = 1;
            }
            $this->position += 3;
            $playerColor = ord($this->header[$this->position++]);

            if ($player->civId === -1) {
                $player->civId = $civilization;
            }
            $player->colorId = $playerColor;
            $player->initialState->position = [round($initCameraX), round($initCameraY)];
            $player->initialState->food = round($food);
            $player->initialState->wood = round($wood);
            $player->initialState->stone = round($stone);
            $player->initialState->gold = round($gold);
            $player->initialState->startingAge = round($startingAge);
            $player->initialState->houseCapacity = round($headroom) + round($population);
            $player->initialState->population = round($population);
            $player->initialState->civilianPop = round($civilianPop);
            $player->initialState->militaryPop = round($militaryPop);
            $player->initialState->extraPop = $player->initialState->population -
                ($player->initialState->civilianPop + $player->initialState->militaryPop);

            if ($version->isTrial) {
                $this->position += 4;
            }
            $this->position += $numPlayers + 70;
            $this->position += $version->isMgx ? 792 : 756;
            $this->position += $version->isMgx ? 41249 : 34277;
            $this->position += $mapSizeX * $mapSizeY;

            // getting exist_object_pos
            $existObjectPos = strpos($this->header, $existObjectSeparator, $this->position);
            if ($existObjectPos === false) {
                throw new \Exception('Could not find existObjectSeparator');
            } else {
                $this->position = $existObjectPos + strlen($existObjectSeparator);
            }

            $objects = $this->read(PlayerObjectsListAnalyzer::class, [
                'players' => array_merge([
                    0 => $gaia,
                ], $playersByIndex),
            ]);

            foreach ($objects->gaiaObjects as &$object) {
                $this->gaiaObjects[] = $object;
            }
            foreach ($objects->playerObjects as &$object) {
                $this->playerObjects[] = $object;
            }
        }

        return (object) [
            'gaia' => $players[0],
            'players' => array_slice($players, 1),
            'gaiaObjects' => $this->gaiaObjects,
            'playerObjects' => $this->playerObjects,
        ];
    }