/*
Process the given text with retext and return the promise of an object with
the keywords and keyphrases extracted.
551663
function processKeywords (text) {
  // for some reason retext is not filtering "it's" out
  const cleanText = text.replace(/'t/g, '')
                        .replace(/'s/g, '')
                        .replace(/'ll/g, '')
                        .replace(/'re/g, '')
                        .replace(/'ve/g, '');

  return new Promise(function (resolve, reject) {
    processor.process(cleanText, function (err, file) {
      if (err) {
        reject(err);
      }
      const space = file.namespace('retext');
      const words = space.keywords.map((w) => ({
        value: nlcstToString(w.matches[0].node),
        score: w.score
      }));
      const phrases = space.keyphrases.map((p) => ({
        value: p.matches[0].nodes.map(nlcstToString).join(''),
        score: p.score
      }));

      resolve({words, phrases});
    });
  });
}
1373151
def run(self, text):
        
        for pp in self.pre_processors:
            text = pp.run(text)
        return text
637156
def _request_process_text(response):
        
        status = 'Failure'
        # Handle document download
        data = response.content
        if data:
            status = 'Success'

        return data, status
1033042
function parseLink (text) {
  if (!text) return ''
  var results = []
  var matches = null
  var link1 = /{@link (\S+?)}/g // {@link someSymbol}
  var link2 = /\[(.+?)\]{@link (\S+?)}/g // [caption here]{@link someSymbol}
  var link3 = /{@link ([^\s}]+?)\|(.+?)}/g // {@link someSymbol|caption here}
  var link4 = /{@link ([^\s}\|]+?) (.+?)}/g // {@link someSymbol Caption Here}

  while ((matches = link4.exec(text)) !== null) {
    results.push({
      original: matches[0],
      caption: matches[2],
      url: matches[1]
    })
    text = text.replace(matches[0], Buffer(matches[0].length).fill(' ').toString())
  }

  while ((matches = link3.exec(text)) !== null) {
    results.push({
      original: matches[0],
      caption: matches[2],
      url: matches[1]
    })
    text = text.replace(matches[0], Buffer(matches[0].length).fill(' ').toString())
  }

  while ((matches = link2.exec(text)) !== null) {
    results.push({
      original: matches[0],
      caption: matches[1],
      url: matches[2]
    })
    text = text.replace(matches[0], Buffer(matches[0].length).fill(' ').toString())
  }

  while ((matches = link1.exec(text)) !== null) {
    results.push({
      original: matches[0],
      caption: matches[1],
      url: matches[1]
    })
    text = text.replace(matches[0], Buffer(matches[0].length).fill(' ').toString())
  }
  return results
}
283084
def parse(text: str) -> Docstring:
    
    ret = Docstring()
    if not text:
        return ret

    # Clean according to PEP-0257
    text = inspect.cleandoc(text)

    # Find first title and split on its position
    match = _titles_re.search(text)
    if match:
        desc_chunk = text[: match.start()]
        meta_chunk = text[match.start() :]
    else:
        desc_chunk = text
        meta_chunk = ""

    # Break description into short and long parts
    parts = desc_chunk.split("\n", 1)
    ret.short_description = parts[0] or None
    if len(parts) > 1:
        long_desc_chunk = parts[1] or ""
        ret.blank_after_short_description = long_desc_chunk.startswith("\n")
        ret.blank_after_long_description = long_desc_chunk.endswith("\n\n")
        ret.long_description = long_desc_chunk.strip() or None

    # Split by sections determined by titles
    matches = list(_titles_re.finditer(meta_chunk))
    if not matches:
        return ret
    splits = []
    for j in range(len(matches) - 1):
        splits.append((matches[j].end(), matches[j + 1].start()))
    splits.append((matches[-1].end(), len(meta_chunk)))

    chunks = {}
    for j, (start, end) in enumerate(splits):
        title = matches[j].group(1)
        if title not in _valid:
            continue
        chunks[title] = meta_chunk[start:end].strip("\n")
    if not chunks:
        return ret

    # Add elements from each chunk
    for title, chunk in chunks.items():
        # Determine indent
        indent_match = re.search(r"^\s+", chunk)
        if not indent_match:
            raise ParseError(f'Can\'t infer indent from "{chunk}"')
        indent = indent_match.group()

        # Check for returns/yeilds (only one element)
        if _sections[title] in ("returns", "yields"):
            part = inspect.cleandoc(chunk)
            ret.meta.append(_build_meta(part, title))
            continue

        # Split based on lines which have exactly that indent
        _re = "^" + indent + r"(?=\S)"
        c_matches = list(re.finditer(_re, chunk, flags=re.M))
        if not c_matches:
            raise ParseError(f'No specification for "{title}": "{chunk}"')
        c_splits = []
        for j in range(len(c_matches) - 1):
            c_splits.append((c_matches[j].end(), c_matches[j + 1].start()))
        c_splits.append((c_matches[-1].end(), len(chunk)))
        for j, (start, end) in enumerate(c_splits):
            part = chunk[start:end].strip("\n")
            ret.meta.append(_build_meta(part, title))

    return ret
1211279
public AnalyzedText analyze(String text, boolean lazy){
		AnalyzedText result = new AnalyzedText(this, text);
		if (!lazy){
			analyzeLength(result);
			analyzeWords(result);
			analyzeKeywords(result);
		}
		return result;
	}
1121305
def parse(text, **options):
    
    _options = copy.copy(DEFAULT_OPTIONS)
    _options.update(options)

    return _normalize(_parse(text, **_options), **_options)
741783
def from_text(cls, text, mapping='mapping'):
        
        graphemes = Counter(grapheme_pattern.findall(text))
        specs = [
            OrderedDict([
                (cls.GRAPHEME_COL, grapheme),
                ('frequency', frequency),
                (mapping, grapheme)])
            for grapheme, frequency in graphemes.most_common()]
        return cls(*specs)
195634
function text(text) {
  if (typeof text === 'undefined') {
    if (this[0]) {
      return this[0].textContent.trim();
    }
    return null;
  }

  for (let i = 0; i < this.length; i += 1) {
    this[i].textContent = text;
  }
  return this;
}
682495
def try_process_text(self, text):
        
        if not text:
            return text
        try:
            return self.process_text(text)
        except (socket.gaierror, socket.timeout, xml.parsers.expat.ExpatError):
            return text
816701
function (text) {
      var inApiSection = false;

      return fs.createReadStream(opts.src)
        .pipe(es.split("\n"))
        .pipe(es.through(function (line) {
          // Hit the start marker.
          if (line === opts.start) {
            // Emit our line (it **is** included).
            this.emit("data", line);

            // Emit our the processed API data.
            this.emit("data", text);

            // Mark that we are **within** API section.
            inApiSection = true;
          }

          // End marker.
          if (line === opts.end) {
            // Mark that we have **exited** API section.
            inApiSection = false;
          }

          // Re-emit lines only if we are not within API section.
          if (!inApiSection) {
            this.emit("data", line);
          }
        }))
        .pipe(es.join("\n"))
        .pipe(es.wait());
    }
1020651
def transform_source(text):
    '''
    '''
    toks = tokenize.generate_tokens(StringIO(text).readline)
    result = []
    replacing_keyword = False
    for toktype, tokvalue, _, _, _ in toks:
        if toktype == tokenize.NAME and tokvalue == 'switch':
            result.extend([
                (tokenize.NAME, 'for'),
                (tokenize.NAME, '__case'),
                (tokenize.NAME, 'in'),
                (tokenize.NAME, '_Switch'),
                (tokenize.OP, '(')
            ])
            replacing_keyword = True
        elif toktype == tokenize.NAME and (tokvalue == 'case' or tokvalue == 'default'):
            result.extend([
                (tokenize.NAME, 'if'),
                (tokenize.NAME, '__case'),
                (tokenize.OP, '(')
            ])
            replacing_keyword = True
        elif replacing_keyword and tokvalue == ':':
            result.extend([
                (tokenize.OP, ')'),
                (tokenize.OP, ':')
            ])
            replacing_keyword = False
        else:
            result.append((toktype, tokvalue))
    return tokenize.untokenize(result)
1024432
@SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops")
    private static Iterable<Directive> transform(final Iterable<String> texts) {
        final Collection<Directive> list = new LinkedList<>();
        for (final String text : texts) {
            try {
                for (final Directive dir : new Directives(text)) {
                    list.add(dir);
                }
            } catch (final SyntaxException ex) {
                throw new IllegalStateException(ex);
            }
        }
        return list;
    }
669502
def generateSequence(self, text, preprocess=False):
    
    # TODO: enable text preprocessing; abstract out the logic in split() into a common method.
    tokens = TextPreprocess().tokenize(text)
    cat = [-1]
    self.sequenceCount += 1
    uniqueID = "q"
    data = self._formatSequence(tokens, cat, self.sequenceCount-1, uniqueID)

    return data
612270
function returnRankedIntents(input, intents) {
    return new Promise((resolve, reject) => {
        let lowerCase = input.toLowerCase();
        let totalScore = 0;

        //loop through all loaded intents
        intents.forEach((intent) => {
            intent.score = 0;
            intent.utterences.forEach((utterence) => {

                let words = utterence.toLowerCase().split(" ");
                let utterenceMatch = 0;
                let ppw = 100 / words.length;


                //how many words in input match intent samples
                words.forEach((word) => {
                    if (lowerCase.includes(word)) {
                        utterenceMatch = utterenceMatch + ppw;
                    }
                });

                if (intent.score < utterenceMatch) {
                    intent.score = utterenceMatch
                }

            });

            //each intent keyword in the input multiplies the intent score by 2
            intent.keywords.forEach((keyword) => {
                if (input.includes(keyword.toLowerCase())) {
                    intent.score = intent.score * 2
                }
            });

            //add intent score to totalscore
            totalScore = totalScore + intent.score;
        });


        //work out confident by deviding each intents score by the total
        intents.forEach((intent) => {
            intent.confidence = intent.score / totalScore
        });

        //Sort intents
        let sorted = intents.sort((a, b) => {
            return b.confidence - a.confidence
        });

        //sent back the classified intents
        resolve(sorted);
    }
    )
}
403826
public CAS process(String text) {
        CAS cas = retrieve();
        if (cas == null)
            return null;

        cas.setDocumentText(text);
        try {
            analysisEngine.process(cas);
        } catch (AnalysisEngineProcessException e) {
            log.warn("Unable to process text " + text, e);
        }

        return cas;


    }
900248
public List<List<Token>> process(final String text) {
        final List<List<Token>> paragraph = new ArrayList<>();
        currentSentence = new ArrayList<>();
        final Tokens tokens = splitText(text);

        while (tokens.hasNext()) {
            final Token t = tokens.next();
            final String trimmedWord = t.text.trim();

            // skip spaces
            if (trimmedWord.isEmpty()) continue;

            if (((mode == WITH_PUNCTUATION) || (mode == WITHOUT_PUNCTUATION && isLetterOrDigit(initChar(t.text))))) {
                boolean canBreakSentence = true;
                if (t.text.contains("'")) {
                    wordContainsApostrophe(t);
                }
                else if (".".equals(trimmedWord)) {
                    canBreakSentence = wordIsFullStop(t);
                }
                else if (":".equals(trimmedWord)) {
                    wordIsColon(tokens, t);
                }
                else currentSentence.add(t);

                // handling the end of a sentence
                if (canBreakSentence && equalss(trimmedWord, ".", ";", "?", "!")) {
                    paragraph.add(currentSentence);
                    currentSentence = new ArrayList<>();
                }
            }
        }

        if (!currentSentence.isEmpty()) paragraph.add(currentSentence);
        return paragraph;
    }