Return all the significant text below the given node as a list of words.
    >>> list(tree_words(parse_minidom('<h1>one</h1> two <div>three<em>four</em></div>')))
    ['one', 'two', 'three', 'four']
546232
def tree_words(node):
    
    for word in split_text(tree_text(node)):
        word = word.strip()
        if word:
            yield word
681094
private List<String> extractWords(@NotNull String text, Language language) {

        List<String> words = textUtils.extractWords(text, language);

        List<String> wordList = words.stream()
                                     .flatMap(word -> {
                                         return textUtils.getNormalForms(word.toLowerCase(), language)
                                                         .stream()
                                                         .filter(element -> textUtils.isNormalWord(element, language));
                                     })
                                     .collect(Collectors.toList());

        return wordList;
    }
1251077
def _get_subelements(self, node):
        
        items = node.find('rdf:Alt', self.NS)
        if items is not None:
            try:
                return items[0].text
            except IndexError:
                return ''

        for xmlcontainer, container, insertfn in XMP_CONTAINERS:
            items = node.find('rdf:{}'.format(xmlcontainer), self.NS)
            if items is None:
                continue
            result = container()
            for item in items:
                insertfn(result, item.text)
            return result
        return ''
1232537
function listNext(node, pred) {
  pred = pred || func.fail;

  const nodes = [];
  while (node) {
    if (pred(node)) { break; }
    nodes.push(node);
    node = node.nextSibling;
  }
  return nodes;
}
223695
def items(self):
		

		L = []
		def aux(node, s):
			s = s + node.char
			if node.output is not nil:
				L.append((s, node.output))

			for child in node.children.values():
				if child is not node:
					aux(child, s)

		aux(self.root, '')
		return iter(L)
625189
def get_words(self):
        

        words = []
        block = self.document().findBlockByLineNumber(0)
        while block.isValid():
            blockWords = foundations.strings.get_words(foundations.strings.to_string(block.text()))
            if blockWords:
                words.extend(blockWords)
            block = block.next()
        return words
220672
def get_phrases(self, ns=None, layer='syntax', cat_key='cat', cat_val='NP'):
        """"""
        if not ns:
            ns = self.ns

        for node_id in select_nodes_by_layer(self, '{0}:{1}'.format(ns, layer)):
            if self.node[node_id][self.ns+':'+cat_key] == cat_val:
                yield node_id
1069735
def _macronize_word(self, word):
        
        head_word = word[0]
        tag = word[1]
        if tag is None:
            logger.info('Tagger {} could not tag {}.'.format(self.tagger, head_word))
            return head_word, tag, head_word
        elif tag == 'U--------':
            return (head_word, tag.lower(), head_word)
        else:
            entries = self._retrieve_morpheus_entry(head_word)
            if entries is None:
                return head_word, tag.lower(), head_word
            matched_entry = [entry for entry in entries if entry[0] == tag.lower()]
            if len(matched_entry) == 0:
                logger.info('No matching Morpheus entry found for {}.'.format(head_word))
                return head_word, tag.lower(), entries[0][2]
            elif len(matched_entry) == 1:
                return head_word, tag.lower(), matched_entry[0][2].lower()
            else:
                logger.info('Multiple matching entries found for {}.'.format(head_word))
                return head_word, tag.lower(), matched_entry[1][2].lower()
1151540
def syllable_tokenize(text: str) -> List[str]:
    

    if not text or not isinstance(text, str):
        return []

    tokens = []
    if text:
        words = word_tokenize(text)
        trie = dict_trie(dict_source=thai_syllables())
        for word in words:
            tokens.extend(word_tokenize(text=word, custom_dict=trie))

    return tokens
418737
def wordify(text):
    
    stopset = set(nltk.corpus.stopwords.words('english'))
    tokens = nltk.WordPunctTokenizer().tokenize(text)
    return [w for w in tokens if w not in stopset]
622898
def extract_sentences(nodes, token_node_indices):
    
    sents = []
    tokens = []
    for i, node in enumerate(nodes):
        if i in token_node_indices:
            if node.features['tiger.pos'] != '$.':
                tokens.append(i)
            else:  # start a new sentence, if 'tiger.pos' is '$.'
                tokens.append(i)
                sents.append(tokens)
                tokens = []
    return sents
1365976
def get_specific_nodes(self, node, names):
        
        nodes = [(x.tagName, x) for x in node.childNodes
                 if x.nodeType == x.ELEMENT_NODE and
                 x.tagName in names]
        return dict(nodes)
1369683
def segs(self, word):
        
        return [m.group('all') for m in self.seg_regex.finditer(word)]
105011
def get_text(nodelist):
    

    s = ''
    for node in nodelist:
        if node.nodeType == Node.TEXT_NODE:
            s += node.nodeValue + ', '

    if len(s)>0: s = s[:-2]
    return s
260643
async def nodes(self, text, opts=None, user=None):
        '''
        
        '''
        return [n async for n in self.eval(text, opts=opts, user=user)]
1168610
def split(text: str) -> List[str]:
    
    return [word for word in SEPARATOR.split(text) if word.strip(' \t')]
1263041
def _get_lines_from_node_text(cls, node):
        
        # TODO more robust parsing
        from lxml import etree
        parts = ([node.text] + list(chain(*([etree.tostring(c, with_tail=False), c.tail] for c in node.getchildren()))) + [node.tail])
        parts = [gf.safe_unicode(p) for p in parts]
        parts = [p.strip() for p in parts if not p.startswith(u"<br ")]
        parts = [p for p in parts if len(p) > 0]
        uparts = []
        for part in parts:
            uparts.append(gf.safe_unicode(part))
        return uparts