splits the word into a character-list of tamil/english
    characters present in the stream
975438
def get_letters_iterable( word ):
    
    WLEN,idx = len(word),0

    while (idx < WLEN):
        c = word[idx]
        #print(idx,hex(ord(c)),len(ta_letters))
        if c in uyir_letter_set or c == ayudha_letter:
            idx = idx + 1
            yield c
        elif c in grantha_agaram_set:
            if idx + 1 < WLEN and word[idx+1] in all_symbol_set:
                c2 = word[idx+1]
                idx = idx + 2
                yield (c + c2)
            else:
                idx = idx + 1
                yield c
        else:
            idx = idx + 1
            yield c
    return
71533
function splitWord (word) {
  Util.assert(word);

  if (word.length < MAX_SAMPLE_SIZE) {
    return [word];
  }

  let chunks = [];
  for (let ii = 0; ii < word.length - MAX_SAMPLE_SIZE + 1; ++ii) {
    chunks.push(word.substring(ii, ii + MAX_SAMPLE_SIZE));
  }
  return chunks;
}
58792
def bmrblex(text):
    
    stream = transform_text(text)

    wordchars = (u"abcdfeghijklmnopqrstuvwxyz"
                 u"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"
                 u"ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ"
                 u"ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ"
                 u"!@$%^&*()_+:;?/>.<,~`|\{[}]-=")

    whitespace = u" \t\v\r\n"
    comment = u"#"
    state = u" "
    token = u""
    single_line_comment = u""

    while len(stream) > 0:
        nextnextchar = stream.popleft()

        while True:
            nextchar = nextnextchar

            if len(stream) > 0:
                nextnextchar = stream.popleft()
            else:
                nextnextchar = u""

            # Process multiline string, comment, or single line comment
            if len(nextchar) > 1:
                state = u" "
                token = nextchar
                break  # emit current token

            elif nextchar in whitespace and nextnextchar in comment and state not in (u"'", u'"'):
                single_line_comment = u""
                state = u"#"

            if state is None:
                token = u""  # past end of file
                break

            elif state == u" ":
                if not nextchar:
                    state = None
                    break

                elif nextchar in whitespace:
                    if token:
                        state = u" "
                        break  # emit current token
                    else:
                        continue

                elif nextchar in wordchars:
                    token = nextchar
                    state = u"a"

                elif nextchar == u"'" or nextchar == u'"':
                    token = nextchar
                    state = nextchar

                else:
                    token = nextchar
                    if token:
                        state = u" "
                        break  # emit current token
                    else:
                        continue

            # Process single-quoted or double-quoted token
            elif state == u"'" or state == u'"':
                token += nextchar
                if nextchar == state:
                    if nextnextchar in whitespace:
                        state = u" "
                        token = token[1:-1]  # remove single or double quotes from the ends
                        break

            # Process single line comment
            elif state == u"#":
                single_line_comment += nextchar
                if nextchar == u"\n":
                    state = u" "
                    break

            # Process regular (unquoted) token
            elif state == u"a":
                if not nextchar:
                    state = None
                    break
                elif nextchar in whitespace:
                    state = u" "
                    if token:
                        break  # emit current token
                    else:
                        continue
                else:
                    token += nextchar

        if nextnextchar:
            stream.appendleft(nextnextchar)

        yield token
        token = u""
1273920
def run(self, line):
        
        words = []
        for word in self.clean_unicode(line.lower()).split():
            if word.startswith('http'):
                continue
            cleaned = self.clean_punctuation(word)
            if len(cleaned) > 1 and cleaned not in self.stopwords:
                words.append(cleaned)
        return words
571534
def coalesce_streams(outputs):
    
    if not outputs:
        return outputs

    new_outputs = []
    streams = {}
    for output in outputs:
        if (output.output_type == 'stream'):
            if output.name in streams:
                streams[output.name].text += output.text
            else:
                new_outputs.append(output)
                streams[output.name] = output
        else:
            new_outputs.append(output)

    # process \r and \b characters
    for output in streams.values():
        old = output.text
        while len(output.text) < len(old):
            old = output.text
            # Cancel out anything-but-newline followed by backspace
            output.text = backspace_pat.sub('', output.text)
        # Replace all carriage returns not followed by newline
        output.text = carriagereturn_pat.sub('', output.text)

    return new_outputs
219348
def string_to_list(string, sep=",", filter_empty=False):
    """"""
    return [value.strip() for value in string.split(sep)
            if (not filter_empty or value)]
666334
def _split_string_to_tokens(text):
  """"""
  if not text:
    return []
  ret = []
  token_start = 0
  # Classify each character in the input string
  is_alnum = [c in _ALPHANUMERIC_CHAR_SET for c in text]
  for pos in xrange(1, len(text)):
    if is_alnum[pos] != is_alnum[pos - 1]:
      token = text[token_start:pos]
      if token != u" " or token_start == 0:
        ret.append(token)
      token_start = pos
  final_token = text[token_start:]
  ret.append(final_token)
  return ret
767665
def idf2txt(txt):
    """"""
    astr = nocomment(txt)
    objs = astr.split(';')
    objs = [obj.split(',') for obj in objs]
    objs = [[line.strip() for line in obj] for obj in objs]
    objs = [[_tofloat(line) for line in obj] for obj in objs]
    objs = [tuple(obj) for obj in objs]
    objs.sort()

    lst = []
    for obj in objs:
        for field in obj[:-1]:
            lst.append('%s,' % (field, ))
        lst.append('%s;\n' % (obj[-1], ))

    return '\n'.join(lst)
795069
def get_tamil_words_iterable( letters ):
        
        #punctuations = u'-,+,/,*,>,<,_,],[,{,},(,)'.split(',')+[',']
        #isspace_or_tamil = lambda x:  not x in punctuations  and tamil.utf8.istamil(x)
        
        # correct algorithm for get-tamil-words
        buf = []
        for idx,let in enumerate(letters):
            if tamil.utf8.istamil( let ):
                buf.append( let )
            else:
                if len(buf) > 0:
                    yield  u"".join( buf )
                    buf = []
        if len(buf) > 0:
            yield u"".join(buf)
536084
private static void readToList(InputStream input, String encoding, List<String> blackPrefixList) {
        InputStreamReader reader = null;
        BufferedReader bufferedReader = null;
        try {
            reader = new InputStreamReader(input, encoding);
            bufferedReader = new BufferedReader(reader);
            String lineText;
            while ((lineText = bufferedReader.readLine()) != null) {
                String pkg = lineText.trim();
                if (pkg.length() > 0) {
                    blackPrefixList.add(pkg);
                }
            }
        } catch (IOException e) {
            if (LOGGER.isWarnEnabled()) {
                LOGGER.warn(e.getMessage(), e);
            }
        } finally {
            closeQuietly(bufferedReader);
            closeQuietly(reader);
        }
    }
851302
def word_to_vector(word):
    
    vector = []
    for char in list(word):
        vector.append(char2int(char))
    return vector
578559
static private void decodeLastCharacter(byte chunk, int characterIndex, byte[] bytes, int index) {
        int byteIndex = index + (characterIndex * 5) / 8;
        int offset = characterIndex % 8;
        switch (offset) {
            case 1:
                bytes[byteIndex] |= chunk >>> 2;
                break;
            case 3:
                bytes[byteIndex] |= chunk >>> 4;
                break;
            case 4:
                bytes[byteIndex] |= chunk >>> 1;
                break;
            case 6:
                bytes[byteIndex] |= chunk >>> 3;
                break;
            case 7:
                bytes[byteIndex] |= chunk;
                break;
        }
    }
733999
def split(self, sep=TOKENS):
        
        if sep != TOKENS:
            return unicode.split(self, sep)
        if len(self) == 0:
            return []
        return [[[x.replace("&slash;", "/") for x in token.split("/")]
            for token in sentence.split(" ")]
                for sentence in unicode.split(self, "\n")]
556505
def eliminate_repeats(text):
    ''''''
    
    bannedwords = read_file('stopwords.txt')
    alphabet = 'abcdefghijklmnopqrstuvwxyz'
    words = text.split()
    standardwords = []
    for word in words:
        newstr = ''
        for char in word:
            if char in alphabet or char in alphabet.upper():
               newstr += char
        if newstr not in standardwords and newstr != '' and newstr not in bannedwords:
            standardwords.append(newstr)
        
    return map(lambda x: x.lower(),standardwords)
214068
def parse(self, stream):
        """"""

        items = OrderedDict()
        for i, line in enumerate(stream):
            line = line.strip()
            if not line or line[0] in ["#", ";", "["] or line.startswith("---"):
                continue
            white_space = "\\s*"
            key = "(?P<key>[^:=;#\s]+?)"
            value = white_space+"[:=\s]"+white_space+"(?P<value>.+?)"
            comment = white_space+"(?P<comment>\\s[;#].*)?"

            key_only_match = re.match("^" + key + comment + "$", line)
            if key_only_match:
                key = key_only_match.group("key")
                items[key] = "true"
                continue

            key_value_match = re.match("^"+key+value+comment+"$", line)
            if key_value_match:
                key = key_value_match.group("key")
                value = key_value_match.group("value")

                if value.startswith("[") and value.endswith("]"):
                    # handle special case of lists
                    value = [elem.strip() for elem in value[1:-1].split(",")]

                items[key] = value
                continue

            raise ConfigFileParserException("Unexpected line %s in %s: %s" % (i,
                getattr(stream, 'name', 'stream'), line))
        return items
1283351
function splitText(text, options) {
  const KIND_NON_CJK = "non-cjk";
  const KIND_CJ_LETTER = "cj-letter";
  const KIND_K_LETTER = "k-letter";
  const KIND_CJK_PUNCTUATION = "cjk-punctuation";

  const nodes = [];

  (options.proseWrap === "preserve"
    ? text
    : text.replace(new RegExp(`(${cjkPattern})\n(${cjkPattern})`, "g"), "$1$2")
  )
    .split(/([ \t\n]+)/)
    .forEach((token, index, tokens) => {
      // whitespace
      if (index % 2 === 1) {
        nodes.push({
          type: "whitespace",
          value: /\n/.test(token) ? "\n" : " "
        });
        return;
      }

      // word separated by whitespace

      if ((index === 0 || index === tokens.length - 1) && token === "") {
        return;
      }

      token
        .split(new RegExp(`(${cjkPattern})`))
        .forEach((innerToken, innerIndex, innerTokens) => {
          if (
            (innerIndex === 0 || innerIndex === innerTokens.length - 1) &&
            innerToken === ""
          ) {
            return;
          }

          // non-CJK word
          if (innerIndex % 2 === 0) {
            if (innerToken !== "") {
              appendNode({
                type: "word",
                value: innerToken,
                kind: KIND_NON_CJK,
                hasLeadingPunctuation: punctuationRegex.test(innerToken[0]),
                hasTrailingPunctuation: punctuationRegex.test(
                  getLast(innerToken)
                )
              });
            }
            return;
          }

          // CJK character
          appendNode(
            punctuationRegex.test(innerToken)
              ? {
                  type: "word",
                  value: innerToken,
                  kind: KIND_CJK_PUNCTUATION,
                  hasLeadingPunctuation: true,
                  hasTrailingPunctuation: true
                }
              : {
                  type: "word",
                  value: innerToken,
                  kind: kRegex.test(innerToken)
                    ? KIND_K_LETTER
                    : KIND_CJ_LETTER,
                  hasLeadingPunctuation: false,
                  hasTrailingPunctuation: false
                }
          );
        });
    });

  return nodes;

  function appendNode(node) {
    const lastNode = getLast(nodes);
    if (lastNode && lastNode.type === "word") {
      if (
        (lastNode.kind === KIND_NON_CJK &&
          node.kind === KIND_CJ_LETTER &&
          !lastNode.hasTrailingPunctuation) ||
        (lastNode.kind === KIND_CJ_LETTER &&
          node.kind === KIND_NON_CJK &&
          !node.hasLeadingPunctuation)
      ) {
        nodes.push({ type: "whitespace", value: " " });
      } else if (
        !isBetween(KIND_NON_CJK, KIND_CJK_PUNCTUATION) &&
        // disallow leading/trailing full-width whitespace
        ![lastNode.value, node.value].some(value => /\u3000/.test(value))
      ) {
        nodes.push({ type: "whitespace", value: "" });
      }
    }
    nodes.push(node);

    function isBetween(kind1, kind2) {
      return (
        (lastNode.kind === kind1 && node.kind === kind2) ||
        (lastNode.kind === kind2 && node.kind === kind1)
      );
    }
  }
}
1110641
public static List<String> split(String input, String punctuationChars, String quoteChars) {
		return split(input, punctuationChars, quoteChars, false, false, false);
	}