Test program for demonstrating the Stemmer.  It reads a file and
stems each word, writing the result to standard out.
Usage: Stemmer file-name
198997
public static void main(String[] args) {
    PorterStemmer s = new PorterStemmer();

		// java org.itc.irst.tcc.kre.util.PorterStemmer
    for (int i = 0; i < args.length; i++) {
      try {
        InputStream in = new FileInputStream(args[i]);
        byte[] buffer = new byte[1024];
        int bufferLen, offset, ch;

        bufferLen = in.read(buffer);
        offset = 0;
        s.reset();

        while(true) {
          if (offset < bufferLen)
            ch = buffer[offset++];
          else {
            bufferLen = in.read(buffer);
            offset = 0;
            if (bufferLen < 0)
              ch = -1;
            else
              ch = buffer[offset++];
          }

          if (Character.isLetter((char) ch)) {
            s.add(Character.toLowerCase((char) ch));
          }
          else {
             s.stem();
             System.out.print(s.toString());
             s.reset();
             if (ch < 0)
               break;
             else {
               System.out.print((char) ch);
             }
           }
        }

        in.close();
      }
      catch (IOException e) {
        System.out.println("error reading " + args[i]);
      }
    }
  }
105025
def build_and_run_file(filename):
    ''' 
    '''
    (directory, name, extension) = get_file_tuple(filename)
    if extension == 'c':
        print(" = = = = = = ", YELLOW, "GCC: Compiling " + filename + " file", \
              RESET, " = = = = = =\n")
        compiler = Compiler(filename)
        out = compiler.compile()
        if out != 0:
            print('Error while compiling. Code:', out, 'Please retry.')
            return out
        print("")

        out = compiler.run()
        return out

    elif extension == 'cpp':
        print(" = = = = = = ", YELLOW, "GPP: Compiling " + filename + " file", \
              RESET, " = = = = = =\n")
        compiler = Compiler(filename)
        out = compiler.compile()
        if out != 0:
            print('Error while compiling. Code:', out, 'Please retry.')
            return out
        print("")

        out = compiler.run()
        return out

    elif extension == 'py':
        print(" = = = = = = ", YELLOW, "PYTHON: Executing " + filename + " file", \
              RESET, " = = = = = =\n")
        compiler = Compiler(filename)
        out = compiler.run()
        return out
    elif extension == 'java':
        command = EXECUTABLE_JAVAC + ' ' + filename
        perform_system_command(command)
        command_run = EXECUTABLE_JAVA + ' ' + name
        test_file = directory + "/" + name + ".input"
        if os.path.exists(test_file):
            command_run += " < " + test_file
        return perform_system_command(command_run)
    else:
        print("Language yet not supported")
        return -1
1002168
def stem(self, word):
        
        word = word.lower()

        step1_success = False
        rv_ending_found = False
        step2a_success = False
        step2b_success = False

        # Every occurrence of 'u' after 'q' is put into upper case.
        for i in range(1, len(word)):
            if word[i-1] == "q" and word[i] == "u":
                word = "".join((word[:i], "U", word[i+1:]))

        # Every occurrence of 'u' and 'i'
        # between vowels is put into upper case.
        # Every occurrence of 'y' preceded or
        # followed by a vowel is also put into upper case.
        for i in range(1, len(word)-1):
            if word[i-1] in self.__vowels and word[i+1] in self.__vowels:
                if word[i] == "u":
                    word = "".join((word[:i], "U", word[i+1:]))

                elif word[i] == "i":
                    word = "".join((word[:i], "I", word[i+1:]))

            if word[i-1] in self.__vowels or word[i+1] in self.__vowels:
                if word[i] == "y":
                    word = "".join((word[:i], "Y", word[i+1:]))

        r1, r2 = self._r1r2_standard(word, self.__vowels)
        rv = self.__rv_french(word, self.__vowels)

        # STEP 1: Standard suffix removal
        for suffix in self.__step1_suffixes:
            if word.endswith(suffix):
                if suffix == "eaux":
                    word = word[:-1]
                    step1_success = True

                elif suffix in ("euse", "euses"):
                    if suffix in r2:
                        word = word[:-len(suffix)]
                        step1_success = True

                    elif suffix in r1:
                        word = "".join((word[:-len(suffix)], "eux"))
                        step1_success = True

                elif suffix in ("ement", "ements") and suffix in rv:
                    word = word[:-len(suffix)]
                    step1_success = True

                    if word[-2:] == "iv" and "iv" in r2:
                        word = word[:-2]

                        if word[-2:] == "at" and "at" in r2:
                            word = word[:-2]

                    elif word[-3:] == "eus":
                        if "eus" in r2:
                            word = word[:-3]
                        elif "eus" in r1:
                            word = "".join((word[:-1], "x"))

                    elif word[-3:] in ("abl", "iqU"):
                        if "abl" in r2 or "iqU" in r2:
                            word = word[:-3]

                    elif word[-3:] in ("i\xE8r", "I\xE8r"):
                        if "i\xE8r" in rv or "I\xE8r" in rv:
                            word = "".join((word[:-3], "i"))

                elif suffix == "amment" and suffix in rv:
                    word = "".join((word[:-6], "ant"))
                    rv = "".join((rv[:-6], "ant"))
                    rv_ending_found = True

                elif suffix == "emment" and suffix in rv:
                    word = "".join((word[:-6], "ent"))
                    rv_ending_found = True

                elif (suffix in ("ment", "ments") and suffix in rv and
                      not rv.startswith(suffix) and
                      rv[rv.rindex(suffix)-1] in self.__vowels):
                    word = word[:-len(suffix)]
                    rv = rv[:-len(suffix)]
                    rv_ending_found = True

                elif suffix == "aux" and suffix in r1:
                    word = "".join((word[:-2], "l"))
                    step1_success = True

                elif (suffix in ("issement", "issements") and suffix in r1
                      and word[-len(suffix)-1] not in self.__vowels):
                    word = word[:-len(suffix)]
                    step1_success = True

                elif suffix in ("ance", "iqUe", "isme", "able", "iste",
                              "eux", "ances", "iqUes", "ismes",
                              "ables", "istes") and suffix in r2:
                    word = word[:-len(suffix)]
                    step1_success = True

                elif suffix in ("atrice", "ateur", "ation", "atrices",
                                "ateurs", "ations") and suffix in r2:
                    word = word[:-len(suffix)]
                    step1_success = True

                    if word[-2:] == "ic":
                        if "ic" in r2:
                            word = word[:-2]
                        else:
                            word = "".join((word[:-2], "iqU"))

                elif suffix in ("logie", "logies") and suffix in r2:
                    word = "".join((word[:-len(suffix)], "log"))
                    step1_success = True

                elif (suffix in ("usion", "ution", "usions", "utions") and
                      suffix in r2):
                    word = "".join((word[:-len(suffix)], "u"))
                    step1_success = True

                elif suffix in ("ence", "ences") and suffix in r2:
                    word = "".join((word[:-len(suffix)], "ent"))
                    step1_success = True

                elif suffix in ("it\xE9", "it\xE9s") and suffix in r2:
                    word = word[:-len(suffix)]
                    step1_success = True

                    if word[-4:] == "abil":
                        if "abil" in r2:
                            word = word[:-4]
                        else:
                            word = "".join((word[:-2], "l"))

                    elif word[-2:] == "ic":
                        if "ic" in r2:
                            word = word[:-2]
                        else:
                            word = "".join((word[:-2], "iqU"))

                    elif word[-2:] == "iv":
                        if "iv" in r2:
                            word = word[:-2]

                elif (suffix in ("if", "ive", "ifs", "ives") and
                      suffix in r2):
                    word = word[:-len(suffix)]
                    step1_success = True

                    if word[-2:] == "at" and "at" in r2:
                        word = word[:-2]

                        if word[-2:] == "ic":
                            if "ic" in r2:
                                word = word[:-2]
                            else:
                                word = "".join((word[:-2], "iqU"))
                break

        # STEP 2a: Verb suffixes beginning 'i'
        if not step1_success or rv_ending_found:
            for suffix in self.__step2a_suffixes:
                if word.endswith(suffix):
                    if (suffix in rv and len(rv) > len(suffix) and
                        rv[rv.rindex(suffix)-1] not in self.__vowels):
                        word = word[:-len(suffix)]
                        step2a_success = True
                    break

        # STEP 2b: Other verb suffixes
            if not step2a_success:
                for suffix in self.__step2b_suffixes:
                    if rv.endswith(suffix):
                        if suffix == "ions" and "ions" in r2:
                            word = word[:-4]
                            step2b_success = True

                        elif suffix in ('eraIent', 'erions', '\xE8rent',
                                        'erais', 'erait', 'eriez',
                                        'erons', 'eront', 'erai', 'eras',
                                        'erez', '\xE9es', 'era', 'iez',
                                        '\xE9e', '\xE9s', 'er', 'ez',
                                        '\xE9'):
                            word = word[:-len(suffix)]
                            step2b_success = True

                        elif suffix in ('assions', 'assent', 'assiez',
                                        'aIent', 'antes', 'asses',
                                        '\xE2mes', '\xE2tes', 'ante',
                                        'ants', 'asse', 'ais', 'ait',
                                        'ant', '\xE2t', 'ai', 'as',
                                        'a'):
                            word = word[:-len(suffix)]
                            rv = rv[:-len(suffix)]
                            step2b_success = True
                            if rv.endswith("e"):
                                word = word[:-1]
                        break

        # STEP 3
        if step1_success or step2a_success or step2b_success:
            if word[-1] == "Y":
                word = "".join((word[:-1], "i"))
            elif word[-1] == "\xE7":
                word = "".join((word[:-1], "c"))

        # STEP 4: Residual suffixes
        else:
            if (len(word) >= 2 and word[-1] == "s" and
                word[-2] not in "aiou\xE8s"):
                word = word[:-1]

            for suffix in self.__step4_suffixes:
                if word.endswith(suffix):
                    if suffix in rv:
                        if (suffix == "ion" and suffix in r2 and
                            rv[-4] in "st"):
                            word = word[:-3]

                        elif suffix in ("ier", "i\xE8re", "Ier",
                                        "I\xE8re"):
                            word = "".join((word[:-len(suffix)], "i"))

                        elif suffix == "e":
                            word = word[:-1]

                        elif suffix == "\xEB" and word[-3:-1] == "gu":
                            word = word[:-1]
                        break

        # STEP 5: Undouble
        if word.endswith(("enn", "onn", "ett", "ell", "eill")):
            word = word[:-1]

        # STEP 6: Un-accent
        for i in range(1, len(word)):
            if word[-i] not in self.__vowels:
                i += 1
            else:
                if i != 1 and word[-i] in ("\xE9", "\xE8"):
                    word = "".join((word[:-i], "e", word[-i+1:]))
                break

        word = (word.replace("I", "i")
                    .replace("U", "u")
                    .replace("Y", "y"))


        return word
73344
def stem(self, form, tag):
        """"""
        key = (form, tag)
        if key not in self.lemma_cache:
            lemma = self.stemmer(*key).word()
            self.lemma_cache[key] = lemma
        return self.lemma_cache[key]
401666
def stem(self):

        

        def s(tokens):
            return [PorterStemmer().stem(t) for t in tokens]
        self.stems = list(map(s, self.tokens))
553291
function stem(token) {
  token = prelude(token.toLowerCase());

  if (token.length == 1)
    return token;

  var regs = regions(token);

  var r1_txt, r2_txt, rv_txt;
  r1_txt = token.substring(regs.r1);
  r2_txt = token.substring(regs.r2);
  rv_txt = token.substring(regs.rv);

  // Step 1
  var beforeStep1 = token;
  var suf, pref2, pref3, letterBefore, letter2Before, i;
  var doStep2a = false;

  if ((suf = endsinArr(r2_txt, ['ance', 'iqUe', 'isme', 'able', 'iste', 'eux', 'ances', 'iqUes', 'ismes', 'ables', 'istes'])) != '') {
    token = token.slice(0, -suf.length); // delete
  } else if ((suf = endsinArr(token, ['icatrice', 'icateur', 'ication', 'icatrices', 'icateurs', 'ications'])) != '') {
    if (endsinArr(r2_txt, ['icatrice', 'icateur', 'ication', 'icatrices', 'icateurs', 'ications']) != '') {
      token = token.slice(0, -suf.length); // delete
    } else {
      token = token.slice(0, -suf.length) + 'iqU'; // replace by iqU
    }
  } else if ((suf = endsinArr(r2_txt, ['atrice', 'ateur', 'ation', 'atrices', 'ateurs', 'ations'])) != '') {
    token = token.slice(0, -suf.length); // delete
  } else if ((suf = endsinArr(r2_txt, ['logie', 'logies'])) != '') {
    token = token.slice(0, -suf.length) + 'log'; // replace with log
  } else if ((suf = endsinArr(r2_txt, ['usion', 'ution', 'usions', 'utions'])) != '') {
    token = token.slice(0, -suf.length) + 'u'; // replace with u
  } else if ((suf = endsinArr(r2_txt, ['ence', 'ences'])) != '') {
    token = token.slice(0, -suf.length) + 'ent'; // replace with ent
  }
  // ement(s)
  else if ((suf = endsinArr(r1_txt, ['issement', 'issements'])) != '') {
    if (!isVowel(token[token.length - suf.length - 1])) {
      token = token.slice(0, -suf.length); // delete
      r1_txt = token.substring(regs.r1);
      r2_txt = token.substring(regs.r2);
      rv_txt = token.substring(regs.rv);
    }
  } else if ((suf = endsinArr(r2_txt, ['ativement', 'ativements'])) != '') {
    token = token.slice(0, -suf.length); // delete
  } else if ((suf = endsinArr(r2_txt, ['ivement', 'ivements'])) != '') {
    token = token.slice(0, -suf.length); // delete
  } else if ((suf = endsinArr(token, ['eusement', 'eusements'])) != '') {
    if ((suf = endsinArr(r2_txt, ['eusement', 'eusements'])) != '')
      token = token.slice(0, -suf.length); // delete
    else if ((suf = endsinArr(r1_txt, ['eusement', 'eusements'])) != '')
      token = token.slice(0, -suf.length) + 'eux'; // replace by eux
    else if ((suf = endsinArr(rv_txt, ['ement', 'ements'])) != '')
      token = token.slice(0, -suf.length); // delete
  } else if ((suf = endsinArr(r2_txt, ['ablement', 'ablements', 'iqUement', 'iqUements'])) != '') {
    token = token.slice(0, -suf.length); // delete
  } else if ((suf = endsinArr(rv_txt, ['ièrement', 'ièrements', 'Ièrement', 'Ièrements'])) != '') {
    token = token.slice(0, -suf.length) + 'i'; // replace by i
  } else if ((suf = endsinArr(rv_txt, ['ement', 'ements'])) != '') {
    token = token.slice(0, -suf.length); // delete
  }
  // ité(s)
  else if ((suf = endsinArr(token, ['icité', 'icités'])) != '') {
    if (endsinArr(r2_txt, ['icité', 'icités']) != '')
      token = token.slice(0, -suf.length); // delete
    else
      token = token.slice(0, -suf.length) + 'iqU'; // replace by iqU
  } else if ((suf = endsinArr(token, ['abilité', 'abilités'])) != '') {
    if (endsinArr(r2_txt, ['abilité', 'abilités']) != '')
      token = token.slice(0, -suf.length); // delete
    else
      token = token.slice(0, -suf.length) + 'abl'; // replace by abl
  } else if ((suf = endsinArr(r2_txt, ['ité', 'ités'])) != '') {
    token = token.slice(0, -suf.length); // delete if in R2
  } else if ((suf = endsinArr(token, ['icatif', 'icative', 'icatifs', 'icatives'])) != '') {
    if ((suf = endsinArr(r2_txt, ['icatif', 'icative', 'icatifs', 'icatives'])) != '') {
      token = token.slice(0, -suf.length); // delete
      r2_txt = token.substring(regs.r2);
      rv_txt = token.substring(regs.rv);
    }
    if ((suf = endsinArr(r2_txt, ['atif', 'ative', 'atifs', 'atives'])) != '') {
      token = token.slice(0, -suf.length - 2) + 'iqU'; // replace with iqU
      r2_txt = token.substring(regs.r2);
      rv_txt = token.substring(regs.rv);
    }
  } else if ((suf = endsinArr(r2_txt, ['atif', 'ative', 'atifs', 'atives'])) != '') {
    token = token.slice(0, -suf.length); // delete
  } else if ((suf = endsinArr(r2_txt, ['if', 'ive', 'ifs', 'ives'])) != '') {
    token = token.slice(0, -suf.length); // delete
  } else if ((suf = endsinArr(token, ['eaux'])) != '') {
    token = token.slice(0, -suf.length) + 'eau'; // replace by eau
  } else if ((suf = endsinArr(r1_txt, ['aux'])) != '') {
    token = token.slice(0, -suf.length) + 'al'; // replace by al
  } else if ((suf = endsinArr(r2_txt, ['euse', 'euses'])) != '') {
    token = token.slice(0, -suf.length); // delete
  } else if ((suf = endsinArr(r1_txt, ['euse', 'euses'])) != '') {
    token = token.slice(0, -suf.length) + 'eux'; // replace by eux
  } else if ((suf = endsinArr(rv_txt, ['amment'])) != '') {
    token = token.slice(0, -suf.length) + 'ant'; // replace by ant
    doStep2a = true;
  } else if ((suf = endsinArr(rv_txt, ['emment'])) != '') {
    token = token.slice(0, -suf.length) + 'ent'; // replace by ent
    doStep2a = true;
  } else if ((suf = endsinArr(rv_txt, ['ment', 'ments'])) != '') {
    // letter before must be a vowel in RV
    letterBefore = token[token.length - suf.length - 1];
    if (isVowel(letterBefore) && endsin(rv_txt, letterBefore + suf)) {
      token = token.slice(0, -suf.length); // delete
      doStep2a = true;
    }
  }

  // re compute regions
  r1_txt = token.substring(regs.r1);
  r2_txt = token.substring(regs.r2);
  rv_txt = token.substring(regs.rv);

  // Step 2a
  var beforeStep2a = token;
  var step2aDone = false;
  if (beforeStep1 === token || doStep2a) {
    step2aDone = true;
    if ((suf = endsinArr(rv_txt, ['îmes', 'ît', 'îtes', 'i', 'ie', 'Ie', 'ies', 'ir', 'ira', 'irai', 'iraIent', 'irais', 'irait', 'iras', 'irent', 'irez', 'iriez', 'irions', 'irons', 'iront', 'is', 'issaIent', 'issais', 'issait', 'issant', 'issante', 'issantes', 'issants', 'isse', 'issent', 'isses', 'issez', 'issiez', 'issions', 'issons', 'it'])) != '') {
      letterBefore = token[token.length - suf.length - 1];
      if (!isVowel(letterBefore) && endsin(rv_txt, letterBefore + suf))
        token = token.slice(0, -suf.length); // delete
    }
  }

  // Step 2b
  if (step2aDone && token === beforeStep2a) {
    if ((suf = endsinArr(rv_txt, ['é', 'ée', 'ées', 'és', 'èrent', 'er', 'era', 'erai', 'eraIent', 'erais', 'erait', 'eras', 'erez', 'eriez', 'erions', 'erons', 'eront', 'ez', 'iez', 'Iez'])) != '') {
      token = token.slice(0, -suf.length); // delete
      r2_txt = token.substring(regs.r2);
      rv_txt = token.substring(regs.rv);
    } else if ((suf = endsinArr(rv_txt, ['ions'])) != '' && endsinArr(r2_txt, ['ions'])) {
      token = token.slice(0, -suf.length); // delete
      r2_txt = token.substring(regs.r2);
      rv_txt = token.substring(regs.rv);
    }
    // add 'Ie' suffix to pass test for 'évanouie'
    else if ((suf = endsinArr(rv_txt, ['âmes', 'ât', 'âtes', 'a', 'ai', 'aIent', 'ais', 'ait', 'ant', 'ante', 'antes', 'ants', 'as', 'asse', 'assent', 'asses', 'assiez', 'assions'])) != '') {
      token = token.slice(0, -suf.length); // delete

      letterBefore = token[token.length - 1];
      if (letterBefore === 'e' && endsin(rv_txt, 'e' + suf))
        token = token.slice(0, -1);

      r2_txt = token.substring(regs.r2);
      rv_txt = token.substring(regs.rv);
    }
  }

  // Step 3
  if (!(token === beforeStep1)) {
    if (token[token.length - 1] === 'Y')
      token = token.slice(0, -1) + 'i';
    if (token[token.length - 1] === 'ç')
      token = token.slice(0, -1) + 'c';
  } // Step 4
  else {
    letterBefore = token[token.length - 1];
    letter2Before = token[token.length - 2];

    if (letterBefore === 's' && ['a', 'i', 'o', 'u', 'è', 's'].indexOf(letter2Before) == -1) {
      token = token.slice(0, -1);
      r1_txt = token.substring(regs.r1);
      r2_txt = token.substring(regs.r2);
      rv_txt = token.substring(regs.rv);
    }

    if ((suf = endsinArr(r2_txt, ['ion'])) != '') {
      letterBefore = token[token.length - suf.length - 1];
      if (letterBefore === 's' || letterBefore === 't') {
        token = token.slice(0, -suf.length); // delete
        r1_txt = token.substring(regs.r1);
        r2_txt = token.substring(regs.r2);
        rv_txt = token.substring(regs.rv);
      }
    }

    if ((suf = endsinArr(rv_txt, ['ier', 'ière', 'Ier', 'Ière'])) != '') {
      token = token.slice(0, -suf.length) + 'i'; // replace by i
      r1_txt = token.substring(regs.r1);
      r2_txt = token.substring(regs.r2);
      rv_txt = token.substring(regs.rv);
    }
    if ((suf = endsinArr(rv_txt, 'e')) != '') {
      token = token.slice(0, -suf.length); // delete
      r1_txt = token.substring(regs.r1);
      r2_txt = token.substring(regs.r2);
      rv_txt = token.substring(regs.rv);
    }
    if ((suf = endsinArr(rv_txt, 'ë')) != '') {
      if (token.slice(token.length - 3, -1) === 'gu')
        token = token.slice(0, -suf.length); // delete
    }
  }

  // Step 5
  if ((suf = endsinArr(token, ['enn', 'onn', 'ett', 'ell', 'eill'])) != '') {
    token = token.slice(0, -1); // delete last letter
  }

  // Step 6
  i = token.length - 1;
  while (i > 0) {
    if (!isVowel(token[i])) {
      i--;
    } else if (i !== token.length - 1 && (token[i] === 'é' || token[i] === 'è')) {
      token = token.substring(0, i) + 'e' + token.substring(i + 1, token.length);
      break;
    } else {
      break;
    }
  }

  return token.toLowerCase();

}
557512
function stemmingProcess(){
                if(find(current_word))
                    return
                
                // Confix Stripping
                // Try to remove prefixes first before suffixes if the specification is met
                if(precedenceAdjustmentSpecification(original_word)){
                    // Step 4, 5
                    removePrefixes();
                    if(find(current_word))
                        return

                    // Step 2, 3
                    removeSuffixes();
                    if(find(current_word)){
                        return
                    }
                    else{
                        // if the trial is failed, restore the original word
                        // and continue to normal rule precedence (suffix first, prefix afterwards)
                        current_word = original_word;
                        removals = [];
                    }
                }

                // Step 2, 3
                removeSuffixes();
                if(find(current_word))
                    return
                
                // Step 4, 5
                removePrefixes();
                if(find(current_word))
                    return

                //ECS Loop Restore Prefixes
                loopRestorePrefixes();
            }
486227
def stem(self, word):
        
        word = word.lower()

        step1_success = False
        step2_success = False

        for i in range(1, len(word)-1):
            if word[i-1] in self.__vowels and word[i+1] in self.__vowels:
                if word[i] == "u":
                    word = "".join((word[:i], "U", word[i+1:]))

                elif word[i] == "i":
                    word = "".join((word[:i], "I", word[i+1:]))

        r1, r2 = self._r1r2_standard(word, self.__vowels)
        rv = self._rv_standard(word, self.__vowels)

        # STEP 0: Removal of plurals and other simplifications
        for suffix in self.__step0_suffixes:
            if word.endswith(suffix):
                if suffix in r1:
                    if suffix in ("ul", "ului"):
                        word = word[:-len(suffix)]

                        if suffix in rv:
                            rv = rv[:-len(suffix)]
                        else:
                            rv = ""

                    elif (suffix == "aua" or suffix == "atei" or
                          (suffix == "ile" and word[-5:-3] != "ab")):
                        word = word[:-2]

                    elif suffix in ("ea", "ele", "elor"):
                        word = "".join((word[:-len(suffix)], "e"))

                        if suffix in rv:
                            rv = "".join((rv[:-len(suffix)], "e"))
                        else:
                            rv = ""

                    elif suffix in ("ii", "iua", "iei",
                                    "iile", "iilor", "ilor"):
                        word = "".join((word[:-len(suffix)], "i"))

                        if suffix in rv:
                            rv = "".join((rv[:-len(suffix)], "i"))
                        else:
                            rv = ""

                    elif suffix in ("a\u0163ie", "a\u0163ia"):
                        word = word[:-1]
                break

        # STEP 1: Reduction of combining suffixes
        while True:

            replacement_done = False

            for suffix in self.__step1_suffixes:
                if word.endswith(suffix):
                    if suffix in r1:
                        step1_success = True
                        replacement_done = True

                        if suffix in ("abilitate", "abilitati",
                                      "abilit\u0103i",
                                      "abilit\u0103\u0163i"):
                            word = "".join((word[:-len(suffix)], "abil"))

                        elif suffix == "ibilitate":
                            word = word[:-5]

                        elif suffix in ("ivitate", "ivitati",
                                        "ivit\u0103i",
                                        "ivit\u0103\u0163i"):
                            word = "".join((word[:-len(suffix)], "iv"))

                        elif suffix in ("icitate", "icitati", "icit\u0103i",
                                        "icit\u0103\u0163i", "icator",
                                        "icatori", "iciv", "iciva",
                                        "icive", "icivi", "iciv\u0103",
                                        "ical", "icala", "icale", "icali",
                                        "ical\u0103"):
                            word = "".join((word[:-len(suffix)], "ic"))

                        elif suffix in ("ativ", "ativa", "ative", "ativi",
                                        "ativ\u0103", "a\u0163iune",
                                        "atoare", "ator", "atori",
                                        "\u0103toare",
                                        "\u0103tor", "\u0103tori"):
                            word = "".join((word[:-len(suffix)], "at"))

                            if suffix in r2:
                                r2 = "".join((r2[:-len(suffix)], "at"))

                        elif suffix in ("itiv", "itiva", "itive", "itivi",
                                        "itiv\u0103", "i\u0163iune",
                                        "itoare", "itor", "itori"):
                            word = "".join((word[:-len(suffix)], "it"))

                            if suffix in r2:
                                r2 = "".join((r2[:-len(suffix)], "it"))
                    else:
                        step1_success = False
                    break

            if not replacement_done:
                break

        # STEP 2: Removal of standard suffixes
        for suffix in self.__step2_suffixes:
            if word.endswith(suffix):
                if suffix in r2:
                    step2_success = True

                    if suffix in ("iune", "iuni"):
                        if word[-5] == "\u0163":
                            word = "".join((word[:-5], "t"))

                    elif suffix in ("ism", "isme", "ist", "ista", "iste",
                                    "isti", "ist\u0103", "i\u015Fti"):
                        word = "".join((word[:-len(suffix)], "ist"))

                    else:
                        word = word[:-len(suffix)]
                break

        # STEP 3: Removal of verb suffixes
        if not step1_success and not step2_success:
            for suffix in self.__step3_suffixes:
                if word.endswith(suffix):
                    if suffix in rv:
                        if suffix in ('seser\u0103\u0163i', 'seser\u0103m',
                                      'ser\u0103\u0163i', 'sese\u015Fi',
                                      'seser\u0103', 'ser\u0103m', 'sesem',
                                      'se\u015Fi', 'ser\u0103', 'sese',
                                      'a\u0163i', 'e\u0163i', 'i\u0163i',
                                      '\xE2\u0163i', 'sei', '\u0103m',
                                      'em', 'im', '\xE2m', 'se'):
                            word = word[:-len(suffix)]
                            rv = rv[:-len(suffix)]
                        else:
                            if (not rv.startswith(suffix) and
                                rv[rv.index(suffix)-1] not in
                                "aeio\u0103\xE2\xEE"):
                                word = word[:-len(suffix)]
                        break

        # STEP 4: Removal of final vowel
        for suffix in ("ie", "a", "e", "i", "\u0103"):
            if word.endswith(suffix):
                if suffix in rv:
                    word = word[:-len(suffix)]
                break

        word = word.replace("I", "i").replace("U", "u")


        return word
1268794
public static void main(String[] args) {
    Treebank treebank = new DiskTreebank();
    treebank.loadPath(args[0]);
    WordStemmer ls = new WordStemmer();
    for (Tree tree : treebank) {
      ls.visitTree(tree);
      System.out.println(tree);
    }
  }
328099
def stem(self, word):
        

        if len(word) <= 2:
            return word
        else:
            word = self.remove_initial_apostrophe(word)
            word = self.set_ys(word)
            self.find_regions(word)

            word = self.strip_possessives(word)
            word = self.replace_suffixes_1(word)
            word = self.replace_suffixes_2(word)
            word = self.replace_ys(word)
            word = self.replace_suffixes_3(word)
            word = self.replace_suffixes_4(word)
            word = self.delete_suffixes(word)
            word = self.process_terminals(word)

            return word
271281
def stem(self, word):
        
        word = word.lower()

        r1 = self._r1_scandinavian(word, self.__vowels)

        # STEP 1
        for suffix in self.__step1_suffixes:
            if r1.endswith(suffix):
                if suffix == "s":
                    if word[-2] in self.__s_ending:
                        word = word[:-1]
                        r1 = r1[:-1]
                else:
                    word = word[:-len(suffix)]
                    r1 = r1[:-len(suffix)]
                break

        # STEP 2
        for suffix in self.__step2_suffixes:
            if r1.endswith(suffix):
                word = word[:-1]
                r1 = r1[:-1]
                break

        # STEP 3
        for suffix in self.__step3_suffixes:
            if r1.endswith(suffix):
                if suffix in ("els", "lig", "ig"):
                    word = word[:-len(suffix)]
                elif suffix in ("fullt", "l\xF6st"):
                    word = word[:-1]
                break


        return word
358868
def parse(self, output):
        
        output = self._get_lines_with_stems(output)
        words = self._make_unique(output)
        return self._parse_for_simple_stems(words)
1085475
def stem(self, word):
        
        word = word.lower()

        step1_success = False

        r1, r2 = self._r1r2_standard(word, self.__vowels)
        rv = self._rv_standard(word, self.__vowels)

        # STEP 0: Attached pronoun
        for suffix in self.__step0_suffixes:
            if word.endswith(suffix):
                if rv.endswith(suffix):
                    if rv[:-len(suffix)].endswith(("i\xE9ndo",
                                                   "\xE1ndo",
                                                   "\xE1r", "\xE9r",
                                                   "\xEDr")):
                        word = (word[:-len(suffix)].replace("\xE1", "a")
                                                   .replace("\xE9", "e")
                                                   .replace("\xED", "i"))
                        r1 = (r1[:-len(suffix)].replace("\xE1", "a")
                                               .replace("\xE9", "e")
                                               .replace("\xED", "i"))
                        r2 = (r2[:-len(suffix)].replace("\xE1", "a")
                                               .replace("\xE9", "e")
                                               .replace("\xED", "i"))
                        rv = (rv[:-len(suffix)].replace("\xE1", "a")
                                               .replace("\xE9", "e")
                                               .replace("\xED", "i"))

                    elif rv[:-len(suffix)].endswith(("ando", "iendo",
                                                     "ar", "er", "ir")):
                        word = word[:-len(suffix)]
                        r1 = r1[:-len(suffix)]
                        r2 = r2[:-len(suffix)]
                        rv = rv[:-len(suffix)]

                    elif (rv[:-len(suffix)].endswith("yendo") and
                          word[:-len(suffix)].endswith("uyendo")):
                        word = word[:-len(suffix)]
                        r1 = r1[:-len(suffix)]
                        r2 = r2[:-len(suffix)]
                        rv = rv[:-len(suffix)]
                break

        # STEP 1: Standard suffix removal
        for suffix in self.__step1_suffixes:
            if word.endswith(suffix):
                if suffix == "amente" and r1.endswith(suffix):
                    step1_success = True
                    word = word[:-6]
                    r2 = r2[:-6]
                    rv = rv[:-6]

                    if r2.endswith("iv"):
                        word = word[:-2]
                        r2 = r2[:-2]
                        rv = rv[:-2]

                        if r2.endswith("at"):
                            word = word[:-2]
                            rv = rv[:-2]

                    elif r2.endswith(("os", "ic", "ad")):
                        word = word[:-2]
                        rv = rv[:-2]

                elif r2.endswith(suffix):
                    step1_success = True
                    if suffix in ("adora", "ador", "aci\xF3n", "adoras",
                                  "adores", "aciones", "ante", "antes",
                                  "ancia", "ancias"):
                        word = word[:-len(suffix)]
                        r2 = r2[:-len(suffix)]
                        rv = rv[:-len(suffix)]

                        if r2.endswith("ic"):
                            word = word[:-2]
                            rv = rv[:-2]

                    elif suffix in ("log\xEDa", "log\xEDas"):
                        word = word.replace(suffix, "log")
                        rv = rv.replace(suffix, "log")

                    elif suffix in ("uci\xF3n", "uciones"):
                        word = word.replace(suffix, "u")
                        rv = rv.replace(suffix, "u")

                    elif suffix in ("encia", "encias"):
                        word = word.replace(suffix, "ente")
                        rv = rv.replace(suffix, "ente")

                    elif suffix == "mente":
                        word = word[:-5]
                        r2 = r2[:-5]
                        rv = rv[:-5]

                        if r2.endswith(("ante", "able", "ible")):
                            word = word[:-4]
                            rv = rv[:-4]

                    elif suffix in ("idad", "idades"):
                        word = word[:-len(suffix)]
                        r2 = r2[:-len(suffix)]
                        rv = rv[:-len(suffix)]

                        for pre_suff in ("abil", "ic", "iv"):
                            if r2.endswith(pre_suff):
                                word = word[:-len(pre_suff)]
                                rv = rv[:-len(pre_suff)]

                    elif suffix in ("ivo", "iva", "ivos", "ivas"):
                        word = word[:-len(suffix)]
                        r2 = r2[:-len(suffix)]
                        rv = rv[:-len(suffix)]
                        if r2.endswith("at"):
                            word = word[:-2]
                            rv = rv[:-2]
                    else:
                        word = word[:-len(suffix)]
                        rv = rv[:-len(suffix)]
                break

        # STEP 2a: Verb suffixes beginning 'y'
        if not step1_success:
            for suffix in self.__step2a_suffixes:
                if (rv.endswith(suffix) and
                    word[-len(suffix)-1:-len(suffix)] == "u"):
                    word = word[:-len(suffix)]
                    rv = rv[:-len(suffix)]
                    break

        # STEP 2b: Other verb suffixes
            for suffix in self.__step2b_suffixes:
                if rv.endswith(suffix):
                    if suffix in ("en", "es", "\xE9is", "emos"):
                        word = word[:-len(suffix)]
                        rv = rv[:-len(suffix)]

                        if word.endswith("gu"):
                            word = word[:-1]

                        if rv.endswith("gu"):
                            rv = rv[:-1]
                    else:
                        word = word[:-len(suffix)]
                        rv = rv[:-len(suffix)]
                    break

        # STEP 3: Residual suffix
        for suffix in self.__step3_suffixes:
            if rv.endswith(suffix):
                if suffix in ("e", "\xE9"):
                    word = word[:-len(suffix)]
                    rv = rv[:-len(suffix)]

                    if word[-2:] == "gu" and rv[-1] == "u":
                        word = word[:-1]
                else:
                    word = word[:-len(suffix)]
                break

        word = (word.replace("\xE1", "a").replace("\xE9", "e")
                    .replace("\xED", "i").replace("\xF3", "o")
                    .replace("\xFA", "u"))


        return word
806274
def _check_stems(files):
    """"""
    used = set()
    for fn in files:
        if os.path.basename(fn) in used:
            logger.warning("%s stem is multiple times in your file list, "
                         "so we don't know "
                         "how to assign it to the sample data in the CSV. "
                         "We are gonna use full path to make a difference, "
                         "that means paired files should be in the same folder. "
                         "If this is a problem, you should rename the files you want "
                         "to merge. Sorry, no possible magic here." % os.path.basename(fn)
                         )
            return True
        used.add(os.path.basename(fn))
    return False
1072646
def write_spelling(token_folder, spelling_file):
    
    token_pattern = r'[a-z]{3,}'
    tokens = []
    for base, dirlist, fnlist in os.walk(token_folder):
        for fn in fnlist:
            fp = os.path.join(base, fn)
            with open(fp) as f:
                toks = re.findall(token_pattern, f.read())
                tokens.extend(toks)

    token_ranked, _ = zip(*Counter(tokens).most_common())
    with open(spelling_file, 'w') as f:
        f.write('\n'.join(token_ranked))
264547
def sents(self, filename=None):
        
        filename = filename if filename else self.test_file
        with io.open(filename, 'r') as fin:
            for line in fin:
                yield line.strip()
563090
def stem(self, s):
        """"""
        if self._stemmer is None:
            return passthrough(s)
        try:
            # try the local attribute `stemmer`, a StemmerI instance first
            # if you use the self.stem method from an unpickled object it may not work
            return getattr(getattr(self, '_stemmer', None), 'stem', None)(s)
        except (AttributeError, TypeError):
            return getattr(getattr(self, '_stemmer', self), 'lemmatize', passthrough)(s)