Cleans up the noising lines, e.g., the continuing blank lines

@param linesOfAPage
a list of lines of one page
@return a list of words of one page
486535
private ArrayList<TextPiece> dataCleaning(ArrayList<TextPiece> linesOfAPage) {
	TextPiece line = null;
	TextPiece preLine = null;
	TextPiece nextLine = null;
	for (int i = 0; i < linesOfAPage.size(); i++) {
	    line = linesOfAPage.get(i);
	    preLine = (i == 0) ? null : linesOfAPage.get(i - 1);
	    nextLine = (i == linesOfAPage.size() - 1) ? null : linesOfAPage
		    .get(i + 1);
	    if ((line.getText().replaceAll(" ", "").length() == 0)
		    && (preLine != null) && (nextLine != null)
		    && (preLine.getEndY() < line.getY())
		    && (line.getEndY() < nextLine.getY())) {
		linesOfAPage.remove(i);
		i--;
		// System.out.println("Removed the next line of: " +
		// preLine.getText());
	    }
	}
	return linesOfAPage;
    }
134834
def trim_lines(lines)
      new_lines = lines.dup
      while new_lines[-1] && new_lines[-1].strip.empty?
        new_lines.pop
      end
      new_lines
    end
32740
def remove_empty_text(utterances: List[Utterance]) -> List[Utterance]:
    
    return [utter for utter in utterances if utter.text.strip() != ""]
329471
String[] tokenize(String sentence) {
    TokenizerME tokenizer = new TokenizerME(tokenModel);
    String cleanString = sentence.replace('’', '\'');  // this is the type of apostrophe that OpenNLP expects
    return tokenizer.tokenize(cleanString);
  }
659849
def remove_initial_spaces_and_mark_message_lines(lines):
    
    i = 0
    while i < len(lines):
        lines[i] = lines[i].lstrip(' ')
        i += 1
    return mark_message_lines(lines)
435855
public static String clear(String text) {
        text = StringCleaner.tokenizeURLs(text);
        text = StringCleaner.tokenizeSmileys(text);
        text = StringCleaner.removeAccents(text);
        text = StringCleaner.removeSymbols(text);
        text = StringCleaner.removeExtraSpaces(text);
        
        return text.toLowerCase(Locale.ENGLISH);
    }
1068730
public boolean isLineOfCode(final String line) {
        if (line.length() < getIndicatorAreaPos() + 1) {
            return false;
        }

        /* Remove white space lines */
        if (line.trim().length() == 0) {
            return false;
        }

        /* Remove comments and special lines */
        if (isComment(line)) {
            return false;
        }

        /*
         * If there is a single token on this line, make sure it is not a
         * compiler directive.
         */
        String[] tokens = line.trim().split("[\\s\\.]+");
        if (tokens.length == 1
                && COMPILER_DIRECTIVES.contains(tokens[0].toUpperCase(Locale
                        .getDefault()))) {
            return false;
        }

        return true;
    }
718957
public static String stripSingleLineComments(String text) {
        if (text == null) {
            return null;
        }

        try {
            StringBuffer output = new StringBuffer();

            // First we strip multi line comments. I think this is important:
            BufferedReader in = new BufferedReader(new StringReader(text));
            while (true) {
                String line = in.readLine();
                if (line == null) {
                    break;
                }

                int cstart = line.indexOf(COMMENT_SINGLELINE_START);
                if (cstart >= 0) {
                    line = line.substring(0, cstart);
                }

                output.append(line);
                output.append('\n');
            }

            return output.toString();
        } catch (IOException e) {
            throw new IllegalArgumentException(e);
        }
    }
802890
def lines(self):
        
        # Cache, because this one is reused very often.
        if self._cache.lines is None:
            self._cache.lines = _ImmutableLineList(self.text.split('\n'))

        return self._cache.lines
940008
def remove_empty_head_lines(text)
      return "" unless text
      lines = text.split("\n")
      header = true
      lines.delete_if{ |line|
        header = false if /\S/ =~ line
        header && /^\s*?$/ =~ line
      }
      lines.join("\n")
    end
839349
public static List<String> removeNotes(Collection<String> logLines) {
        List<String> r = new ArrayList<>(logLines.size());
        for (String l : logLines)
            r.add(removeNotes(l));
        return r;
    }
322630
def remove_blank_lines(string):
    
    return "\n".join(line
                     for line in string.split("\n")
                     if len(line.strip()))
1104775
public static ArrayList<String> getFilteredLinesFromFile(final String fileName,
                                                           final String filter,
                                                           final String removeBeforeToken,
                                                           final String removeAfterToken) throws IOException{
    final ArrayList<String> filteredLines = new ArrayList<>();
    try (final BufferedReader in = new BufferedReader(
            new InputStreamReader(new FileInputStream(fileName), StandardCharsets.UTF_8))) {
      String line = "";
      while ((line = in.readLine()) != null) {
        if (line.trim().length() == 0) {
          continue;
        }
        if (line.contains(filter)) {
          String trimedLine;
          if (removeBeforeToken != null) {
            final String[] p = line.split(removeBeforeToken);
            if (p.length > 1) {
              trimedLine = p[p.length-1];
            } else {
              trimedLine = line.trim();
            }
          } else {
            trimedLine = line.trim();
          }
          if (removeAfterToken != null) {
            final String[] p = trimedLine.split(removeAfterToken);
            if (p.length > 1) {
              trimedLine = p[0];
            }
          }
          filteredLines.add(trimedLine);
        }
      }
    }
    return filteredLines;
  }
246442
private static int flushWordBuffer(final StringBuilder lineBuffer, final StringBuilder wordBuffer, final boolean prepend) {

		int wordCount = 0;

		if (wordBuffer.length() > 0) {

			final String word = wordBuffer.toString().replaceAll("[\\n\\t]+", " ");
			if (StringUtils.isNotBlank(word)) {

				if (prepend) {

					lineBuffer.insert(0, word);

				} else {

					lineBuffer.append(word);
				}

				// increase word count
				wordCount = 1;
			}

			wordBuffer.setLength(0);
		}

		return wordCount;
	}
1051379
public static Stream<String> lines(final Reader reader) {
        N.checkArgNotNull(reader);

        return of(createLazyLineIterator(null, null, Charsets.UTF_8, reader, false));
    }
284312
public boolean removeLeadingEmptyLines ()
  {
    boolean bWasEmpty = false;
    Line aLine = m_aLines;
    while (aLine != null && aLine.m_bIsEmpty)
    {
      removeLine (aLine);
      aLine = m_aLines;
      bWasEmpty = true;
    }
    return bWasEmpty;
  }
510025
public void removeTrailingEmptyLines ()
  {
    Line aLine = m_aLineTail;
    while (aLine != null && aLine.m_bIsEmpty)
    {
      removeLine (aLine);
      aLine = m_aLineTail;
    }
  }