Detects documents encoded in HTML code

@param wordsByPage
the extracted texts in the word level in a document
@return A boolean value: true/false
546484
private boolean judgeHTMLCode(ArrayList<ArrayList<TextPiece>> wordsByPage) {
	ArrayList<TextPiece> wordsOfPageOne = wordsByPage.get(0);
	boolean isHTMLCode = true;
	int i = 0;
	/*
	 * do not have to check all words, Checking 10 words is enough
	 */
	while (i < Math.min(20, wordsOfPageOne.size())) {
	    TextPiece currentWord = wordsOfPageOne.get(i);
	    String thisText = currentWord.getText();
	    if ((thisText.length() % 2 != 0)
		    && (thisText.matches("([A-Z]([A-Z]|[0-9]))*") == false)) {
		isHTMLCode = false;
		break;
	    }
	    i++;
	}
	// if (isHTMLCode)
	// System.out.println("This document is coded in HTML code!");
	return isHTMLCode;
    }
534250
private void extractFiguresFromAPage(ArrayList<TextPiece> linesOfAPage,
	    ArrayList<TextPiece> wordsOfAPage, int i, File pdfFile,
	    ArrayList<ArrayList<TextPiece>> linesByPage) {
	if (linesOfAPage.size() > 30) { // if this page is very short, it
					// usually does not contain tables
	    /*
	     * Checks for table keyword
	     */
	    boolean figureKwdExist = false;
	    int prevTableEndIndex = 0;
	    int k = -1;
	    for (TextPiece t : linesOfAPage) {
		k++;
		String textWithoutSpace = t.getText().replace(" ", "");
		TextPiece prevLine = (k == 0) ? null : linesOfAPage.get(k - 1);
		if (getMatchedFigureKeyword(textWithoutSpace) != null) {
		    figureKwdExist = true;
		    String figKwd = getMatchedFigureKeyword(textWithoutSpace);
		    System.out.println("In page " + (i + 1) + ": "
			    + textWithoutSpace);
		}
	    }
	} else {
	    /*
	     * Impossible to contain tables because of few texts in the page
	     */
	}
    }
1241291
static boolean isText(String contentType) {
        return contentType.contains("text") || contentType.contains("json")
                || contentType.contains("xml") || contentType.contains("html");
    }
91650
public static boolean isInHeader() {
    final NodeList scriptList = Browser.getDocument().getHead().getElementsByTagName("script");
    for (int i = 0; i < scriptList.getLength(); i++) {
      final ScriptElement scriptTag = (ScriptElement) scriptList.item(i);
      if (StringUtils.contains(scriptTag.getSrc(), WebshimResources.POLIFILLER_URL)) {
        return true;
      }
    }
    return false;
  }
1309951
private static boolean detectNameWordInSentenceByPosition(final List<Token> _text, final int _pos) {
        boolean isFirstWord = false;
        boolean nextWordIsName = false;
        if (_pos == 0 || !isLetterOrDigit((_text.get(_pos - 1).text.charAt(0)))) {
            isFirstWord = true;
            //noinspection SimplifiableIfStatement
            if (_text.size() > _pos + 1) {
                final String plus1 = _text.get(_pos + 1).text;
                nextWordIsName = ("of".equalsIgnoreCase(plus1) || "'s".equalsIgnoreCase(plus1))
                                 ? ((_text.size() > (_pos + 2)) && isName(_text.get(_pos + 2).text, false, false))
                                 : isName(plus1, false, false);
            }
            else nextWordIsName = false;
        }
        //noinspection UnnecessaryLocalVariable
        final boolean isName = isName(_text.get(_pos).text, isFirstWord, nextWordIsName);

        /*
        String wordType = dict.checkup(Strings.toEngLowerCase(_text.get(_pos));
        if (isFirstWord && !isName && wordType != null && wordType.startsWith("JJ")) {
            // if the first word is determined not to be a name but it is an adj.,
            // and if the second word is a name, we consider the first word to be a name as well.
            if (isName(_text.get(_pos + 1), false))
                return true;
        }
        */

        return isName;
    }
1355174
public boolean showNextPage(AbstractWizardPage nextPage) {
		if (nextPage == null) {
			// Next page is null. Updating buttons and ignoring request.
			updateButtons();
			return false;
		}
		if (currentPage != null) {
			pageHistory.push(currentPage);
		}
		setPage(nextPage);
		return true;
	}
521225
public boolean filter(Event event) {
        LOG.info("StringContentFilter called");
        
        if (wordsToFilter != null) {
            for (String filterWord : wordsToFilter) {
                if (event.getContent() != null
                        && -1 != event.getContent().indexOf(filterWord)) {
                    return true;
                }
            }
        }
        return false;
    }
471399
function containsWords(wordlist, str) {
    var i;
    var matches = [];
    for (i = 0; i < wordlist.length; i++) {
        var re = new RegExp("\\b" + wordlist[i] + "\\b", "i");
        if (re.exec(str)) {
            matches.push(wordlist[i]);
        }
    }
    return matches;
}
1048961
public static String parsedPageToHtml( ParsedPage pp ){
		StringBuilder result = new StringBuilder();
		result.append(getHtmlHeader());

		if( pp != null ) {
    		//Title
    		result.append(
    			"<table class=\"ParsedPage\">\n"+
    			"<tr><th class=\"ParsedPage\">ParsedPage: \n" +
    			pp.getName()+
    			"</th></tr>\n");

//    		if( pp.aboutArticle()!=null ){
//    			result.append("<tr><td class=\"ParsedPage\">\n");
//    			result.append("About Article:" + contentElementToHtml( pp.aboutArticle() ));
//    			result.append("</td></tr>\n");
//    		}

    		//Sections
    		result.append(
    			"<tr><td class=\"ParsedPage\">\n"		);
    		for( Section s: pp.getSections() ) {
				result.append( sectionToHtml( s ));
			}
    		result.append(
    			"</td></tr>\n");

    		//Categories
    		if( pp.getCategoryElement()!= null ){
    			result.append("<tr><td class=\"ParsedPage\">\n");
    			result.append("Categories:\n"  + contentElementToHtml( pp.getCategoryElement() ));
    			result.append("</td></tr>\n");
    		}

    		//Languages
    		if( pp.getLanguagesElement()!= null ){
    			result.append("<tr><td class=\"ParsedPage\">\n");
    			result.append("Languages:\n"  + contentElementToHtml( pp.getLanguagesElement() ));
    			result.append("</td></tr>\n");
    		}

    		//Finalize
    		result.append("</table>\n");
        }

		result.append(getHtmlFooter());

		return result.toString();
	}
1336346
public boolean contains(String encoding) {
        if (encoding.equalsIgnoreCase("sendrecv") || 
                encoding.equalsIgnoreCase("fmtp") ||
                encoding.equalsIgnoreCase("audio") ||
                encoding.equalsIgnoreCase("AS") || 
                encoding.equalsIgnoreCase("IP4")) {
            return true;
        }
        for (int i = 0; i < count; i++) {
            if (md[i].contains(encoding)) {
                return true;
            }
        }
        return false;
    }
900344
public boolean isInDictionary(final CharSequence word) {
    try {
      byteBuffer = charSequenceToBytes(word);
    } catch (UnmappableInputException e) {
      return false;
    }

    // Try to find a partial match in the dictionary.
    final MatchResult match = matcher.match(matchResult, byteBuffer.array(), 0, byteBuffer.remaining(), rootNode);

    // Make sure the word doesn't contain a separator if there is an exact match
    if (containsSeparators && match.kind == EXACT_MATCH) {
      containsSeparators = false;
      for (int i=0; i<word.length(); i++) {
        if (word.charAt(i) == dictionaryMetadata.getSeparator()) {
          containsSeparators = true;
          break;
        }
      }
    }
    
    if (match.kind == EXACT_MATCH && !containsSeparators) {
      return true;
    }

    return containsSeparators && match.kind == SEQUENCE_IS_A_PREFIX && byteBuffer.remaining() > 0
        && fsa.getArc(match.node, dictionaryMetadata.getSeparator()) != 0;
  }
1191997
public static boolean containsWord(final String text, final String word) {

    if (text == null || word == null) {
      return false;
    }
    if (text.contains(word)) {
      Matcher matcher = matchText(text);
      for (; matcher.find();) {
        String matchedWord = matcher.group(0);
        if (matchedWord.equals(word)) {
          return true;
        }
      }
    }
    return false;
  }
1215734
private String detectBinaryTypes(byte[] bytes) {
		// NB: there are a lot of specific mimetypes we can detect by file
		// magic, but we don't want to make too much effort here.
		// "application/octet-stream" is fine if file looks very much like
		// binary but don't know what exactly. Returned value is not really
		// used beyond being non-null. We're only concerned with file formats
		// frequently sent out with imprecise Content-Type.
		switch (bytes[0]) {
		case (byte)0xFF:
			if (bytes[1] == (byte)0xFE) {
				// UTF-16LE BOM
				return null;
			} else if ((bytes[1] & 0xFE) == (byte)0xFA) {
				// audio/mp3
				return "audio/mp3";
			} else if (bytes[1] == (byte)0xD8) {
				// image/jpeg - commonly <FF><D8><FF><E0> or <FF><D8><FF><E1>
				return "image/jpeg";
			}
			// WordPerfect (<FF>WPC) falls in this category.
			// (WordPerfect also has <D8>WPC
			return BINARY_FILE;
		case (byte)0xFE:
			if (bytes[1] == (byte)0xFF) {
				// UTF-16BE BOM
				return null;
			}
			break;
		case (byte)0xF7:
			if (bytes[1] == 0x02 && bytes[2] == 0x01) {
				// unconfirmed.
				return "application/x-dvi";
			}
			break;
		case (byte)0xEF:
			if (bytes[1] == (byte)0xBB && bytes[2] == (byte)0xBF) {
				// UTF-8 BOM
				return null;
			}
			break;
		case (byte)0xD0:
			if (bytes[1] == (byte)0xCF && bytes[2] == 0x11 && bytes[2] == (byte)0xE1) {
				// MS Word.Document.8
				return BINARY_FILE;
			}
			break;
		case (byte)0xCA:
			if (bytes[1] == (byte)0xFE && bytes[2] == (byte)0xBA && bytes[3] == (byte)0xBE) {
				// application/java (class files)
				return "application/java";
			}
			break;
		case (byte)0x89:
			if (bytes[1] == 'P' && bytes[2] == 'N' && bytes[3] == 'G') {
				return "image/png";
			}
			break;
		case 0x00:
			if (bytes[1] == 0x00) {
				// Windows icon resource falls in this category.
				return BINARY_FILE;
			} else if (bytes[1] == 0x01) {
				// TTF?
				return BINARY_FILE;
			}
			break;
		case 0x01:
			if (bytes[1] == (byte)0xB3 || bytes[1] == (byte)0xBA) {
				return "video/mpeg";
			} else if (bytes[1] == 0x00) {
				// very likely is a binary file
				return BINARY_FILE;
			}
			break;
		case 0x1F:
			if (bytes[1] == (byte)0x8B) {
				return "application/x-gzip";
			} else if (bytes[1] == (byte)0x9D) {
				// followed by 0x90
				return "application/x-compress";
			}
			break;
		case '%':
			if (bytes[1] == 'P' && bytes[2] == 'D' && bytes[3] == 'F' && bytes[4] == '-') {
				return "application/pdf";
			} else if (bytes[1] == '!' && bytes[2] == 'P' && bytes[3] == 'S' && bytes[4] == '-') {
				// application/postscript, Type 1 font.
				return "application/postscript";
			}
			break;
		case 'B':
			if (bytes[1] == 'Z' && bytes[2] == 'h') {
				return "application/x-bzip2";
			}
			break;
		case 'F':
			if (bytes[1] == 'W' && bytes[2] == 'S') {
				// followed by <04> or <05>. Also 'CWS<07>?
				return "application/x-shockwave-flash";
			} else if (bytes[1] == 'L' && bytes[2] == 'V' && bytes[3] == 0x01) {
				return "video/x-flv";
			}
			break;
		case 'G':
			if (bytes[1] == 'I' && bytes[2] == 'F' && bytes[3] == '8') {
				// GIF87a or GIF89a
				return "image/gif";
			}
			break;
		case 'M':
			if (bytes[1] == 'Z') {
				// MS-DOS Executable (MZ <00-FF><00-01>)
				if (bytes[3] == 0x00 || bytes[3] == 0x01) {
					return "application/x-dosexec";
				}
			} else if (bytes[1] == 'S' && bytes[2] == 'C' && bytes[3] == 'F') {
				// MS cab file
				return "application/vnd.ms-cab-compressed";
			} else if (bytes[1] == 'T' && bytes[2] == 'h' && bytes[3] == 'd') {
				// MIDI
				return "audio/midi"; // or "application/x-midi"
			}
			break;
		case 'P':
			if (bytes[1] == 'K' && bytes[2] == 0x03 && bytes[3] == 0x04) {
				return "application/zip";
			} else if (bytes[1] == 'E' && bytes[2] == 0x00 && bytes[3] == 0x00 && bytes[4] == 'M' && bytes[5] == 'S') {
				// Windows PE
				return BINARY_FILE;
			}
			break;
		case 'm':
			if (bytes[1] == 'o' && bytes[2] == 'o' && bytes[3] == 'v') {
				return "video/quicktime";
			} else if (bytes[1] == 'd' && bytes[2] == 'a' && bytes[3] == 't') {
				return "video/quicktime";
			}
			break;
		case '{':
			if (bytes[1] == '\\' && bytes[2] == 'r' && bytes[3] == 't' && bytes[4] == 'f' && bytes[5] == '1') {
				return "application/rtf";
			}
			break;
		}

		if (bytes[2] == '-' && bytes[3] == 'l' && bytes[4] == 'h' && bytes[5] == '5' && bytes[6] == '-') {
			// LZH archive
			return BINARY_FILE;
		}
		// Other formats we may want to add
		// "RIFF" <?><?><?><?> "WAVEfmt " - "audio/wav"
		// <DB><A5><2D><00> - commonly with suffix .doc. files command doesn't know this format.
		// <C5><D0><D3><C6><1E><00><00><00> - some EPS has a binary header starting with this, before "%!PS-"
		return null;
	}
904418
private boolean htmlSentenceTerminatorFound(String str, int index) {
        for (int i = 0; i < sentenceTerminators.length; i++) {
            String terminator = sentenceTerminators[i];
            if (str.regionMatches(true, index, terminator,
                                  0, terminator.length())) {
                    return true;
            }
        }
        return false;
    }
375647
public static boolean is(String word){      
        if(word == null){
            return false;
        }
        word = word.trim();
        return isStopChar(word) || stopwords.contains(word);
    }
1260914
public boolean contains(String word) {
		if(word.length()>maxLen)
			return false;
		return dict.contains(word);
	}
1129191
function encodeBookWithPage(book, page) {
    var file = page.getFile();

    var result = encodeBook(book);
    result.page = encodePage(page, book.getSummary());
    result.file = encodeFile(file);

    return result;
}