This is used by the GUI when adding a sentence to LHS or RHS.
@return
199869
public DSLSentence copy() {
        final DSLSentence copy = new DSLSentence();

        copy.drl = getDrl();
        copy.definition = getDefinition();
        copy.values = mapCopy(getValues());

        return copy;
    }
377736
public List<String> getSpannedWords() {
    if (isTerminal()) {
      return words;
    } else {
      List<String> words = Lists.newArrayList();
      words.addAll(left.getSpannedWords());
      words.addAll(right.getSpannedWords());
      return words;
    }
  }
55637
public AnalyzedSentence copy(AnalyzedSentence sentence) {
    AnalyzedTokenReadings[] copyTokens = new AnalyzedTokenReadings[sentence.getTokens().length];
    for (int i = 0; i < copyTokens.length; i++) {
      AnalyzedTokenReadings analyzedTokens = sentence.getTokens()[i];
      copyTokens[i] = new AnalyzedTokenReadings(analyzedTokens.getReadings(), analyzedTokens.getStartPos());
      copyTokens[i].setHistoricalAnnotations(analyzedTokens.getHistoricalAnnotations());
      copyTokens[i].setChunkTags(analyzedTokens.getChunkTags());
      if (analyzedTokens.isImmunized()) {
        copyTokens[i].immunize();
      }
      if (analyzedTokens.isIgnoredBySpeller()) {
        copyTokens[i].ignoreSpelling();
      }
      copyTokens[i].setWhitespaceBefore(sentence.getTokens()[i].isWhitespaceBefore());
    }
    return new AnalyzedSentence(copyTokens, sentence.whPositions, sentence.getTokensWithoutWhitespace(), sentence.getPreDisambigTokensWithoutWhitespace());
  }
3584
private Sentence constructSentence(List<Token> tokens) throws IOException {
        Sentence sentence;
        try {
            sentence = new SimpleSentence(tokens, strict);
        } catch (IllegalArgumentException e) {
            throw new IOException(e.getMessage());
        }
        return sentence;
    }
1325746
public String text2() {
        StringBuilder text = new StringBuilder();
        for (Change aDiff : getChangeList()) {
            if (aDiff.operation != Operation.DELETE) {
                text.append(aDiff.text);
            }
        }
        return text.toString();
    }
1039348
public String text() {
        String previousText = null;
        StringBuilder buffer = null;
        for (Object child : this) {
            String text = null;
            if (child instanceof String) {
                text = (String) child;
            } else if (child instanceof Node) {
                text = ((Node) child).text();
            }
            if (text != null) {
                if (previousText == null) {
                    previousText = text;
                } else {
                    if (buffer == null) {
                        buffer = new StringBuilder();
                        buffer.append(previousText);
                    }
                    buffer.append(text);
                }
            }
        }
        if (buffer != null) {
            return buffer.toString();
        }
        if (previousText != null) {
            return previousText;
        }
        return "";
    }
307472
private JCheckBox getChkParseRobotsTxt() {
		if (parseRobotsTxt == null) {
			parseRobotsTxt = new JCheckBox();
			parseRobotsTxt.setText(Constant.messages.getString("spider.options.label.robotstxt"));
		}
		return parseRobotsTxt;
	}
1225621
public Grammar getFragmentGrammar() {
		/*
		 * Fragment Content
		 */
		Grammar builtInFragmentContentGrammar = new BuiltInFragmentContent();

		/*
		 * Fragment
		 */
		fragmentGrammar = new Fragment("Fragment");
		fragmentGrammar.addProduction(new StartDocument(),
				builtInFragmentContentGrammar);

		return fragmentGrammar;
	}
311408
public Paragraph paragraph() {
        Paragraph paragraph = new Paragraph(before.getLeading());
        paragraph.add(before);
        if (numbered) {
            paragraph.addSpecial(new Chunk(String.valueOf(pageN), before.getFont()));
        }
        if (after != null) {
            paragraph.addSpecial(after);
        }
        paragraph.setAlignment(alignment);
        return paragraph;
    }
1021771
public String tokenizeToText() {

    final StringBuilder sb = new StringBuilder();
    if (isNoSeg) {
      String[] sentences = text.toArray(new String[text.size()]);
      final List<List<Token>> tokens = tokenizer.tokenize(sentences);
      for (final List<Token> tokSentence : tokens) {
        for (final Token tok : tokSentence) {
          String tokenValue = tok.getTokenValue();
          sb.append(tokenValue.trim()).append(DELIMITER);
        }
        sb.append(LINE_BREAK);
      }
    } else {
      final String[] sentences = segmenter.segmentSentence();
      final List<List<Token>> tokens = tokenizer.tokenize(sentences);
      for (final List<Token> tokSentence : tokens) {
        for (final Token token : tokSentence) {
          String tokenValue = token.getTokenValue();
          if (tokenValue.equals(RuleBasedSegmenter.PARAGRAPH)) {
            sb.append(DEFAULT_TOKEN_VALUE).append(LINE_BREAK);
          } else {
            sb.append(tokenValue.trim()).append(DELIMITER);
          }
        }
        sb.append(LINE_BREAK);
      }
    }
    return sb.toString().trim();
  }
710561
public Set<String> getPureKeywords() {
		Set<String> kws = this.pureSarlKeywords == null ? null : this.pureSarlKeywords.get();
		if (kws == null) {
			kws = new HashSet<>();
			kws.add(getAsKeyword());
			kws.add(getRequiresKeyword());
			kws.add(getWithKeyword());
			kws.add(getItKeyword());
			kws.add(getArtifactKeyword());
			kws.add(getAnnotationKeyword());
			kws.add(getSpaceKeyword());
			kws.add(getOnKeyword());
			kws.add(getCapacityKeyword());
			kws.add(getEventKeyword());
			kws.add(getVarKeyword());
			kws.add(getAgentKeyword());
			kws.add(getDispatchKeyword());
			kws.add(getUsesKeyword());
			kws.add(getSkillKeyword());
			kws.add(getDefKeyword());
			kws.add(getFiresKeyword());
			kws.add(getOverrideKeyword());
			kws.add(getIsStaticAssumeKeyword());
			kws.add(getBehaviorKeyword());
			kws.add(getExtensionExtensionKeyword());
			kws.add(getValKeyword());
			kws.add(getTypeofKeyword());
			kws.add(getOccurrenceKeyword());
			this.pureSarlKeywords = new SoftReference<>(kws);
		}
		return Collections.unmodifiableSet(kws);
	}
914205
public HString head() {
      return tokens()
                .stream()
                .filter(t -> t
                                .parent()
                                .isEmpty())
                .map(Cast::<HString>as)
                .findFirst()
                .orElseGet(() -> tokens()
                                    .stream()
                                    .filter(t -> !this.overlaps(t.parent()))
                                    .map(Cast::<HString>as)
                                    .findFirst()
                                    .orElse(this));
   }
200092
private void processLeaf(Leaf leaf, List<String> sentence, List<Span> names) {

    if (leaf != null && leftContractionPart == null) {

      int startOfNamedEntity = -1;
      String leafTag = leaf.getSecondaryTag();

      if (leafTag != null) {
        if (leafTag.contains("<sam->")) {
          String[] lexemes = underlinePattern.split(leaf.getLexeme());
          if (lexemes.length > 1) {
            for (int i = 0; i < lexemes.length - 1; i++) {
              sentence.add(lexemes[i]);
              
              String[] conts = ContractionUtility.expand(lexemes[i]);
              if(conts != null) {
                int end = sentence.size();
                int start = end - 1;
                Span s = new Span(start, end, "default");
                names.add(s);
                {
                  Span[] ss = {s};
                  System.out.println(Arrays.toString(Span.spansToStrings(ss, sentence.toArray(new String[sentence.size()]))));
                }
              }
            }
          }
          leftContractionPart = lexemes[lexemes.length - 1];
          return;
        }
        // if (leaf.getLexeme().contains("_") && leaf.getLexeme().length() > 3)
        // {
        // String tag = leaf.getFunctionalTag();
        // if (tags != null) {
        // if (tags.contains(tag)) {
        // namedEntityTag = leaf.getFunctionalTag();
        // }
        // } else {
        // namedEntityTag = leaf.getFunctionalTag();
        // }
        // }
      }

      // if (contraction) {
      // startOfNamedEntity = sentence.size();
      // }
      //
      sentence.addAll(Arrays.asList(leaf.getLexeme()));// .split("_")
      //
      // if (contraction) {
      // names
      // .add(new Span(startOfNamedEntity, sentence.size()));
      // }

    } else {
      // will handle the contraction
      String tag = leaf.getSecondaryTag();
      String right = leaf.getLexeme();
      if (tag != null && tag.contains("<-sam>")) {
        String[] parts = underlinePattern.split(leaf.getLexeme());
        if(parts != null) {
          // try to join only the first
          String c = PortugueseContractionUtility.toContraction(
              leftContractionPart, parts[0]);

          if (c != null) {
            sentence.add(c);
            names.add(new Span(sentence.size() - 1, sentence.size(), "default"));
          }
          
          for (int i = 1; i < parts.length; i++) {
            sentence.add(parts[i]);
          }
        } else {
          right = leaf.getLexeme();
          String c = PortugueseContractionUtility.toContraction(
              leftContractionPart, right);

          if (c != null) {
            sentence.add(c);
            names.add(new Span(sentence.size() - 1, sentence.size(), "default"));
          } else {
            System.err.println("missing " + leftContractionPart + " + " + right);
            sentence.add(leftContractionPart);
            sentence.add(right);
          }
        }
      } else {
        System.err.println("unmatch" + leftContractionPart + " + " + right);
      }
      leftContractionPart = null;
    }

  }
1024967
public List<Tokenizer.Token> tokenize(String sentence) {
      Sequence seq = Sequence.create(Chars.asList(sentence.toCharArray())
                                          .stream()
                                          .map(Object::toString));
      Labeling labels = model.label(seq);

      int n = sentence.length();
      int[][] spans = new int[n + 1][2];
      double[] best = new double[n + 1];
      best[0] = 1d;

      for (int i = 1; i <= n; i++) {
         for (int j = Math.max(0, i - maxSpanSize); j < i; j++) {
            int w = i - j;
            String span = sentence.substring(j, j + w);
            double score = scoreWord(j, j + w, span, labels);
            double segmentScore = multi(best[i - w], score);
            logFiner("'{0}' > '{1}' : {2} : {3}",
                     sentence.substring(spans[i - w][0], spans[i - w][1]),
                     span,
                     segmentScore,
                     best[i]);
            if (segmentScore >= best[i]) {
               best[i] = segmentScore;
               spans[i][0] = j;
               spans[i][1] = j + w;
            }
         }
      }
      int i = n;
      LinkedList<Tokenizer.Token> tokens = new LinkedList<>();
      while (i > 0) {
         String token = StringUtils.trim(sentence.substring(spans[i][0], spans[i][1]));
         if (!StringUtils.isNullOrBlank(token)) {
            tokens.addFirst(
               new Tokenizer.Token(token, TokenType.UNKNOWN, spans[i][0], spans[i][0] + token.length(), 0));
         }
         i = i - (spans[i][1] - spans[i][0]);
      }
      i = 0;
      for (Tokenizer.Token token : tokens) {
         token.index = i;
         i++;
      }

      return tokens;
   }
784527
@Override
  public IRTerminalStatement getLeastSignificantTerminalStatement()
  {
    if( _defaultStatements.isEmpty())
    {
      return null;
    }
    IRContinueStatement caseStmtContinue = null;
    if( _cases != null )
    {
      outer:
      for( int i = 0; i < _cases.size(); i++ )
      {
        List caseStatements = _cases.get(i).getStatements();
        if( caseStatements != null && caseStatements.size() > 0 )
        {
          for( int iStmt = 0; iStmt < caseStatements.size(); iStmt++ )
          {
            IRStatement statement = (IRStatement) caseStatements.get(iStmt);
            // Note that the statement can be null if it's just a semicolon
            IRTerminalStatement terminalStmt = (statement == null ? null : statement.getLeastSignificantTerminalStatement());
            if( terminalStmt != null && !(terminalStmt instanceof IRBreakStatement) )
            {
              if( terminalStmt instanceof IRContinueStatement )
              {
                caseStmtContinue = (IRContinueStatement)terminalStmt;
              }
              continue outer;
            }
          }
          return null;
        }
      }
    }
    for( int i = 0; i < _defaultStatements.size(); i++ )
    {
      IRTerminalStatement terminalStmt = _defaultStatements.get( i ).getLeastSignificantTerminalStatement();
      if( terminalStmt != null && !(terminalStmt instanceof IRBreakStatement) )
      {
        return caseStmtContinue != null ? caseStmtContinue : terminalStmt;
      }
    }
    return null;
  }
1093866
public TextImpl replaceWithText() {
        TextImpl replacement = new TextImpl(document, getData());
        parent.insertBefore(replacement, this);
        parent.removeChild(this);
        return replacement;
    }
910184
public ExampleSet copy()
	{
		ExampleSet copy = new SentenceSetCopy();
		for (int i=0;i<size();i++)
			copy.add(x(i), y(i), id(i));

		return copy;
	}