Creates a chart for recognition of a sentence of length N
960226
function Chart(N) {
  logger.debug("Chart: " + N);
  this.N = N;
  this.outgoing_edges = new Array(N+1);
  this.incoming_edges = new Array(N+1);
  
  var i;
  for (i = 0; i <= N; i++) {
    this.outgoing_edges[i] = {};
    this.incoming_edges[i] = {};
  }
  
}
302799
def _get_wv(sentence, ignore=False):
    '''
    
    '''
    global _vectors
    vectors = []
    for y in sentence:
        y_ = any2unicode(y).strip()
        if y_ not in _stopwords:
            syns = nearby(y_)[0]
            # print("sentence %s word: %s" %(sentence, y_))
            # print("sentence %s word nearby: %s" %(sentence, " ".join(syns)))
            c = []
            try:
                c.append(_vectors.word_vec(y_))
            except KeyError as error:
                if ignore:
                    continue
                else:
                    logging.warning("not exist in w2v model: %s" % y_)
                    # c.append(np.zeros((100,), dtype=float))
                    random_state = np.random.RandomState(seed=(hash(y_) % (2**32 - 1)))
                    c.append(random_state.uniform(low=-10.0, high=10.0, size=(100,)))
            for n in syns:
                if n is None: continue
                try:
                    v = _vectors.word_vec(any2unicode(n))
                except KeyError as error:
                    # v = np.zeros((100,), dtype=float)
                    random_state = np.random.RandomState(seed=(hash(n) % (2 ** 32 - 1)))
                    v = random_state.uniform(low=10.0, high=10.0, size=(100,))
                c.append(v)
            r = np.average(c, axis=0)
            vectors.append(r)
    return vectors
691984
def pprint(string, token=[WORD, POS, CHUNK, PNP], column=4):
    
    if isinstance(string, basestring):
        print("\n\n".join([table(sentence, fill=column) for sentence in Text(string, token)]))
    if isinstance(string, Text):
        print("\n\n".join([table(sentence, fill=column) for sentence in string]))
    if isinstance(string, Sentence):
        print(table(string, fill=column))
561900
def generateFeatures(numFeatures):
  
  # Capital letters, lowercase letters, numbers
  candidates = ([chr(i+65) for i in xrange(26)] +
                [chr(i+97) for i in xrange(26)] +
                [chr(i+48) for i in xrange(10)])

  if numFeatures > len(candidates):
    candidates = ["F{}".format(i) for i in xrange(numFeatures)]
    return candidates

  return candidates[:numFeatures]
582998
def _get_span(self, m):
        
        return (m.sentence.id, m.char_start, m.char_end)
388464
def pad(lines, delim):
	
	"""
	Populates text into chunks. If the delim was & then
	['12 & 344', '344 & 8', '8 & 88'] would be stored in chunks as
	[['12', '344', '8'], ['344', '8', '88']]
	"""
	chunks = []
	for i in range(len(lines)):
		line = lines[i]
		sections = line.split(delim)
		for j in range(len(sections)):
			if len(chunks) <= j:
				chunks.append([])
			chunks[j].append(sections[j])

	"""
	Calculates & Stores the max length of chunks
	"""
	max_lengths = []
	for i in range(len(chunks)):
		_max = max([len(j) for j in chunks[i]])
		max_lengths.append(_max)

	"""
	Pads the children of chunks according to the chunk's max length" \
	"""
	for i in range(len(chunks)):
		for j in range(len(chunks[i])):
			chunks[i][j] += (max_lengths[i] - len(chunks[i][j])) * ' '
	new_lines = ['' for i in range(len(lines))]

	for i in range(len(chunks)):
		for j in range(len(chunks[i])):
			new_lines[j] += chunks[i][j]
	return new_lines
879318
def convert_examples_to_features(examples, seq_length, tokenizer):
    """"""

    features = []
    for (ex_index, example) in enumerate(examples):
        tokens_a = tokenizer.tokenize(example.text_a)

        tokens_b = None
        if example.text_b:
            tokens_b = tokenizer.tokenize(example.text_b)

        if tokens_b:
            # Modifies `tokens_a` and `tokens_b` in place so that the total
            # length is less than the specified length.
            # Account for [CLS], [SEP], [SEP] with "- 3"
            _truncate_seq_pair(tokens_a, tokens_b, seq_length - 3)
        else:
            # Account for [CLS] and [SEP] with "- 2"
            if len(tokens_a) > seq_length - 2:
                tokens_a = tokens_a[0:(seq_length - 2)]

        # The convention in BERT is:
        # (a) For sequence pairs:
        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
        #  type_ids:   0   0  0    0    0     0      0   0    1  1  1   1  1   1
        # (b) For single sequences:
        #  tokens:   [CLS] the dog is hairy . [SEP]
        #  type_ids:   0   0   0   0  0     0   0
        #
        # Where "type_ids" are used to indicate whether this is the first
        # sequence or the second sequence. The embedding vectors for `type=0` and
        # `type=1` were learned during pre-training and are added to the wordpiece
        # embedding vector (and position vector). This is not *strictly* necessary
        # since the [SEP] token unambigiously separates the sequences, but it makes
        # it easier for the model to learn the concept of sequences.
        #
        # For classification tasks, the first vector (corresponding to [CLS]) is
        # used as as the "sentence vector". Note that this only makes sense because
        # the entire model is fine-tuned.
        tokens = []
        input_type_ids = []
        tokens.append("[CLS]")
        input_type_ids.append(0)
        for token in tokens_a:
            tokens.append(token)
            input_type_ids.append(0)
        tokens.append("[SEP]")
        input_type_ids.append(0)

        if tokens_b:
            for token in tokens_b:
                tokens.append(token)
                input_type_ids.append(1)
            tokens.append("[SEP]")
            input_type_ids.append(1)

        input_ids = tokenizer.convert_tokens_to_ids(tokens)

        # The mask has 1 for real tokens and 0 for padding tokens. Only real
        # tokens are attended to.
        input_mask = [1] * len(input_ids)

        # Zero-pad up to the sequence length.
        while len(input_ids) < seq_length:
            input_ids.append(0)
            input_mask.append(0)
            input_type_ids.append(0)

        assert len(input_ids) == seq_length
        assert len(input_mask) == seq_length
        assert len(input_type_ids) == seq_length

        if ex_index < 5:
            logger.info("*** Example ***")
            logger.info("unique_id: %s" % (example.unique_id))
            logger.info("tokens: %s" % " ".join([str(x) for x in tokens]))
            logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
            logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask]))
            logger.info(
                "input_type_ids: %s" % " ".join([str(x) for x in input_type_ids]))

        features.append(
            InputFeatures(
                unique_id=example.unique_id,
                tokens=tokens,
                input_ids=input_ids,
                input_mask=input_mask,
                input_type_ids=input_type_ids))
    return features
967257
def feature_detector(self, tokens, index, history):
        

        word = tokens[index]
        if index == 0: # At the beginning of the sentence
            prevword = prevprevword = None
            prevtag = prevprevtag = None
            #word = word.lower() # Lowercase at the beginning of sentence
        elif index == 1:
            prevword = tokens[index-1] # Note: no lowercase
            prevprevword = None
            prevtag = history[index-1]
            prevprevtag = None
        else:
            prevword = tokens[index-1]
            prevprevword = tokens[index-2]
            prevtag = history[index-1]
            prevprevtag = history[index-2]

        if re.match('[0-9]+([\.,][0-9]*)?|[0-9]*[\.,][0-9]+$', word):
            # Included "," as decimal point
            shape = 'number'
        elif re.compile('\W+$', re.UNICODE).match(word):
            # Included unicode flag
            shape = 'punct'
        elif re.match('([A-ZÄÖÜ]+[a-zäöüß]*-?)+$', word):
            # Included dash for dashed words and umlauts
            shape = 'upcase'
        elif re.match('[a-zäöüß]+', word):
            # Included umlauts
            shape = 'downcase'
        elif re.compile("\w+", re.UNICODE).match(word):
            # Included unicode flag
            shape = 'mixedcase'
        else:
            shape = 'other'

        features = {
            'prevtag': prevtag,
            'prevprevtag': prevprevtag,
            'word': word,
            'word.lower': word.lower(),
            'suffix3': word.lower()[-3:],
            #'suffix2': word.lower()[-2:],
            #'suffix1': word.lower()[-1:],
            'preffix1': word[:1], # included
            'prevprevword': prevprevword,
            'prevword': prevword,
            'prevtag+word': '%s+%s' % (prevtag, word),
            'prevprevtag+word': '%s+%s' % (prevprevtag, word),
            'prevword+word': '%s+%s' % (prevword, word),
            'shape': shape
            }
        return features
738049
def p_statement_plot(p):
    
    p[0] = make_sentence('PLOT',
                         make_typecast(TYPE.ubyte, p[2], p.lineno(3)),
                         make_typecast(TYPE.ubyte, p[4], p.lineno(3)))
134343
def _create_matrix(self, sentences, threshold, tf_metrics, idf_metrics):
        
        # create matrix |sentences|×|sentences| filled with zeroes
        sentences_count = len(sentences)
        matrix = numpy.zeros((sentences_count, sentences_count))
        degrees = numpy.zeros((sentences_count, ))

        for row, (sentence1, tf1) in enumerate(zip(sentences, tf_metrics)):
            for col, (sentence2, tf2) in enumerate(zip(sentences, tf_metrics)):
                matrix[row, col] = self.cosine_similarity(sentence1, sentence2, tf1, tf2, idf_metrics)

                if matrix[row, col] > threshold:
                    matrix[row, col] = 1.0
                    degrees[row] += 1
                else:
                    matrix[row, col] = 0

        for row in range(sentences_count):
            for col in range(sentences_count):
                if degrees[row] == 0:
                    degrees[row] = 1

                matrix[row][col] = matrix[row][col] / degrees[row]

        return matrix
135232
def as_text(self, max_rows=0, sep=" | "):
        """"""
        if not max_rows or max_rows > self.num_rows:
            max_rows = self.num_rows
        omitted = max(0, self.num_rows - max_rows)
        labels = self._columns.keys()
        fmts = self._get_column_formatters(max_rows, False)
        rows = [[fmt(label, label=True) for fmt, label in zip(fmts, labels)]]
        for row in itertools.islice(self.rows, max_rows):
            rows.append([f(v, label=False) for v, f in zip(row, fmts)])
        lines = [sep.join(row) for row in rows]
        if omitted:
            lines.append('... ({} rows omitted)'.format(omitted))
        return '\n'.join([line.rstrip() for line in lines])
977729
def text2sentences(text, labels):
    '''
    
    '''
    sentence = ''
    for i, label in enumerate(labels):
        if label == '1':
            if sentence:
                yield sentence
            sentence = ''
        else:
            sentence += text[i]
    if sentence:
        yield sentence
247031
def _shape_text(self, text, colsep=u"\t", rowsep=u"\n",
                    transpose=False, skiprows=0, comments='#'):
        """"""
        assert colsep != rowsep
        out = []
        text_rows = text.split(rowsep)[skiprows:]
        for row in text_rows:
            stripped = to_text_string(row).strip()
            if len(stripped) == 0 or stripped.startswith(comments):
                continue
            line = to_text_string(row).split(colsep)
            line = [try_to_parse(to_text_string(x)) for x in line]
            out.append(line)
        # Replace missing elements with np.nan's or None's
        if programs.is_module_installed('numpy'):
            from numpy import nan
            out = list(zip_longest(*out, fillvalue=nan))
        else:
            out = list(zip_longest(*out, fillvalue=None))
        # Tranpose the last result to get the expected one
        out = [[r[col] for r in out] for col in range(len(out[0]))]
        if transpose:
            return [[r[col] for r in out] for col in range(len(out[0]))]
        return out
79514
def pretty_description(description, wrap_at=None, indent=0):
    
    if wrap_at is None or wrap_at < 0:
        width = console_width(default=79)
        if wrap_at is None:
            wrap_at = width
        else:
            wrap_at += width

    indent = ' ' * indent
    text_wrapper = textwrap.TextWrapper(
        width=wrap_at, replace_whitespace=False,
        initial_indent=indent, subsequent_indent=indent)
    new_desc = []
    for line in description.split('\n'):
        new_desc.append(line.replace('\n', '').strip())
    while not new_desc[0]:
        del new_desc[0]
    while not new_desc[-1]:
        del new_desc[-1]
    separators = [i for i, l in enumerate(new_desc) if not l]
    paragraphs = []
    if separators:
        start, end = 0, separators[0]
        paragraphs.append(new_desc[start:end])
        for i in range(len(separators) - 1):
            start = end + 1
            end = separators[i + 1]
            paragraphs.append(new_desc[start:end])
        paragraphs.append(new_desc[end + 1:])
        return '\n\n'.join(text_wrapper.fill(' '.join(p)) for p in paragraphs)
    return text_wrapper.fill(' '.join(new_desc))
975427
def _get_ckw_span(fulltext, spans):
    """"""
    _MAXIMUM_SEPARATOR_LENGTH = max(
        [len(_separator)
         for _separator in
         current_app.config["CLASSIFIER_VALID_SEPARATORS"]]
    )
    if spans[0] < spans[1]:
        words = (spans[0], spans[1])
        dist = spans[1][0] - spans[0][1]
    else:
        words = (spans[1], spans[0])
        dist = spans[0][0] - spans[1][1]

    if dist == 0:
        # Two keywords are adjacent. We have a match.
        return (min(words[0] + words[1]),
                max(words[0] + words[1]))  # FIXME: huh, this is a bug?! a sum?
    elif dist <= _MAXIMUM_SEPARATOR_LENGTH:
        separator = fulltext[words[0][1]:words[1][0] + 1]
        # Check the separator.
        if separator.strip() in current_app.config[
                "CLASSIFIER_VALID_SEPARATORS"]:
            return (min(words[0] + words[1]), max(words[0] + words[1]))

    # There is no inclusion.
    return None
1132508
function displayMetrics(file) {
    var prefix = '';

    var code = fs.readFileSync(file, 'utf-8');

    // We remove any shebang at the start of the file since that isn't valid
    // Javascript and will trip up the parser.

    code = code.replace(/^#!.*\n/, '');

    var metrics = complexity(code);

    // Walk the metrics structure and add each member to the table rows.

    function walk(data, name) {
        // Fade out anonymous functions.

        if (name.indexOf('anon@') === 0) {
            name = name.grey;
        }

        rows.push([ prefix + name, data.ecc, data.arity, data.codeLines, data.commentLines, Math.round(100 * data.commentLines / data.codeLines) ]);

        // Add two spaces to the prefix before the next depth, to illustrate
        // the hierarchy in the table.

        prefix += '  ';
        _.each(data.children, walk);
        prefix = prefix.slice(0,prefix.length-2);
    }
    walk(metrics, path.basename(file).blue.bold);
}
137796
def _render_corpus_row(self, label, ngrams):
        """"""
        row = ('<tr>\n<td>{label}</td>\n<td>{first}</td>\n<td>{only}</td>\n'
               '<td>{last}</td>\n</tr>')
        cell_data = {'label': label}
        for period in ('first', 'only', 'last'):
            cell_data[period] = ', '.join(ngrams[label][period])
        return row.format(**cell_data)