Normalize parsed types in a comment line. If one or more types in the line contains alternatives,
this routine generate a list of 'overloaded' normalized signatures.
335438
List<List<MessageType>> normalizeTypes(int idx, List<MessageType> msgTypes) {
        if (msgTypes.size() == idx) return Collections.singletonList(Collections.emptyList());
        MessageType head = msgTypes.get(idx);
        List<List<MessageType>> buf = new ArrayList<>();
        for (MessageType alternative : head.accept(normalizeVisitor, null)) {
            for (List<MessageType> rest : normalizeTypes(idx + 1, msgTypes)) {
                List<MessageType> temp = new ArrayList<>(rest);
                temp.add(0, alternative);
                buf.add(temp);
            }
        }
        return buf;
    }
197577
def type_of_line(line):
    """"""

    if line_is_comment(line):
        return "comment"
    elif line_is_executable(line):
        return "exec_stmt"
    elif line_is_pgm_unit_end(line):
        return "pgm_unit_end"
    else:
        if line_is_pgm_unit_start(line):
            return "pgm_unit_start"
        else:
            return "other"
1253859
def normalize_type(self, names):
        """"""
        self.name = names[0]
        self.is_kwarg = False
        self.is_arg = False
        self.names = []

        try:
            # http://stackoverflow.com/a/16488383/5006 uses ask forgiveness because
            # of py2/3 differences of integer check
            self.index = int(self.name)
            self.name = ""
            self.is_arg = True

        except ValueError:
            self.is_kwarg = True
            self.names = names
472359
def comment_limiters(self, value):
        

        if value is not None:
            assert type(value) in (tuple, list), "'{0}' attribute: '{1}' type is not 'tuple' or 'list'!".format(
                "comment_limiters", value)
            for element in value:
                assert type(element) is unicode, "'{0}' attribute: '{1}' type is not 'unicode'!".format(
                    "comment_limiters", element)
        self.__comment_limiters = value
892411
def normalize(self):
        
        corefs = []
        types = []
        avms = []
        for term in self._terms:
            if isinstance(term, TypeTerm):
                types.append(term)
            elif isinstance(term, AVM):
                term.normalize()
                avms.append(term)
            elif isinstance(term, Coreference):
                corefs.append(term)
            else:
                raise TdlError('unexpected term {}'.format(term))
        self._terms = corefs + types + avms
533689
def _filter(self, text, context, encoding):
        """"""

        docstrings = []
        strings = []
        comments = []
        prev_token_type = tokenize.NEWLINE
        indent = ''
        name = None
        stack = [(context, 0, self.MODULE)]
        last_comment = False
        possible_fmt_str = None

        src = io.StringIO(text)

        for token in tokenizer(src.readline):
            token_type = token[0]
            value = token[1]
            line = str(token[2][0])
            line_num = token[2][0]

            value = token[1]
            line = str(token[2][0])

            # Track function and class ancestry
            if token_type == tokenize.NAME:
                possible_fmt_str = None
                if value in ('def', 'class'):
                    name = value
                elif name:
                    parent = stack[-1][2]
                    prefix = ''
                    if parent != self.MODULE:
                        prefix = '.' if parent == self.CLASS else ', '
                    if name == 'class':
                        stack.append(('%s%s' % (prefix, value), len(indent), self.CLASS))
                    elif name == 'def':
                        stack.append(('%s%s()' % (prefix, value), len(indent), self.FUNCTION))
                    name = None
                elif not F_SUPPORT and value in FMT_STR:
                    possible_fmt_str = (prev_token_type, value)
            elif token_type != tokenize.STRING:
                possible_fmt_str = None

            if token_type == tokenize.COMMENT and self.comments:
                # Capture comments
                if (
                    self.group_comments and
                    last_comment and
                    comments and (comments[-1][2] + 1) == line_num
                ):
                    # Group multiple consecutive comments
                    comments[-1][0] += '\n' + value[1:]
                    comments[-1][2] = line_num
                else:
                    if len(stack) > 1:
                        loc = "%s(%s): %s" % (stack[0][0], line, ''.join([crumb[0] for crumb in stack[1:]]))
                    else:
                        loc = "%s(%s)" % (stack[0][0], line)
                    comments.append([value[1:], loc, line_num])
            if token_type == tokenize.STRING:
                # Capture docstrings.
                # If we captured an `INDENT` or `NEWLINE` previously we probably have a docstring.
                # `NL` means end of line, but not the end of the Python code line (line continuation).
                if (
                    (prev_token_type in PREV_DOC_TOKENS) or
                    (possible_fmt_str and possible_fmt_str[0] in PREV_DOC_TOKENS)
                ):
                    if self.docstrings:
                        value = value.strip()
                        if possible_fmt_str and value.startswith(("'", "\"")):
                            value = possible_fmt_str[1] + value
                        string, is_bytes = self.process_strings(value, docstrings=True)
                        if string:
                            loc = "%s(%s): %s" % (stack[0][0], line, ''.join([crumb[0] for crumb in stack[1:]]))
                            docstrings.append(
                                filters.SourceText(string, loc, 'utf-8', 'py-docstring')
                            )
                elif self.strings:
                    value = value.strip()
                    if possible_fmt_str and value.startswith(("'", "\"")):
                        value = possible_fmt_str[1] + value
                    string, is_bytes = self.process_strings(value)
                    if string:
                        loc = "%s(%s): %s" % (stack[0][0], line, ''.join([crumb[0] for crumb in stack[1:]]))
                        strings.append(filters.SourceText(string, loc, 'utf-8', 'py-string'))

            if token_type == tokenize.INDENT:
                indent = value
            elif token_type == tokenize.DEDENT:
                indent = indent[:-4]
                if len(stack) > 1 and len(indent) <= stack[-1][1]:
                    stack.pop()

            # We purposefully avoid storing comments as comments can come before docstrings,
            # and that can mess up our logic. So if the token is a comment we won't track it,
            # and comments are always followed with `NL` so we ignore that as well.
            # We only care that docstrings are preceded by `NEWLINE`.
            if token_type != tokenize.COMMENT and not (last_comment and token_type == tokenize.NL):
                prev_token_type = token_type
                last_comment = False
            else:
                last_comment = True

        final_comments = []
        for comment in comments:
            final_comments.append(filters.SourceText(textwrap.dedent(comment[0]), comment[1], encoding, 'py-comment'))

        return docstrings + final_comments + strings
249571
static String normalizeResponseWrapper(final String type) {
        if (!getTypeParameters(type).isEmpty() && isAssignableTo(type, Types.GENERIC_ENTITY))
            return getTypeParameters(type).get(0);
        return type;
    }
137049
def normalise(self, string, line_num):
		
		stripped = string.strip()
		if stripped != string:
			self.strip_errors.append(line_num)

		nfc_pos = [index
					for index, char in enumerate(stripped)
					if char in self.nfc_chars]

		parts = []
		start_pos = 0

		for pos in nfc_pos:
			if pos > 0:
				parts.append(self.norm_f(stripped[start_pos:pos]))

			parts.append(stripped[pos])
			start_pos = pos + 1

		if start_pos < len(stripped):
			parts.append(self.norm_f(stripped[start_pos:]))

		norm = ''.join(parts)

		if norm != stripped:
			self.norm_errors.append(line_num)

		return norm
424912
def normalize_text(input_text,
                   dictionary_mode='ipadic',
                   new_line_replaced='。',
                   is_replace_eos=True,
                   is_kana=True,
                   is_ascii=True,
                   is_digit=True):
    # type: (text_type,text_type,text_type,bool,bool,bool,bool)->text_type
    
    if is_replace_eos:
        without_new_line = input_text.replace('\n', new_line_replaced)
    else:
        without_new_line = new_line_replaced

    if dictionary_mode=='neologd' and is_neologdn_valid:
        return neologdn.normalize(normalize_text_normal_ipadic(without_new_line))
    elif dictionary_mode=='neologd' and is_neologdn_valid == False:
        raise Exception("You could not call neologd dictionary bacause you do NOT install the package neologdn.")
    else:
        return normalize_text_normal_ipadic(without_new_line, kana=is_kana, ascii=is_ascii, digit=is_digit)
1072916
def combine_comments(comments):
    '''
    
    '''
    if isinstance(comments, list):
        for idx in range(len(comments)):
            if not isinstance(comments[idx], six.string_types):
                comments[idx] = six.text_type(comments[idx])
    else:
        if not isinstance(comments, six.string_types):
            comments = [six.text_type(comments)]
        else:
            comments = [comments]
    return ' '.join(comments).strip()
853046
def parse_signature(self, statement, element, module=None):
        
        #If the signature changes, the user might not have had a chance to add the
        #detailed member information for it yet. Here
        #we will just update the modifiers and attributes. Also, since all the mods
        #etc. will be overwritten, we don't need to handle replace separately.
        smatch = self.RE_SIG.match(statement)
        result = (None, None, None)
        eresult = None

        if smatch is not None:
            name = smatch.group("name").strip()
            modifiers = smatch.group("modifiers") or []
            codetype = smatch.group("codetype")

            #If the exec is a function, we also may have a type and kind specified.
            if codetype.lower() == "function":
                dtype = smatch.group("type")
                kind = smatch.group("kind")
                if module is None:
                    element.update(name, modifiers, dtype, kind)
                else:
                    eresult = Function(name, modifiers, dtype, kind, module)
            else:
                if module is None:
                    element.update(name, modifiers)
                else:
                    eresult = Subroutine(name, modifiers, module)

            #The parameter sets are actually driven by the body of the executable 
            #rather than the call signature. However, the declarations will be
            #interpreted as members if we don't add the parameters to the ordered
            #list of parameter names. Overwrite that list with the new names.
            params = re.split("[\s,]+", smatch.group("parameters").lower())
            if eresult is None:
                element.paramorder = params
            else:
                eresult.paramorder = params

            result = (eresult, smatch.start(), smatch.end())

        return result
1152964
def _search_type_in_type_comment(self, code):
        
        for p in self.PEP0484_TYPE_COMMENT_PATTERNS:
            match = p.search(code)
            if match:
                return [match.group(1)]
856936
def _pythonized_comments(tokens):
    
    is_after_colon = True
    for token_type, token_text in tokens:
        if is_after_colon and (token_type in pygments.token.String):
            token_type = pygments.token.Comment
        elif token_text == ':':
            is_after_colon = True
        elif token_type not in pygments.token.Comment:
            is_whitespace = len(token_text.rstrip(' \f\n\r\t')) == 0
            if not is_whitespace:
                is_after_colon = False
        yield token_type, token_text
1215653
def normalize_unicode(text):
    
    if isinstance(text, six.text_type):
        return unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf8')
    else:
        return text
789025
def _ignore_comments(lines_enum):
    
    for line_number, line in lines_enum:
        line = COMMENT_RE.sub('', line)
        line = line.strip()
        if line:
            yield line_number, line
903335
def stylize_comment_block(lines):
  
  normal, sep, in_code = range(3)
  state = normal
  for line in lines:
    indented = line.startswith('    ')
    empty_line = line.strip() == ''

    if state == normal and empty_line:
      state = sep
    elif state in [sep, normal] and indented:
      yield ''
      if indented:
        yield '.. code-block:: javascript'
        yield ''
        yield line
        state = in_code
      else:
        state = normal
    elif state == sep and not empty_line:
      yield ''
      yield line
      state = normal
    else:
      yield line
      if state == in_code and not (indented or empty_line):
        sep = normal
403138
function(source) {

    // Remove comment markup from comments
    var comment = source.split(/\/\/[ ]?/).join('').split('/*').join('').split('*/').join(''),
      additionalKssParams = {},
      _this = this;

    comment = standardizeLinebreaks(comment);
    comment = trimLinebreaks(comment);

    comment.split('\n\n').forEach(function(markUpBlock) {

      var varName = markUpBlock.match(/([^:^\n]*):[\s\S]*\n/);
      if (varName && varName[1] !== undefined) {
        varName = varName[1].trim();
      }
      if (varName && varName.substring(0, 3) === 'sg-') {
        additionalKssParams[varName] = _this.getValue(varName, markUpBlock);
      }

    });

    return additionalKssParams;
  }