Short code parser
198358
public function includeChildren($args)
    {
        if (isset($args['Folder'])) {
            $children = $this->getManifest()->getChildrenFor(
                Controller::join_links(dirname($this->record->getPath()), $args['Folder'])
            );
        } else {
            $children = $this->getManifest()->getChildrenFor(
                dirname($this->record->getPath())
            );
        }

        if (isset($args['Exclude'])) {
            $exclude = explode(',', $args['Exclude']);

            foreach ($children as $k => $child) {
                foreach ($exclude as $e) {
                    if ($child->Link == Controller::join_links($this->record->Link(), strtolower($e), '/')) {
                        unset($children[$k]);
                    }
                }
            }
        }

        return $this->customise(
            new ArrayData(
                array(
                    'Children' => $children
                )
            )
        )->renderWith('Includes/DocumentationPages');
    }
1106373
function subParse(parser, pos, extensions,parens) {
    var p = new parser.constructor(parser.options, parser.input, pos);
    if (extensions)
        for (var k in extensions)
            p[k] = extensions[k] ;

    var src = parser ;
    var dest = p ;
    ['inFunction','inAsyncFunction','inAsync','inGenerator','inModule'].forEach(function(k){
        if (k in src)
            dest[k] = src[k] ;
    }) ;
    if (parens)
        p.options.preserveParens = true ;
    p.nextToken();
    return p;
}
1008723
def structparser(token):
    """"""
    m = STRUCT_PACK_RE.match(token)
    if not m:
        return [token]
    else:
        endian = m.group('endian')
        if endian is None:
            return [token]
        # Split the format string into a list of 'q', '4h' etc.
        formatlist = re.findall(STRUCT_SPLIT_RE, m.group('fmt'))
        # Now deal with mulitiplicative factors, 4h -> hhhh etc.
        fmt = ''.join([f[-1] * int(f[:-1]) if len(f) != 1 else
                       f for f in formatlist])
        if endian == '@':
            # Native endianness
            if byteorder == 'little':
                endian = '<'
            else:
                assert byteorder == 'big'
                endian = '>'
        if endian == '<':
            tokens = [REPLACEMENTS_LE[c] for c in fmt]
        else:
            assert endian == '>'
            tokens = [REPLACEMENTS_BE[c] for c in fmt]
    return tokens
723852
function createCodeReader(text) {
		var index = 0,
			line = 1,
			column = 1,
			length,
			EOF = undefined,
			currentChar,
			nextReadBeginsLine;

		text = text.replace(/\r\n/g, "\n").replace(/\r/g, "\n"); //normalize line endings to unix

		length = text.length;
		currentChar = length > 0 ? text.charAt(0) : EOF;

		function getCharacters(count) {
			var value;
			if (count === 0) {
				return "";
			}

			count = count || 1;

			value = text.substring(index + 1, index + count + 1);
			return value === "" ? EOF : value;
		}

		return {
			toString: function() {
				return "length: " + length + ", index: " + index + ", line: " + line + ", column: " + column + ", current: [" + currentChar + "]";
			},

			peek: function(count) {
				return getCharacters(count);
			},

			substring: function() {
				return text.substring(index);
			},

			peekSubstring: function() {
				return text.substring(index + 1);
			},

			read: function(count) {
				var value = getCharacters(count),
					newlineCount,
					lastChar;

				if (value === "") {
					//this is a result of reading/peeking/doing nothing
					return value;
				}

				if (value !== EOF) {
					//advance index
					index += value.length;
					column += value.length;

					//update line count
					if (nextReadBeginsLine) {
						line++;
						column = 1;
						nextReadBeginsLine = false;
					}

					newlineCount = value.substring(0, value.length - 1).replace(/[^\n]/g, "").length;
					if (newlineCount > 0) {
						line += newlineCount;
						column = 1;
					}

					lastChar = last(value);
					if (lastChar === "\n") {
						nextReadBeginsLine = true;
					}

					currentChar = lastChar;
				} else {
					index = length;
					currentChar = EOF;
				}

				return value;
			},

			text: function() { return text; },

			getLine: function() { return line; },
			getColumn: function() { return column; },
			isEof: function() { return index >= length; },
			isSol: function() { return column === 1; },
			isSolWs: function() {
				var temp = index,
					c;
				if (column === 1) {
					return true;
				}

				//look backward until we find a newline or a non-whitespace character
				while ((c = text.charAt(--temp)) !== "") {
					if (c === "\n") {
						return true;
					}
					if (!/\s/.test(c)) {
						return false;
					}
				}

				return true;
			},
			isEol: function() { return nextReadBeginsLine; },
			EOF: EOF,
			current: function() { return currentChar; }
		};
	}
1319163
function parseLength (parser) {
  const length = parser.buffer.length - 1
  var offset = parser.offset
  var number = 0

  while (offset < length) {
    const c1 = parser.buffer[offset++]
    if (c1 === 13) {
      parser.offset = offset + 1
      return number
    }
    number = (number * 10) + (c1 - 48)
  }
}
1334712
def parse_subprogram(subprogram, params, comment, code, 
                         before_contains=nil, function=nil, prefix=nil)
      subprogram.singleton = false
      prefix = "" if !prefix
      arguments = params.sub(/\(/, "").sub(/\)/, "").split(",") if params
      args_comment, params_opt = 
        find_arguments(arguments, code.sub(/^s*?contains\s*?(!.*?)?$.*/im, ""),
                       nil, nil, true)
      params_opt = "( " + params_opt + " ) " if params_opt
      subprogram.params = params_opt || ""
      namelist_comment = find_namelists(code, before_contains)

      block_comment = find_comments comment
      if function
        subprogram.comment = "<b><em> Function </em></b> :: <em>#{prefix}</em>\n"
      else
        subprogram.comment = "<b><em> Subroutine </em></b> :: <em>#{prefix}</em>\n"
      end
      subprogram.comment << args_comment if args_comment
      subprogram.comment << block_comment if block_comment
      subprogram.comment << namelist_comment if namelist_comment

      # For output source code
      subprogram.start_collecting_tokens
      subprogram.add_token Token.new(1,1).set_text(code)

      subprogram
    end
736148
function(fn) {
  var code = fn.toString();
  var match = code.match(/^(?:(?:function)?[\s]*\(([^)]*)\)|([^\=\(]+))/);
  var args = (match[1] || match[2]);
  if (typeof args === 'undefined') {
    return [];
  }
  return args.split(',').map(function(item) {
    return item.trim();
  }).filter(function(item) {
    return item;
  });
}
572696
def parse_code_ul(self, url, ul):
        """"""
        li_list = ul.find_all('li', recursive=False)
        li = li_list[0]
        span_title = li.find('span',
                             attrs={'class': re.compile(r'TM\d+Code')},
                             recursive=False)

        section = Section(span_title.attrs['id'], span_title.text.strip())
        div_italic = li.find('div', attrs={'class': 'italic'}, recursive=False)
        if div_italic:
            section.content = div_italic.text.strip()
        span_link = li.find('span',
                            attrs={'class': 'codeLienArt'},
                            recursive=False)
        if span_link:
            a_link = span_link.find('a', recursive=False)
            if self.with_articles:
                service = self.section_service
                section.articles = service.articles(self.id_code,
                                                    section.id_section,
                                                    self.date_pub)
            else:
                section.articles = a_link.text.strip()
            section.url_section = cleanup_url(
                urljoin(url, a_link.attrs['href']))
        section.children = [self.parse_code_ul(url, child)
                            for child in li.find_all('ul', recursive=False)]
        return section
1277473
def find_prototypes(code):
    

    prots = []
    lines = code.split('\n')
    for line in lines:
        m = re.match("\s*" + re_func_prot, line)
        if m is not None:
            rtype, name, args = m.groups()[:3]
            if args == 'void' or args.strip() == '':
                args = []
            else:
                args = [tuple(arg.strip().split(' '))
                        for arg in args.split(',')]
            prots.append((name, args, rtype))

    return prots
625157
def main():
    """"""

    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('learn', help="learning source codes directory")
    parser.add_argument('keywords', help="output keywords file, JSON")
    parser.add_argument(
        '-n', '--nbkeywords', type=int, default=10000,
        help="the number of keywords to keep")
    parser.add_argument(
        '-d', '--debug', default=False, action='store_true',
        help="show debug messages")

    args = parser.parse_args()
    config_logging(args.debug)

    learn_path = Path(args.learn)
    keywords_path = Path(args.keywords)
    nb_keywords = args.nbkeywords

    languages = config_dict('languages.json')
    exts = {ext: lang for lang, exts in languages.items() for ext in exts}

    term_count = Counter()
    document_count = Counter()
    pos = 0
    LOGGER.info("Reading files form %s", learn_path)
    for pos, path in enumerate(Path(learn_path).glob('**/*'), 1):
        if pos % STEP == 0:
            LOGGER.debug("Processed %d", pos)
            gc.collect()  # Cleanup dirt

        if not path.is_file() or not exts.get(path.suffix.lstrip('.')):
            continue

        counter = _extract(path)
        term_count.update(counter)
        document_count.update(counter.keys())

    nb_terms = sum(term_count.values())
    nb_documents = pos - 1
    if not nb_documents:
        LOGGER.error("No source files found in %s", learn_path)
        raise RuntimeError('No source files in {}'.format(learn_path))

    LOGGER.info("%d unique terms found", len(term_count))

    terms = _most_frequent(
        (term_count, nb_terms), (document_count, nb_documents), nb_keywords)

    keywords = {
        token: int(hashlib.sha1(token.encode()).hexdigest(), 16)
        for token in terms
    }

    with keywords_path.open('w') as keywords_file:
        json.dump(keywords, keywords_file, indent=2, sort_keys=True)
    LOGGER.info("%d keywords written into %s", len(keywords), keywords_path)
    LOGGER.debug("Exit OK")
545653
function parse(code) {
  var opts = arguments.length <= 1 || arguments[1] === undefined ? {} : arguments[1];

  opts.allowHashBang = true;
  opts.sourceType = "module";
  opts.ecmaVersion = Infinity;
  opts.plugins = {
    jsx: true,
    flow: true
  };
  opts.features = {};

  for (var key in _transformation2["default"].pipeline.transformers) {
    opts.features[key] = true;
  }

  var ast = babylon.parse(code, opts);

  if (opts.onToken) {
    // istanbul ignore next

    var _opts$onToken;

    (_opts$onToken = opts.onToken).push.apply(_opts$onToken, ast.tokens);
  }

  if (opts.onComment) {
    // istanbul ignore next

    var _opts$onComment;

    (_opts$onComment = opts.onComment).push.apply(_opts$onComment, ast.comments);
  }

  return ast.program;
}
1120165
def parse(self, parser):
        """"""
        lineno = next(parser.stream).lineno
        num_called_num = False

        # find all the variables referenced.  Additionally a variable can be
        # defined in the body of the trans block too, but this is checked at
        # a later state.
        plural_expr = None
        plural_expr_assignment = None
        variables = {}
        trimmed = None
        while parser.stream.current.type != 'block_end':
            if variables:
                parser.stream.expect('comma')

            # skip colon for python compatibility
            if parser.stream.skip_if('colon'):
                break

            name = parser.stream.expect('name')
            if name.value in variables:
                parser.fail('translatable variable %r defined twice.' %
                            name.value, name.lineno,
                            exc=TemplateAssertionError)

            # expressions
            if parser.stream.current.type == 'assign':
                next(parser.stream)
                variables[name.value] = var = parser.parse_expression()
            elif trimmed is None and name.value in ('trimmed', 'notrimmed'):
                trimmed = name.value == 'trimmed'
                continue
            else:
                variables[name.value] = var = nodes.Name(name.value, 'load')

            if plural_expr is None:
                if isinstance(var, nodes.Call):
                    plural_expr = nodes.Name('_trans', 'load')
                    variables[name.value] = plural_expr
                    plural_expr_assignment = nodes.Assign(
                        nodes.Name('_trans', 'store'), var)
                else:
                    plural_expr = var
                num_called_num = name.value == 'num'

        parser.stream.expect('block_end')

        plural = None
        have_plural = False
        referenced = set()

        # now parse until endtrans or pluralize
        singular_names, singular = self._parse_block(parser, True)
        if singular_names:
            referenced.update(singular_names)
            if plural_expr is None:
                plural_expr = nodes.Name(singular_names[0], 'load')
                num_called_num = singular_names[0] == 'num'

        # if we have a pluralize block, we parse that too
        if parser.stream.current.test('name:pluralize'):
            have_plural = True
            next(parser.stream)
            if parser.stream.current.type != 'block_end':
                name = parser.stream.expect('name')
                if name.value not in variables:
                    parser.fail('unknown variable %r for pluralization' %
                                name.value, name.lineno,
                                exc=TemplateAssertionError)
                plural_expr = variables[name.value]
                num_called_num = name.value == 'num'
            parser.stream.expect('block_end')
            plural_names, plural = self._parse_block(parser, False)
            next(parser.stream)
            referenced.update(plural_names)
        else:
            next(parser.stream)

        # register free names as simple name expressions
        for var in referenced:
            if var not in variables:
                variables[var] = nodes.Name(var, 'load')

        if not have_plural:
            plural_expr = None
        elif plural_expr is None:
            parser.fail('pluralize without variables', lineno)

        if trimmed is None:
            trimmed = self.environment.policies['ext.i18n.trimmed']
        if trimmed:
            singular = self._trim_whitespace(singular)
            if plural:
                plural = self._trim_whitespace(plural)

        node = self._make_node(singular, plural, variables, plural_expr,
                               bool(referenced),
                               num_called_num and have_plural)
        node.set_lineno(lineno)
        if plural_expr_assignment is not None:
            return [plural_expr_assignment, node]
        else:
            return node
321770
def _parse_literal_list(self, indent):
        """"""
        if self._cur_token['type'] not in self._literals:
            raise Exception(
                "Parser failed, _parse_literal_list was called on non-literal"
                " {} on line {}.".format(
                    repr(self._cur_token['value']), self._cur_token['line']
                )
            )

        # find next token after whitespace without incrementing
        temp_position = self._cur_position
        while (
            temp_position < self.num_tokens-1 and (
                self.tokens[temp_position]['type'] is TT.ws or
                self.tokens[temp_position]['type'] in self._literals
            )
        ):
            temp_position += 1
        next_token = self.tokens[temp_position]

        # end of stream
        if next_token['type'] is TT.ws:
            return self._cur_token['value']
        elif next_token['type'] is TT.comma:
            return self._parse_comma_list()
        elif next_token['type'] is TT.lbreak:
            while (
                temp_position < self.num_tokens-1 and
                self.tokens[temp_position]['type'] in (TT.lbreak, TT.ws)
            ):
                temp_position += 1
            if self.tokens[temp_position]['type'] in self._literals:
                return self._parse_newline_list(indent)
            else:
                rval = self._cur_token['value']
                self._increment()
                return rval
        else:
            rval = self._cur_token['value']
            self._increment()
            return rval
973952
function Parser(options) {
  this.options = merge({}, {
    headers: {
      included: false,
      downcase: true,
      upcase: true
    },
    delimiter: 'tab',
    decimalSign: 'comma'
  }, options);
}
667032
def _clean_code(code):
    
    lines = []
    # cleanup lines, the parser is very sensitive to extra spaces,...
    for l in code.splitlines():
        # remove last .
        if l.endswith('.'):
            l = l[:-1]
        # the parser doe not like VALUE xxx.
        if "VALUE" in l:
            l = l[:l.find("VALUE")]
        # the parser does not like extra spaces between "PIC X(xxx)" and "."
        indent = len(l) - len(l.lstrip())
        tokens = l.split(" ")
        while "" in tokens:
            tokens.remove("")
        if tokens and not tokens[-1].endswith("."):
            tokens[-1] += "."
        lines.append(" " * indent + " ".join(tokens))

    return lines
787772
def parse_code(url):
    
    result = urlparse(url)
    query = parse_qs(result.query)
    return query['code']
598895
public static final short parseShort(CharSequence cs)
    {
        if (CharSequences.startsWith(cs, "0b"))
        {
            return parseShort(cs, 2, 2, cs.length());
        }
        if (CharSequences.startsWith(cs, "0x"))
        {
            return parseShort(cs, 16, 2, cs.length());
        }
        return parseShort(cs, 10);
    }