TODO: Docstring for _handle_file_ast.

        :node: TODO
        :scope: TODO
        :ctxt: TODO
        :stream: TODO
        :returns: TODO
544315
def _handle_file_ast(self, node, scope, ctxt, stream):
        
        self._root = ctxt = fields.Dom(stream)
        ctxt._pfp__scope = scope
        self._root._pfp__name = "__root"
        self._root._pfp__interp = self
        self._dlog("handling file AST with {} children".format(len(node.children())))

        for child in node.children():
            self._handle_node(child, scope, ctxt, stream)

        ctxt._pfp__process_fields_metadata()

        return ctxt
978717
def _handle_assignment(self, node, scope, ctxt, stream):
        
        def add_op(x,y): x += y
        def sub_op(x,y): x -= y
        def div_op(x,y): x /= y
        def mod_op(x,y): x %= y
        def mul_op(x,y): x *= y
        def xor_op(x,y): x ^= y
        def and_op(x,y): x &= y
        def or_op(x,y): x |= y
        def lshift_op(x,y): x <<= y
        def rshift_op(x,y): x >>= y
        def assign_op(x,y): x._pfp__set_value(y)

        switch = {
            "+="    : add_op,
            "-="    : sub_op,
            "/="    : div_op,
            "%="    : mod_op,
            "*="    : mul_op,
            "^="    : xor_op,
            "&="    : and_op,
            "|="    : or_op,
            "<<="    : lshift_op,
            ">>="    : rshift_op,
            "="        : assign_op
        }

        self._dlog("handling assignment")
        field = self._handle_node(node.lvalue, scope, ctxt, stream)
        self._dlog("field = {}".format(field))
        value = self._handle_node(node.rvalue, scope, ctxt, stream)

        if node.op is None:
            self._dlog("value = {}".format(value))
            field._pfp__set_value(value)
        else:
            self._dlog("value {}= {}".format(node.op, value))
            if node.op not in switch:
                raise errors.UnsupportedAssignmentOperator(node.coord, node.op)
            switch[node.op](field, value)
795755
def _handle_unary_op(self, node, scope, ctxt, stream):
        
        self._dlog("handling unary op {}".format(node.op))

        special_switch = {
            "parentof"            : self._handle_parentof,
            "exists"            : self._handle_exists,
            "function_exists"    : self._handle_function_exists,
            "p++"                : self._handle_post_plus_plus,
            "p--"                : self._handle_post_minus_minus,
        }

        switch = {
            # for ++i and --i
            "++":        lambda x,v: x.__iadd__(1),
            "--":        lambda x,v: x.__isub__(1),

            "~":        lambda x,v: ~x,
            "!":        lambda x,v: not x,
            "-":        lambda x,v: -x,
            "sizeof":    lambda x,v: (fields.UInt64()+x._pfp__width()),
            "startof":    lambda x,v: (fields.UInt64()+x._pfp__offset),
        }

        if node.op not in switch and node.op not in special_switch:
            raise errors.UnsupportedUnaryOperator(node.coord, node.op)

        if node.op in special_switch:
            return special_switch[node.op](node, scope, ctxt, stream)

        field = self._handle_node(node.expr, scope, ctxt, stream)
        if type(field) is type:
            field = field()
        res = switch[node.op](field, 1)
        if type(res) is bool:
            new_res = field.__class__()
            new_res._pfp__set_value(1 if res == True else 0)
            res = new_res
        return res
33893
def _handle_exists(self, node, scope, ctxt, stream):
        
        res = fields.Int()
        try:
            self._handle_node(node.expr, scope, ctxt, stream)
            res._pfp__set_value(1)
        except AttributeError:
            res._pfp__set_value(0)
        return res
961542
def _parse(self):
        
        #TODO manage decorators
        #TODO manage default params with strings escaping chars as (, ), ', ', #, ...
        #TODO manage elements ending with comments like: def func(param): # blabla
        elem_list = []
        reading_element = None
        reading_docs = None
        waiting_docs = False
        elem = ''
        raw = ''
        start = 0
        end = 0

        try:
            if self.input_file == '-':
                fd = sys.stdin
            else:
                fd = open(self.input_file)

            self.input_lines = fd.readlines()

            if self.input_file != '-':
                fd.close()

        except IOError:
            msg = BaseException('Failed to open file "' + self.input_file + '". Please provide a valid file.')
            raise msg
        for i, ln in enumerate(self.input_lines):
            l = ln.strip()
            if reading_element:
                elem += l
                if l.endswith(':'):
                    reading_element = 'end'
            elif (l.startswith('def ') or l.startswith('class ')) and not reading_docs:
                if self.ignore_private and l[l.find(' '):].strip().startswith("__"):
                    continue
                reading_element = 'start'
                elem = l
                m = re.match(r'^(\s*)[dc]{1}', ln)
                if m is not None and m.group(1) is not None:
                    spaces = m.group(1)
                else:
                    spaces = ''
                if re.search(r''':(|\s*#[^'"]*)$''', l):
                    reading_element = 'end'
            if reading_element == 'end':
                reading_element = None
                # if currently reading an element content
                waiting_docs = True
                # *** Creates the DocString object ***
                e = DocString(elem.replace('\n', ' '), spaces, quotes=self.quotes,
                              input_style=self.input_style,
                              output_style=self.output_style,
                              first_line=self.first_line,
                              **self.kwargs)
                elem_list.append({'docs': e, 'location': (-i, -i)})
            else:
                if waiting_docs and ('"""' in l or "'''" in l):
                    # start of docstring bloc
                    if not reading_docs:
                        start = i
                        # determine which delimiter
                        idx_c = l.find('"""')
                        idx_dc = l.find("'''")
                        lim = '"""'
                        if idx_c >= 0 and idx_dc >= 0:
                            if idx_c < idx_dc:
                                lim = '"""'
                            else:
                                lim = "'''"
                        elif idx_c < 0:
                            lim = "'''"
                        reading_docs = lim
                        raw = ln
                        # one line docstring
                        if l.count(lim) == 2:
                            end = i
                            elem_list[-1]['docs'].parse_docs(raw)
                            elem_list[-1]['location'] = (start, end)
                            reading_docs = None
                            waiting_docs = False
                            reading_element = False
                            raw = ''
                    # end of docstring bloc
                    elif waiting_docs and lim in l:
                        end = i
                        raw += ln
                        elem_list[-1]['docs'].parse_docs(raw)
                        elem_list[-1]['location'] = (start, end)
                        reading_docs = None
                        waiting_docs = False
                        reading_element = False
                        raw = ''
                    # inside a docstring bloc
                    elif waiting_docs:
                        raw += ln
                # no docstring found for current element
                elif waiting_docs and l != '' and reading_docs is None:
                    waiting_docs = False
                else:
                    if reading_docs is not None:
                        raw += ln
        if self.convert_only:
            i = 0
            while i < len(elem_list):
                if elem_list[i]['docs'].get_input_docstring() is None:
                    elem_list.pop(i)
                else:
                    i += 1
        self.docs_list = elem_list

        self.parsed = True
        return elem_list
63503
function here (rstream, opts) {
    opts = opts || {}

    var resources = []
    var sort = opts.sort
    var namespace = opts.name
    var transform = opts.transform
    var prefix = opts.prefix
    var done // flag for resouce stream
    var waitting

    function collectResources(cb) {
        if (waitting) {
            waitting.push(cb)
        } else if (done) {
            cb(resources)
        } else if (rstream._done) {
            cb(rstream._resources || [])
        } else {
            waitting = []
            rstream.pipe(map(function (file, next) {
                resources.push(file)
                next(null, file)
            })).on('end', function () {
                rstream._done = done = true
                rstream._resources = resources
                cb(resources)
                var queue = waitting
                waitting = null
                setTimeout(function () {
                    queue.forEach(function (c) {
                        c(resources)
                    })
                })
            })
        }
    }
    /**
     * Handle each template file independ
     */
    return es.map(function (tpl, cb) {
        if (tpl.isStream()) {
            this.emit('error', new PluginError(PLUGIN_NAME, 'Stream is not supported for template target!'))
            return cb(null, tpl)
        }
        if (tpl.isNull()) {
            this.emit('error', new PluginError(PLUGIN_NAME, 'Template target should not be null!'))
            return cb(null, tpl)
        }

        var content = tpl.contents.toString()
        var ast = parser(content)

        collectResources(function (resources) {
            var count = 0
            function walk(node, scope) {
                var output = ''
                var isBlock
                switch(node.nodeType) {
                    // root node of ast
                    case 1:
                        output += node.childNodes.map(function (n) {
                            return walk(n, scope)
                        }).join('')
                        break
                    // open tag
                    case 2:
                        isBlock = true
                    // self closing tag
                    case 3:
                        // expression form tag
                        var expression = isBlock ? node.openHTML : node.outerHTML
                        var starttag = isBlock ? expression : expression.replace(/\/-->$/, '-->')
                        var endtag = isBlock ? node.closeHTML : '<!--/here-->'
                        var exprObj = Tag.expr(expression)
                        if (exprObj) {
                            output += starttag
                            if ( 
                                (!namespace && exprObj.namespace) || 
                                (namespace && (exprObj.namespace !== namespace && exprObj.namespace !== '*') ) 
                            ) {
                                if (node.childNodes.length) {
                                    output += node.childNodes.map(function (n) {
                                        return n.toString()
                                    }).join('')
                                }
                                output += endtag
                                break
                            } else {
                                var tagObj = {
                                    inline: exprObj.inline,
                                    wrap: exprObj.wrap,
                                    namespace: exprObj.namespace,
                                    wrapresult: false,
                                    relative: true // default set relative, if not url path
                                }
                                var rs = resources.slice(0)

                                rs = rs.reduce(function (next, file) {
                                    if (exprObj.regexp.test(file.path)) {
                                        next.push(file)
                                    }
                                    return next
                                }, [])


                                if (rs.length > 1 && sort) {
                                    rs = sort(rs, tpl, tagObj)
                                }

                                output += rs.map(function (file) {
                                    var result = true
                                    var opts = _.extend({}, tagObj)

                                    file = file.clone()
                                    // marked as dirty will not set relative path
                                    if (typeof prefix === 'function') {
                                        file.path = prefix(file, tpl, opts)
                                    } else if (typeof prefix === 'string') {
                                        file.path = prefix + file.relative
                                    }
                                    if (transform) {
                                        result = transform(file, tpl, opts)
                                    }
                                    count ++
                                    if (result === true) {
                                        return opts.inline 
                                            ? Tag.inline(file, tpl, opts) 
                                            : Tag.transform(file, tpl, opts)
                                    } else if (result && opts.wrapresult) {
                                        return Tag.transform(result, opts)
                                    } else if (result) {
                                        return result
                                    } else {
                                        count --
                                        return ''
                                    }
                                }).join('')
                            }
                            output += endtag

                        } else {
                            this.emit('error', new PluginError(PLUGIN_NAME, 'Unvalid expression:"' + expression +'"'))
                            break
                        }
                        break
                    case 4:
                        output += node.nodeValue
                }
                return output   
            }
            tpl.contents = new Buffer(walk(ast))
            gutil.log(
                PLUGIN_NAME + ': ' + colors.yellow(count ? '✔ ' : '❗ ') +
                colors.cyan(namespace ? '[' + namespace + '] ' : ' ' ) + 'Release', 
                colors.green(count), 
                'files to', 
                colors.blue(path.basename(tpl.path))
            )
            cb(null, tpl)
        })
    })
}
968125
def doc_stream(path):
    
    with open(path, 'r') as f:
        for line in f:
            if line.strip():
                yield line
767709
def get_docstring(filename, verbose=False):
    

    doc = None

    try:
        # Thank you, Habbie, for this bit of code :-)
        M = ast.parse(''.join(open(filename)))
        for child in M.body:
            if isinstance(child, ast.Assign):
                if 'DOCUMENTATION' in (t.id for t in child.targets):
                    doc = yaml.load(child.value.s)
    except:
        if verbose == True:
            traceback.print_exc()
            print "unable to parse %s" % filename
    return doc
639717
function processFile() {
  return (file, callback) => {
    let parsed;
    if (file.meta !== undefined && file.meta.layout !== undefined) {
      const ext = path.extname(file.meta.layout);
      parsed = path.parse(`${trimExtension(file.path)}${ext}`);
    } else {
      parsed = path.parse(file.path);
    }

    const filePath = {
      dir: parsed.dir,
      name: parsed.name,
      ext: parsed.ext,
    };
    const meta = Object.assign({}, file.meta, { filePath });
    async.nextTick(callback, null, Object.assign({}, file, { meta }));
  };
}
391217
def _handle_typename(self, node, scope, ctxt, stream):
        
        self._dlog("handling typename")

        return self._handle_node(node.type, scope, ctxt, stream)
101503
function handleTernGetFile(request) {

            function replyWith(name, txt) {
                _postMessageByPass({
                    type: MessageIds.TERN_GET_FILE_MSG,
                    file: name,
                    text: txt
                });
            }

            var name = request.file;

            /**
             * Helper function to get the text of a given document and send it to tern.
             * If DocumentManager successfully gets the file's text then we'll send it to the tern node domain.
             * The Promise for getDocumentText() is returned so that custom fail functions can be used.
             *
             * @param {string} filePath - the path of the file to get the text of
             * @return {jQuery.Promise} - the Promise returned from DocumentMangaer.getDocumentText()
             */
            function getDocText(filePath) {
                if (!FileSystem.isAbsolutePath(filePath) || // don't handle URLs
                        filePath.slice(0, 2) === "//") { // don't handle protocol-relative URLs like //example.com/main.js (see #10566)
                    return (new $.Deferred()).reject().promise();
                }

                var file = FileSystem.getFileForPath(filePath),
                    promise = DocumentManager.getDocumentText(file);

                promise.done(function (docText) {
                    resolvedFiles[name] = filePath;
                    numResolvedFiles++;
                    replyWith(name, filterText(docText));
                });
                return promise;
            }

            /**
             * Helper function to find any files in the project that end with the
             * name we are looking for.  This is so we can find requirejs modules
             * when the baseUrl is unknown, or when the project root is not the same
             * as the script root (e.g. if you open the 'brackets' dir instead of 'brackets/src' dir).
             */
            function findNameInProject() {
                // check for any files in project that end with the right path.
                var fileName = name.substring(name.lastIndexOf("/") + 1);

                function _fileFilter(entry) {
                    return entry.name === fileName;
                }

                ProjectManager.getAllFiles(_fileFilter).done(function (files) {
                    var file;
                    files = files.filter(function (file) {
                        var pos = file.fullPath.length - name.length;
                        return pos === file.fullPath.lastIndexOf(name);
                    });

                    if (files.length === 1) {
                        file = files[0];
                    }
                    if (file) {
                        getDocText(file.fullPath).fail(function () {
                            replyWith(name, "");
                        });
                    } else {
                        replyWith(name, "");
                    }
                });
            }

            if (!isFileExcludedInternal(name)) {
                getDocText(name).fail(function () {
                    getDocText(rootTernDir + name).fail(function () {
                        // check relative to project root
                        getDocText(projectRoot + name)
                            // last look for any files that end with the right path
                            // in the project
                            .fail(findNameInProject);
                    });
                });
            }
        }
1035561
def _handle_struct_ref(self, node, scope, ctxt, stream):
        
        self._dlog("handling struct ref")

        # name
        # field
        struct = self._handle_node(node.name, scope, ctxt, stream)

        try:
            sub_field = getattr(struct, node.field.name)
        except AttributeError as e:
            # should be able to access implicit array items by index OR
            # access the last one's members directly without index
            #
            # E.g.:
            # 
            # local int total_length = 0;
            # while(!FEof()) {
            #     HEADER header;
            #   total_length += header.length;
            # }
            if isinstance(struct, fields.Array) and struct.implicit:
                last_item = struct[-1]
                sub_field = getattr(last_item, node.field.name)
            else:
                raise

        return sub_field
1265493
def generate_ast(path):
    
    if os.path.isfile(path):
        with open(path, 'r') as f:
            try:
                tree = ast.parse(f.read())
                return PytTransformer().visit(tree)
            except SyntaxError:  # pragma: no cover
                global recursive
                if not recursive:
                    _convert_to_3(path)
                    recursive = True
                    return generate_ast(path)
                else:
                    raise SyntaxError('The ast module can not parse the file'
                                      ' and the python 2 to 3 conversion'
                                      ' also failed.')
    raise IOError('Input needs to be a file. Path: ' + path)
667613
def parse(input_: Union[str, FileStream], source: str) -> Optional[str]:
    

    # Step 1: Tokenize the input stream
    error_listener = ParseErrorListener()
    if not isinstance(input_, FileStream):
        input_ = InputStream(input_)
    lexer = jsgLexer(input_)
    lexer.addErrorListener(error_listener)
    tokens = CommonTokenStream(lexer)
    tokens.fill()
    if error_listener.n_errors:
        return None

    # Step 2: Generate the parse tree
    parser = jsgParser(tokens)
    parser.addErrorListener(error_listener)
    parse_tree = parser.doc()
    if error_listener.n_errors:
        return None

    # Step 3: Transform the results the results
    parser = JSGDocParser()
    parser.visit(parse_tree)

    if parser.undefined_tokens():
        for tkn in parser.undefined_tokens():
            print("Undefined token: " + tkn)
        return None

    return parser.as_python(source)
821690
def _capture_as_text(capture: Callable[..., Any]) -> str:
    """"""
    if not icontract._represent._is_lambda(a_function=capture):
        signature = inspect.signature(capture)
        param_names = list(signature.parameters.keys())

        return "{}({})".format(capture.__qualname__, ", ".join(param_names))

    lines, lineno = inspect.findsource(capture)
    filename = inspect.getsourcefile(capture)
    decorator_inspection = icontract._represent.inspect_decorator(lines=lines, lineno=lineno, filename=filename)

    call_node = decorator_inspection.node

    capture_node = None  # type: Optional[ast.Lambda]

    if len(call_node.args) > 0:
        assert isinstance(call_node.args[0], ast.Lambda), \
            ("Expected the first argument to the snapshot decorator to be a condition as lambda AST node, "
             "but got: {}").format(type(call_node.args[0]))

        capture_node = call_node.args[0]

    elif len(call_node.keywords) > 0:
        for keyword in call_node.keywords:
            if keyword.arg == "capture":
                assert isinstance(keyword.value, ast.Lambda), \
                    "Expected lambda node as value of the 'capture' argument to the decorator."

                capture_node = keyword.value
                break

        assert capture_node is not None, "Expected to find a keyword AST node with 'capture' arg, but found none"
    else:
        raise AssertionError(
            "Expected a call AST node of a snapshot decorator to have either args or keywords, but got: {}".format(
                ast.dump(call_node)))

    capture_text = decorator_inspection.atok.get_text(capture_node.body)

    return capture_text
415695
def _handle_for(self, node, scope, ctxt, stream):
        
        self._dlog("handling for")
        if node.init is not None:
            # perform the init
            self._handle_node(node.init, scope, ctxt, stream)

        while node.cond is None or self._handle_node(node.cond, scope, ctxt, stream):
            if node.stmt is not None:
                try:
                    # do the for body
                    self._handle_node(node.stmt, scope, ctxt, stream)
                except errors.InterpBreak as e:
                    break
                
                # we still need to interpret the "next" statement,
                # so just pass
                except errors.InterpContinue as e:
                    pass

            if node.next is not None:
                # do the next statement
                self._handle_node(node.next, scope, ctxt, stream)
1328559
def get_docstring_and_rest(filename):
    
    node, content = parse_source_file(filename)

    if node is None:
        return SYNTAX_ERROR_DOCSTRING, content, 1

    if not isinstance(node, ast.Module):
        raise TypeError("This function only supports modules. "
                        "You provided {0}".format(node.__class__.__name__))
    if not (node.body and isinstance(node.body[0], ast.Expr) and
            isinstance(node.body[0].value, ast.Str)):
        raise ValueError(('Could not find docstring in file "{0}". '
                          'A docstring is required by sphinx-gallery '
                          'unless the file is ignored by "ignore_pattern"')
                         .format(filename))

    if LooseVersion(sys.version) >= LooseVersion('3.7'):
        docstring = ast.get_docstring(node)
        assert docstring is not None  # should be guaranteed above
        # This is just for backward compat
        if len(node.body[0].value.s) and node.body[0].value.s[0] == '\n':
            # just for strict backward compat here
            docstring = '\n' + docstring
        ts = tokenize.tokenize(BytesIO(content.encode()).readline)
        # find the first string according to the tokenizer and get its end row
        for tk in ts:
            if tk.exact_type == 3:
                lineno, _ = tk.end
                break
        else:
            lineno = 0
    else:
        # this block can be removed when python 3.6 support is dropped
        docstring_node = node.body[0]
        docstring = docstring_node.value.s
        # python2.7: Code was read in bytes needs decoding to utf-8
        # unless future unicode_literals is imported in source which
        # make ast output unicode strings
        if hasattr(docstring, 'decode') and not isinstance(docstring, unicode):
            docstring = docstring.decode('utf-8')
        lineno = docstring_node.lineno  # The last line of the string.

    # This get the content of the file after the docstring last line
    # Note: 'maxsplit' argument is not a keyword argument in python2
    rest = '\n'.join(content.split('\n')[lineno:])
    lineno += 1
    return docstring, rest, lineno