convert notebooks to Python script after save with nbconvert

    replaces `ipython notebook --script`
421597
def script_post_save(model, os_path, contents_manager, **kwargs):
    
    from nbconvert.exporters.script import ScriptExporter

    if model['type'] != 'notebook':
        return

    global _script_exporter

    if _script_exporter is None:
        _script_exporter = ScriptExporter(parent=contents_manager)

    log = contents_manager.log

    base, ext = os.path.splitext(os_path)
    # py_fname = base + '.py'
    script, resources = _script_exporter.from_filename(os_path)
    script_fname = base + resources.get('output_extension', '.txt')
    log.info("Saving script /%s", to_api_path(script_fname, contents_manager.root_dir))

    with io.open(script_fname, 'w', encoding='utf-8') as f:
        f.write(script)
797808
def _detect_notebook() -> bool:
    
    try:
        from IPython import get_ipython
        from ipykernel import zmqshell
    except ImportError:
        return False
    kernel = get_ipython()
    try:
        from spyder.utils.ipython.spyder_kernel import SpyderKernel

        if isinstance(kernel.kernel, SpyderKernel):
            return False
    except (ImportError, AttributeError):
        pass
    return isinstance(kernel, zmqshell.ZMQInteractiveShell)
611212
def enable_notebook():
    """"""
    try:
        from IPython.core.getipython import get_ipython
    except ImportError:
        raise ImportError('This feature requires IPython 1.0+')
    ip = get_ipython()
    f = ip.display_formatter.formatters['text/html']
    f.for_type(np.ndarray, _array_to_html)
524773
def convert(cls, style_dict, num_format_str=None):
        

        # Create a XlsxWriter format object.
        props = {}

        if num_format_str is not None:
            props['num_format'] = num_format_str

        if style_dict is None:
            return props

        if 'borders' in style_dict:
            style_dict = style_dict.copy()
            style_dict['border'] = style_dict.pop('borders')

        for style_group_key, style_group in style_dict.items():
            for src, dst in cls.STYLE_MAPPING.get(style_group_key, []):
                # src is a sequence of keys into a nested dict
                # dst is a flat key
                if dst in props:
                    continue
                v = style_group
                for k in src:
                    try:
                        v = v[k]
                    except (KeyError, TypeError):
                        break
                else:
                    props[dst] = v

        if isinstance(props.get('pattern'), str):
            # TODO: support other fill patterns
            props['pattern'] = 0 if props['pattern'] == 'none' else 1

        for k in ['border', 'top', 'right', 'bottom', 'left']:
            if isinstance(props.get(k), str):
                try:
                    props[k] = ['none', 'thin', 'medium', 'dashed', 'dotted',
                                'thick', 'double', 'hair', 'mediumDashed',
                                'dashDot', 'mediumDashDot', 'dashDotDot',
                                'mediumDashDotDot',
                                'slantDashDot'].index(props[k])
                except ValueError:
                    props[k] = 2

        if isinstance(props.get('font_script'), str):
            props['font_script'] = ['baseline', 'superscript',
                                    'subscript'].index(props['font_script'])

        if isinstance(props.get('underline'), str):
            props['underline'] = {'none': 0, 'single': 1, 'double': 2,
                                  'singleAccounting': 33,
                                  'doubleAccounting': 34}[props['underline']]

        return props
1163625
def load_module(self, fullname):
        """"""
        path = find_notebook(fullname, self.path)

        print ("importing Jupyter notebook from %s" % path)

        # load the notebook object
        with io.open(path, 'r', encoding='utf-8') as f:
            nb = read(f, 4)


        # create the module and add it to sys.modules if name in sys.modules:
        #    return sys.modules[name]
        mod = types.ModuleType(fullname)
        mod.__file__ = path
        mod.__loader__ = self
        mod.__dict__['get_ipython'] = get_ipython
        sys.modules[fullname] = mod

        # extra work to ensure that magics that would affect the user_ns
        # actually affect the notebook module's ns
        save_user_ns = self.shell.user_ns
        self.shell.user_ns = mod.__dict__

        try:
          for cell in nb.cells:
            if cell.cell_type == 'code':
                # transform the input to executable Python
                code = self.shell.input_transformer_manager.transform_cell(cell.source)
                # run the code in themodule
                exec(code, mod.__dict__)
        finally:
            self.shell.user_ns = save_user_ns
        return mod
80095
def disable_notebook():
    """"""
    try:
        from IPython.core.getipython import get_ipython
    except ImportError:
        raise ImportError('This feature requires IPython 1.0+')
    ip = get_ipython()
    f = ip.display_formatter.formatters['text/html']
    f.type_printers.pop(np.ndarray, None)
438951
def auto(self, enabled=True, **kwargs):
        
        self.namespace = self.get_namespace()
        self.notebook_name = "{notebook}"
        self._timestamp = tuple(time.localtime())
        kernel = r'var kernel = IPython.notebook.kernel; '
        nbname = r"var nbname = IPython.notebook.get_notebook_name(); "
        nbcmd = (r"var name_cmd = '%s.notebook_name = \"' + nbname + '\"'; " % self.namespace)
        cmd = (kernel + nbname + nbcmd + "kernel.execute(name_cmd); ")
        display(Javascript(cmd))
        time.sleep(0.5)
        self._auto=enabled
        self.param.set_param(**kwargs)

        tstamp = time.strftime(" [%Y-%m-%d %H:%M:%S]", self._timestamp)
        print("Automatic capture is now %s.%s"
              % ('enabled' if enabled else 'disabled',
                 tstamp if enabled else ''))
906166
def _save_notebook(self, os_path, nb):
        """"""
        with self.atomic_writing(os_path, encoding='utf-8') as f:
            if ftdetect(os_path) == 'notebook':
                nbformat.write(nb, f, version=nbformat.NO_CONVERT)
            elif ftdetect(os_path) == 'markdown':
                nbjson = nbformat.writes(nb, version=nbformat.NO_CONVERT)
                markdown = convert(nbjson,
                                   informat='notebook',
                                   outformat='markdown',
                                   strip_outputs=self.strip_outputs)
                f.write(markdown)
855766
def _read_notebook(self, os_path, as_version=4):
        """"""
        with self.open(os_path, 'r', encoding='utf-8') as f:
            try:
                if ftdetect(os_path) == 'notebook':
                    return nbformat.read(f, as_version=as_version)
                elif ftdetect(os_path) == 'markdown':
                    nbjson = convert(os_path,
                                     informat='markdown',
                                     outformat='notebook')
                    return nbformat.reads(nbjson, as_version=as_version)
            except Exception as e:
                raise web.HTTPError(
                    400,
                    u"Unreadable Notebook: %s %r" % (os_path, e),
                )
1126264
def scrub_output_pre_save(model, **kwargs):
    """"""
    # only run on notebooks
    if model['type'] != 'notebook':
        return
    # only run on nbformat v4
    if model['content']['nbformat'] != 4:
        return

    for cell in model['content']['cells']:
        if cell['cell_type'] != 'code':
            continue
        cell['outputs'] = []
        cell['execution_count'] = None
824611
def make_example_scripts_docs(spth, npth, rpth):
    

    # Ensure that output directory exists
    mkdir(rpth)

    # Iterate over index files
    for fp in glob(os.path.join(spth, '*.rst')) + \
              glob(os.path.join(spth, '*', '*.rst')):
        # Index basename
        b = os.path.basename(fp)
        # Index dirname
        dn = os.path.dirname(fp)
        # Name of subdirectory of examples directory containing current index
        sd = os.path.split(dn)
        # Set d to the name of the subdirectory of the root directory
        if dn == spth:  # fp is the root directory index file
            d = ''
        else:           # fp is a subdirectory index file
            d = sd[-1]
        # Path to corresponding subdirectory in docs directory
        fd = os.path.join(rpth, d)
        # Ensure notebook subdirectory exists
        mkdir(fd)
        # Filename of index file to be constructed
        fn = os.path.join(fd, b)
        # Process current index file if corresponding notebook file
        # doesn't exist, or is older than index file
        if update_required(fp, fn):
            print('Converting %s                ' % os.path.join(d, b),
                  end='\r')
            # Convert script index to docs index
            rst_to_docs_rst(fp, fn)

    # Iterate over example scripts
    for fp in sorted(glob(os.path.join(spth, '*', '*.py'))):
        # Name of subdirectory of examples directory containing current script
        d = os.path.split(os.path.dirname(fp))[1]
        # Script basename
        b = os.path.splitext(os.path.basename(fp))[0]
        # Path to corresponding notebook
        fn = os.path.join(npth, d, b + '.ipynb')
        # Path to corresponding sphinx doc file
        fr = os.path.join(rpth, d, b + '.rst')
        # Only proceed if script and notebook exist
        if os.path.exists(fp) and os.path.exists(fn):
            # Convert notebook to rst if notebook is newer than rst
            # file or if rst file doesn't exist
            if update_required(fn, fr):
                fnb = os.path.join(d, b + '.ipynb')
                print('Processing %s                ' % fnb, end='\r')
                script_and_notebook_to_rst(fp, fn, fr)
        else:
            print('WARNING: script %s or notebook %s not found' %
                  (fp, fn))
1102168
def convert_notebooks(self):
        """"""
        fnames = self.get_selected_filenames()
        if not isinstance(fnames, (tuple, list)):
            fnames = [fnames]
        for fname in fnames:
            self.convert_notebook(fname)
438624
def npz_convert(self, infile, item):
        """"""
        data = np.load(infile)
        labels = self._labels(data)
        features = data['features']
        self._write(item, labels, features)
1195105
def make_notebook(cell_list):
    
    import utool as ut
    header = ut.codeblock(
        '''
        {
         "cells": [
        '''
    )

    footer = ut.codeblock(
        '''
         ],
         "metadata": {
          "kernelspec": {
           "display_name": "Python 2",
           "language": "python",
           "name": "python2"
          },
          "language_info": {
           "codemirror_mode": {
            "name": "ipython",
            "version": 2
           },
           "file_extension": ".py",
           "mimetype": "text/x-python",
           "name": "python",
           "nbconvert_exporter": "python",
           "pygments_lexer": "ipython2",
           "version": "2.7.6"
          }
         },
         "nbformat": 4,
         "nbformat_minor": 0
        }
        ''')

    cell_body = ut.indent(',\n'.join(cell_list), '  ')
    notebook_str = header + '\n' + cell_body +  '\n' +  footer
    try:
        import json
        json.loads(notebook_str)
    except ValueError as ex:
        ut.printex(ex, 'Invalid notebook JSON')
        raise
    return notebook_str
3668
def save_as(self, name=None, close=False):
        """"""
        current_client = self.get_current_client()
        current_client.save()
        original_path = current_client.get_filename()
        if not name:
            original_name = osp.basename(original_path)
        else:
            original_name = name
        filename, _selfilter = getsavefilename(self, _("Save notebook"),
                                               original_name, FILES_FILTER)
        if filename:
            nb_contents = nbformat.read(original_path, as_version=4)
            nbformat.write(nb_contents, filename)
            if not close:
                self.close_client(save=True)
            self.create_new_client(filename=filename)
164014
def jupyter_notebook(script_blocks, gallery_conf):
    
    first_cell = gallery_conf.get("first_notebook_cell", "%matplotlib inline")
    work_notebook = jupyter_notebook_skeleton()
    if first_cell is not None:
        add_code_cell(work_notebook, first_cell)
    fill_notebook(work_notebook, script_blocks)

    return work_notebook
687215
def convert(srctree, dsttree=dsttree, readonly=False, dumpall=False,
            ignore_exceptions=False, fullcomp=False):
    

    if fullcomp:
        allow_ast_comparison()

    parse_file = code_to_ast.parse_file
    find_py_files = code_to_ast.find_py_files
    srctree = os.path.normpath(srctree)

    if not readonly:
        dsttree = os.path.normpath(dsttree)
        logging.info('')
        logging.info('Trashing ' + dsttree)
        shutil.rmtree(dsttree, True)

    unknown_src_nodes = set()
    unknown_dst_nodes = set()
    badfiles = set()
    broken = []

    oldpath = None

    allfiles = find_py_files(srctree, None if readonly else dsttree)
    for srcpath, fname in allfiles:
        # Create destination directory
        if not readonly and srcpath != oldpath:
            oldpath = srcpath
            if srcpath >= srctree:
                dstpath = srcpath.replace(srctree, dsttree, 1)
                if not dstpath.startswith(dsttree):
                    raise ValueError("%s not a subdirectory of %s" %
                                     (dstpath, dsttree))
            else:
                assert srctree.startswith(srcpath)
                dstpath = dsttree
            os.makedirs(dstpath)

        srcfname = os.path.join(srcpath, fname)
        logging.info('Converting %s' % srcfname)
        try:
            srcast = parse_file(srcfname)
        except SyntaxError:
            badfiles.add(srcfname)
            continue

        try:
            dsttxt = to_source(srcast)
        except:
            if not ignore_exceptions:
                raise
            dsttxt = ''

        if not readonly:
            dstfname = os.path.join(dstpath, fname)
            try:
                with open(dstfname, 'wb') as f:
                    f.write(out_prep(dsttxt))
            except UnicodeEncodeError:
                badfiles.add(dstfname)

        # As a sanity check, make sure that ASTs themselves
        # round-trip OK
        try:
            dstast = ast.parse(dsttxt) if readonly else parse_file(dstfname)
        except SyntaxError:
            dstast = []
        if fullcomp:
            unknown_src_nodes.update(strip_tree(srcast))
            unknown_dst_nodes.update(strip_tree(dstast))
            bad = srcast != dstast
        else:
            bad = not fast_compare(srcast, dstast)
        if dumpall or bad:
            srcdump = dump_tree(srcast)
            dstdump = dump_tree(dstast)
            logging.warning('    calculating dump -- %s' %
                            ('bad' if bad else 'OK'))
            if bad:
                broken.append(srcfname)
            if dumpall or bad:
                if not readonly:
                    try:
                        with open(dstfname[:-3] + '.srcdmp', 'wb') as f:
                            f.write(out_prep(srcdump))
                    except UnicodeEncodeError:
                        badfiles.add(dstfname[:-3] + '.srcdmp')
                    try:
                        with open(dstfname[:-3] + '.dstdmp', 'wb') as f:
                            f.write(out_prep(dstdump))
                    except UnicodeEncodeError:
                        badfiles.add(dstfname[:-3] + '.dstdmp')
                elif dumpall:
                    sys.stdout.write('\n\nAST:\n\n    ')
                    sys.stdout.write(srcdump.replace('\n', '\n    '))
                    sys.stdout.write('\n\nDecompile:\n\n    ')
                    sys.stdout.write(dsttxt.replace('\n', '\n    '))
                    sys.stdout.write('\n\nNew AST:\n\n    ')
                    sys.stdout.write('(same as old)' if dstdump == srcdump
                                     else dstdump.replace('\n', '\n    '))
                    sys.stdout.write('\n')

    if badfiles:
        logging.warning('\nFiles not processed due to syntax errors:')
        for fname in sorted(badfiles):
            logging.warning('    %s' % fname)
    if broken:
        logging.warning('\nFiles failed to round-trip to AST:')
        for srcfname in broken:
            logging.warning('    %s' % srcfname)

    ok_to_strip = 'col_offset _precedence _use_parens lineno _p_op _pp'
    ok_to_strip = set(ok_to_strip.split())
    bad_nodes = (unknown_dst_nodes | unknown_src_nodes) - ok_to_strip
    if bad_nodes:
        logging.error('\nERROR -- UNKNOWN NODES STRIPPED: %s' % bad_nodes)
    logging.info('\n')
    return broken