Implements the storage cell magic for ipython notebooks.

  Args:
    line: the contents of the storage line.
  Returns:
    The results of executing the cell.
437762
def storage(line, cell=None):
  
  parser = datalab.utils.commands.CommandParser(prog='storage', description="""
Execute various storage-related operations. Use "%storage <command> -h"
for help on a specific command.
""")

  # TODO(gram): consider adding a move command too. I did try this already using the
  # objects.patch API to change the object name but that fails with an error:
  #
  # Value 'newname' in content does not agree with value 'oldname'. This can happen when a value
  # set through a parameter is inconsistent with a value set in the request.
  #
  # This is despite 'name' being identified as writable in the storage API docs.
  # The alternative would be to use a copy/delete.
  copy_parser = parser.subcommand('copy',
                                  'Copy one or more GCS objects to a different location.')
  copy_parser.add_argument('-s', '--source', help='The name of the object(s) to copy', nargs='+')
  copy_parser.add_argument('-d', '--destination', required=True,
                           help='The copy destination. For multiple source items this must be a '
                                'bucket.')
  copy_parser.set_defaults(func=_storage_copy)

  create_parser = parser.subcommand('create', 'Create one or more GCS buckets.')
  create_parser.add_argument('-p', '--project', help='The project associated with the objects')
  create_parser.add_argument('-b', '--bucket', help='The name of the bucket(s) to create',
                             nargs='+')
  create_parser.set_defaults(func=_storage_create)

  delete_parser = parser.subcommand('delete', 'Delete one or more GCS buckets or objects.')
  delete_parser.add_argument('-b', '--bucket', nargs='*',
                             help='The name of the bucket(s) to remove')
  delete_parser.add_argument('-o', '--object', nargs='*',
                             help='The name of the object(s) to remove')
  delete_parser.set_defaults(func=_storage_delete)

  list_parser = parser.subcommand('list', 'List buckets in a project, or contents of a bucket.')
  list_parser.add_argument('-p', '--project', help='The project associated with the objects')
  group = list_parser.add_mutually_exclusive_group()
  group.add_argument('-o', '--object',
                     help='The name of the objects(s) to list; can include wildchars',
                     nargs='?')
  group.add_argument('-b', '--bucket',
                     help='The name of the buckets(s) to list; can include wildchars',
                     nargs='?')
  list_parser.set_defaults(func=_storage_list)

  read_parser = parser.subcommand('read',
                                  'Read the contents of a storage object into a Python variable.')
  read_parser.add_argument('-o', '--object', help='The name of the object to read',
                           required=True)
  read_parser.add_argument('-v', '--variable', required=True,
                           help='The name of the Python variable to set')
  read_parser.set_defaults(func=_storage_read)

  view_parser = parser.subcommand('view', 'View the contents of a storage object.')
  view_parser.add_argument('-n', '--head', type=int, default=20,
                           help='The number of initial lines to view')
  view_parser.add_argument('-t', '--tail', type=int, default=20,
                           help='The number of lines from end to view')
  view_parser.add_argument('-o', '--object', help='The name of the object to view',
                           required=True)
  view_parser.set_defaults(func=_storage_view)

  write_parser = parser.subcommand('write',
                                   'Write the value of a Python variable to a storage object.')
  write_parser.add_argument('-v', '--variable', help='The name of the source Python variable',
                            required=True)
  write_parser.add_argument('-o', '--object', required=True,
                            help='The name of the destination GCS object to write')
  write_parser.add_argument('-c', '--content_type', help='MIME type', default='text/plain')
  write_parser.set_defaults(func=_storage_write)

  return datalab.utils.commands.handle_magic_line(line, cell, parser)
361204
def feed_line(self, line):
        
        self.line += 1
        pos = 0
        while pos < len(line):
            loc_start = TextLocationSingle(self.filename, self.line, pos + 1)
            if self.state is State.NORMAL:
                item_re = RE_TOKEN
                thing = 'token'
            elif self.state is State.STRING:
                item_re = RE_STRING_ITEM
                thing = 'escape sequence'
            elif self.state is State.BINARRAY:
                item_re = RE_BINARRAY_ITEM[self.binarray_base]
                thing = 'binarray item'
            else:
                assert 0
            match = item_re.match(line, pos)
            if not match:
                raise ReadError(f'{loc_start}: unknown {thing}')
            pos = match.end()
            loc_end = TextLocationSingle(self.filename, self.line, pos + 1)
            loc = loc_start - loc_end
            if match['ws_error'] is not None:
                raise ReadError(f'{loc_end}: no whitespace after token')
            if self.state is State.NORMAL:
                # Normal state -- read tokens.
                if match['lparen'] is not None:
                    self.stack.append(StackEntryList(loc_start, []))
                elif match['rparen'] is not None:
                    if not self.stack:
                        raise ReadError(f'{loc}: unmatched closing paren')
                    top = self.stack.pop()
                    if not isinstance(top, StackEntryList):
                        top.raise_unclosed_error()
                    yield from self._feed_node(top.items, top.start - loc_end)
                elif match['symbol'] is not None:
                    value = Symbol(match['symbol'])
                    yield from self._feed_node(value, loc)
                elif match['sexpr_comment'] is not None:
                    self.stack.append(StackEntryComment(loc))
                elif match['bool_value'] is not None:
                    value = match['bool_value'] == '@true'
                    yield from self._feed_node(value, loc)
                elif match['nil_value'] is not None:
                    yield from self._feed_node(None, loc)
                elif match['int_or_word'] is not None:
                    if match['number'] is not None:
                        value = int(match['number'], 0)
                    elif match['raw_char'] is not None:
                        value = ord(match['raw_char'])
                    elif match['simple_escape'] is not None:
                        value = ord(ESCAPE_TO_CHAR[match['simple_escape']])
                    elif match['hex_code'] is not None:
                        value = int(match['hex_code'], 16)
                        if value not in range(0x110000):
                            raise ReadError(
                                    f'{loc}: not a valid unicode codepoint')
                    else:
                        assert 0
                    if match['word_width'] is not None:
                        width = int(match['word_width'])
                        if value < 0:
                            value += 1 << width
                        if value not in range(1 << width):
                            raise ReadError(f'{loc}: word value out of range')
                        value = BinWord(width, value)
                    yield from self._feed_node(value, loc)
                elif match['array_width'] is not None:
                    self.binarray_base = {
                        '0b': 2,
                        '0o': 8,
                        None: 10,
                        '0x': 16,
                    }[match['array_base']]
                    self.binarray_data = []
                    self.binarray_width = int(match['array_width'])
                    self.token_start = loc_start
                    self.state = State.BINARRAY
                elif match['start_quote'] is not None:
                    self.state = State.STRING
                    self.token_start = loc_start
                    self.string_buffer = StringIO()
                    if match['string_width'] is not None:
                        self.binarray_width = int(match['string_width'])
                    else:
                        self.binarray_width = None
            elif self.state is State.STRING:
                # Inside a string.
                if match['end_quote'] is not None:
                    self.state = State.NORMAL
                    value = self.string_buffer.getvalue()
                    loc = self.token_start - loc_end
                    if self.binarray_width is not None:
                        vals = [ord(x) for x in value]
                        for x in vals:
                            if x not in range(1 << self.binarray_width):
                                raise ReadError(
                                        f'{loc}: character code out of range')
                        value = BinArray(vals, width=self.binarray_width)
                    yield from self._feed_node(value, loc)
                elif match['raw_chars'] is not None:
                    self.string_buffer.write(match['raw_chars'])
                elif match['simple_escape'] is not None:
                    c = ESCAPE_TO_CHAR[match['simple_escape']]
                    self.string_buffer.write(c)
                elif match['hex_code'] is not None:
                    code = int(match['hex_code'], 16)
                    if code not in range(0x110000):
                        raise ReadError(
                                f'{loc}: not a valid unicode codepoint')
                    self.string_buffer.write(chr(code))
                else:
                    assert 0
            elif self.state is State.BINARRAY:
                # In a BinArray.
                if match['rparen'] is not None:
                    self.state = State.NORMAL
                    value = BinArray(self.binarray_data,
                                     width=self.binarray_width)
                    loc = self.token_start - loc_end
                    yield from self._feed_node(value, loc)
                elif match['digits'] is not None:
                    value = int(match['digits'], self.binarray_base)
                    if value < 0:
                        value += 1 << self.binarray_width
                    if value not in range(1 << self.binarray_width):
                        raise ReadError(f'{loc}: word value out of range')
                    self.binarray_data.append(value)
            else:
                assert 0
718278
def do_execute(self):
        
        if self.storagehandler is None:
            return "No storage handler available!"
        expr = str(self.resolve_option("expression")).replace(
            "{X}", str(self.storagehandler.storage[str(self.resolve_option("storage_name"))]))
        expr = self.storagehandler.expand(expr)
        self.storagehandler.storage[self.resolve_option("storage_name")] = eval(expr)
        self._output.append(self.input)
        return None
404096
def do_rm(self, line):
        ""
        table, line = self.get_table_params(line)
        expected, line = self.get_expected(line)

        args = self.getargs(line)

        if "-v" in args:
            ret = "ALL_OLD"
            args.remove("-v")
        else:
            ret = None

        hkey = self.get_typed_key_value(table, args[0], True)
        rkey = self.get_typed_key_value(table, args[1], False) if len(args) > 1 else None

        item = table.new_item(hash_key=hkey, range_key=rkey)
        item = item.delete(expected_value=expected, return_values=ret)
        self.pprint(item)

        if self.consumed:
            print "consumed units:", item.consumed_units
1075318
def print_line(line):
    
    sys.__stdout__.write("{}\n".format(line))
    sys.__stdout__.flush()
1305007
def do_put(self, line):
        ""
        table, line = self.get_table_params(line)
        expected, line = self.get_expected(line)
        if expected:
            print "expected: not yet implemented"
            return

        if line.startswith('(') or line.startswith('['):
            print "batch: not yet implemented"
            return

            list = self.get_list(line)
            wlist = self.conn.new_batch_write_list()
            wlist.add_batch(table, [ table.new_item(None, None, item) for item in list ])
            response = self.conn.batch_write_item(wlist)
            consumed = response['Responses'][table.table_name]['ConsumedCapacityUnits']

            if 'UnprocessedItems' in response and response['UnprocessedItems']:
                print ""
                print "unprocessed: ", response['UnprocessedItems']
                print ""
        else:
            item = json.loads(line)
            table.put_item(item)
            consumed = None

        if self.consumed and consumed:
            print "consumed units:", consumed
1306829
def do_refresh(self, line):
        ""
        table = self.get_table(line)
        table.refresh(True)
        self.pprint(self.conn.describe_table(table.name))
810336
def get_line(thing):
    
    try:
        return inspect.getsourcelines(thing)[1]
    except TypeError:
        # Might be a property
        return inspect.getsourcelines(thing.fget)[1]
    except Exception as e:
        # print(thing)
        raise e
1324716
def do_save(self, line):
        
        config_file = self._split_args(line, 0, 1)[0]
        self._command_processor.get_session().save(config_file)
        if config_file is None:
            config_file = (
                self._command_processor.get_session().get_default_pickle_file_path()
            )
        self._print_info_if_verbose("Saved session to file: {}".format(config_file))
499638
def scroll_to(self, line):
        
        self._buffer.scroll(line - self._start_line)
        self._start_line = line
211933
def decode(cls, line):
        
        if line.encoded:
            line.value = stringToTextValues(line.value,
                listSeparator=cls.listSeparator)
            line.encoded=False
1034169
def push(cell)
      cell = Workbook::Cell.new(cell, {row:self}) unless cell.class == Workbook::Cell
      super(cell)
    end
37182
def add_output(self, line):
        
        Deployment.objects.filter(pk=self.id).update(output=CF('output')+line)
1144031
def magic_run(self, line):
        
        i = float('inf')
        if line.strip():
            i = int(line)

        try:
            with warnings.catch_warnings(record=True) as w:
                self.interpreter.run(i)
                for warning_message in w:
                    # TODO should this be stdout or stderr
                    stream_content = {'name': 'stdout', 'text': 'Warning: ' + str(warning_message.message) + '\n'}
                    self.send_response(self.iopub_socket, 'stream', stream_content)
        except iarm.exceptions.EndOfProgram as e:
            f_name = self.interpreter.program[self.interpreter.register['PC'] - 1].__name__
            f_name = f_name[:f_name.find('_')]
            message = "Error in {}: ".format(f_name)
            stream_content = {'name': 'stdout', 'text': message + str(e) + '\n'}
            self.send_response(self.iopub_socket, 'stream', stream_content)
        except Exception as e:
            for err in e.args:
                stream_content = {'name': 'stderr', 'text': str(err)}
                self.send_response(self.iopub_socket, 'stream', stream_content)
            return {'status': 'error',
                    'execution_count': self.execution_count,
                    'ename': type(e).__name__,
                    'evalue': str(e),
                    'traceback': '???'}
727290
def run_cell(self, cell):
        """"""
        logging.info('Running cell:\n%s\n', cell.input)
        self.kc.execute(cell.input)
        reply = self.kc.get_shell_msg()
        status = reply['content']['status']
        traceback_text = ''
        if status == 'error':
            traceback_text = 'Cell raised uncaught exception: \n' + \
                '\n'.join(reply['content']['traceback'])
            logging.info(traceback_text)
        else:
            logging.info('Cell returned')

        outs = list()
        while True:
            try:
                msg = self.kc.get_iopub_msg(timeout=1)
                if msg['msg_type'] == 'status':
                    if msg['content']['execution_state'] == 'idle':
                        break
            except Empty:
                # execution state should return to idle
                # before the queue becomes empty,
                # if it doesn't, something bad has happened
                raise

            content = msg['content']
            msg_type = msg['msg_type']

            # IPython 3.0.0-dev writes pyerr/pyout in the notebook format
            # but uses error/execute_result in the message spec. This does the
            # translation needed for tests to pass with IPython 3.0.0-dev
            notebook3_format_conversions = {
                'error': 'pyerr',
                'execute_result': 'pyout'
            }
            msg_type = notebook3_format_conversions.get(msg_type, msg_type)

            out = NotebookNode(output_type=msg_type)

            if 'execution_count' in content:
                cell['prompt_number'] = content['execution_count']
                out.prompt_number = content['execution_count']

            if msg_type in ('status', 'pyin', 'execute_input'):
                continue
            elif msg_type == 'stream':
                out.stream = content['name']
                # in msgspec 5, this is name, text
                # in msgspec 4, this is name, data
                if 'text' in content:
                    out.text = content['text']
                else:
                    out.text = content['data']
            elif msg_type in ('display_data', 'pyout'):
                for mime, data in content['data'].items():
                    try:
                        attr = self.MIME_MAP[mime]
                    except KeyError:
                        raise NotImplementedError(
                            'unhandled mime type: %s' % mime
                        )

                    # In notebook version <= 3 JSON data is stored as a string
                    # Evaluation of IPython2's JSON gives strings directly
                    # Therefore do not encode for IPython versions prior to 3
                    json_encode = (
                            IPython.version_info[0] >= 3 and
                            mime == "application/json")

                    data_out = data if not json_encode else json.dumps(data)
                    setattr(out, attr, data_out)
            elif msg_type == 'pyerr':
                out.ename = content['ename']
                out.evalue = content['evalue']
                out.traceback = content['traceback']
            elif msg_type == 'clear_output':
                outs = list()
                continue
            else:
                raise NotImplementedError(
                    'unhandled iopub message: %s' % msg_type
                )
            outs.append(out)
        cell['outputs'] = outs

        if status == 'error':
            raise NotebookError(traceback_text)
164014
def jupyter_notebook(script_blocks, gallery_conf):
    
    first_cell = gallery_conf.get("first_notebook_cell", "%matplotlib inline")
    work_notebook = jupyter_notebook_skeleton()
    if first_cell is not None:
        add_code_cell(work_notebook, first_cell)
    fill_notebook(work_notebook, script_blocks)

    return work_notebook
752813
def getCell(self, row, width=None):
        ''
        cellval = wrapply(self.getValue, row)
        typedval = wrapply(self.type, cellval)

        if isinstance(typedval, TypedWrapper):
            if isinstance(cellval, TypedExceptionWrapper):  # calc failed
                exc = cellval.exception
                if cellval.forwarded:
                    dispval = str(cellval)  # traceback.format_exception_only(type(exc), exc)[-1].strip()
                else:
                    dispval = options.disp_error_val
                return DisplayWrapper(cellval.val, error=exc.stacktrace,
                                        display=dispval,
                                        note=options.note_getter_exc,
                                        notecolor='color_error')
            elif typedval.val is None:  # early out for strict None
                return DisplayWrapper(None, display='',  # force empty display for None
                                            note=options.disp_note_none,
                                            notecolor='color_note_type')
            elif isinstance(typedval, TypedExceptionWrapper):  # calc succeeded, type failed
                return DisplayWrapper(typedval.val, display=str(cellval),
                                            error=typedval.exception.stacktrace,
                                            note=options.note_type_exc,
                                            notecolor='color_warning')
            else:
                return DisplayWrapper(typedval.val, display=str(typedval.val),
                                            note=options.note_type_exc,
                                            notecolor='color_warning')

        elif isinstance(typedval, threading.Thread):
            return DisplayWrapper(None,
                                display=options.disp_pending,
                                note=options.note_pending,
                                notecolor='color_note_pending')

        dw = DisplayWrapper(cellval)

        try:
            dw.display = self.format(typedval) or ''

            if width and isNumeric(self):
                dw.display = dw.display.rjust(width-1)

            # annotate cells with raw value type in anytype columns, except for strings
            if self.type is anytype and type(cellval) is not str:
                typedesc = typemap.get(type(cellval), None)
                dw.note = typedesc.icon if typedesc else options.note_unknown_type
                dw.notecolor = 'color_note_type'

        except Exception as e:  # formatting failure
            e.stacktrace = stacktrace()
            dw.error = e
            try:
                dw.display = str(cellval)
            except Exception as e:
                dw.display = str(e)
            dw.note = options.note_format_exc
            dw.notecolor = 'color_warning'

        return dw