ipynb import and export

Reading and writing dialogs as Jupyter notebooks

Dialogs are stored as Jupyter notebooks (.ipynb files), so any notebook tool can open them. This module handles the conversion:

Message Type	Cell Type	Notes
`note`	markdown	Direct mapping
`prompt`	markdown	Has `solveit_ai` metadata; content + AI response joined with separator
`code`	code	Outputs preserved as standard notebook outputs
`raw`	raw	Direct mapping

Writing converts Dialog -> .ipynb, Reading does the reverse.

import random,os
from tempfile import mkdtemp
from fastcore.test import *

random.seed(7)
tstdir = Path(mkdtemp())
dlg = Dialog('dlg')
nt_msg = dlg.mk_message('A *test* dialog', msg_type=snote)
nt_msg.mk_attachment(b'not really a png', 'image/png')
code_msg = dlg.mk_message('1+1', msg_type=scode, output=code_output('2'))
ai_msg = dlg.mk_message('Add them.', msg_type=sprompt, output='The answer is **2**.')
raw_msg = dlg.mk_message('plain text', msg_type=sraw)
dlg

dlg

A test dialog
1+1 ⇒ [{‘output_type’: ‘execute_result’, ‘metadata’: {}, ‘data’: …
Add them. ⇒ [{‘output_type’: ‘display_data’, ‘metadata’: {‘is_ai_res’: …
plain text

Writing

Attachments look like this:

  {
   "attachments": {
    "image.png": {
     "image/png": "iVBO...kJggg=="
    }
  },

source

att2dict

def att2dict(
    att
):

Call self as a function.

Prompts are special: they contain both user input and AI response in a single markdown cell. We:

Mark them with solveit_ai: true in cell metadata
Join content + response with a separator (reply_sep) when writing
Split on that separator when reading back

The separator includes a hidden HTML comment to avoid collisions with normal content.

source

split_cell_src

def split_cell_src(
    cell
):

Split cell source into (content, ai_reply_or_None)

source

Message.cell_meta

def cell_meta():

Metadata dict to write: meta plus demoted meta_attrs fields, falsy values omitted

cell_meta assembles everything to_cell will write as cell metadata, and is the override point when a host’s in-memory types differ from the file’s: convert after super(), before nbformat validates the cell (solveit stores its UI flags as ints, but the ipynb schema types collapsed/hide_input as boolean).

source

Message.to_cell

def to_cell(
    version:int=2
):

Convert message to an nbformat cell

test_eq(Message().to_cell()['metadata'],{})

meta_attrs is how a host teaches serialization about its own fields without this library knowing them: declare attribute → metadata key, and cell_meta/cell2msg demote/promote them, with falsy values omitted from the file. Anything not declared still round-trips untouched inside meta:

class NoteMsg(Message): meta_attrs = dict(bookmark='bookmark')
class NoteDlg(Dialog): msg_cls = NoteMsg

bookmark_cell = NoteMsg(bookmark=9).to_cell()
test_eq(bookmark_cell['metadata']['bookmark'], 9)
test_eq(NoteMsg().to_cell()['metadata'], {})  # default/absent values aren't written

When a host’s in-memory type differs from what the file should carry (the ipynb schema types some keys, and validation runs as the cell is built), it converts in a cell_meta override:

class FlagMsg(Message):
    meta_attrs = dict(collapsed='collapsed')
    def cell_meta(self): return {k: bool(v) for k,v in super().cell_meta().items()}

assert FlagMsg(collapsed=1).to_cell()['metadata']['collapsed'] is True

# Prompts serialize as markdown with the reply appended after `reply_sep`
pr_msg = dlg.mk_message('What is 2+2?', output='The answer is 4.', msg_type='prompt')
pr_cell = pr_msg.to_cell()
test_eq(pr_cell['cell_type'], 'markdown')
test_eq(pr_cell['metadata']['solveit_ai'], True)
assert 'What is 2+2?' in pr_cell['source']
assert reply_sep in pr_cell['source']
assert 'The answer is 4.' in pr_cell['source']

# A prompt without a reply gets no separator
pr_empty = dlg.mk_message('Hello?', output='', msg_type='prompt')
assert reply_sep not in pr_empty.to_cell()['source']

source

get_ipynb

def get_ipynb(
    dlg:Dialog, version:int=2, msgs:NoneType=None
):

Notebook object for dlg; msgs defaults to all its messages

source

write_ipynb

def write_ipynb(
    dlg:Dialog, fname:NoneType=None, version:int=2, msgs:NoneType=None, **kwargs
):

Write dlg as a notebook, or return the JSON string if fname is None; kwargs (e.g. uid/gid) pass to atomic_save

write_ipynb(dlg, fname=tstdir/'dlg.ipynb', version=2)

source

Dialog.write

def write(
    base_path, version:int=2, msgs:NoneType=None, **kwargs
):

Call self as a function.

dlg.write(tstdir)

source

ipynb_cells

def ipynb_cells(
    path, nm, prefix:NoneType=None, suffix:NoneType=None
):

Call self as a function.

test_eq(len(ipynb_cells(tstdir, dlg.name)), len(dlg.messages))
ipynb_cells(tstdir, dlg.name)[0]

{'attachments': {'2530bb1d-6d13-4cde-8623-7b2ed91e3f72': {'image/png': 'bm90IHJlYWxseSBhIHBuZw=='}},
 'cell_type': 'markdown',
 'id': 'a54dca18',
 'metadata': {},
 'source': 'A *test* dialog'}

Reading

source

dict2att

def dict2att(
    att_id, att_data
):

Convert attachment dict to Attachment object

source

Dialog.cell2msg

def cell2msg(
    cell
):

Convert single notebook cell to message object

back = NoteDlg('t').cell2msg(bookmark_cell)
test_eq(back.bookmark, 9)
test_eq(back.meta, {})  # promoted out of `meta` into the attribute

# Test roundtrip prompts
pr_back = dlg.cell2msg(pr_cell)
test_eq(pr_back.content, 'What is 2+2?')
test_eq(pr_back.msg_type, 'prompt')
test_eq(pr_back.ai_res, 'The answer is 4.')

source

read_ipynb

def read_ipynb(
    fname, cls:type=Dialog, name:NoneType=None
):

Read a dialog from notebook file fname (.ipynb added if missing), constructing via cls; name defaults to the file stem

source

Dialog.from_cells

def from_cells(
    cells
):

Call self as a function.

dlg = read_ipynb(tstdir/'dlg')
dlg

dlg

A test dialog
1+1 ⇒ [{‘data’: {‘text/plain’: ‘2’}, ‘execution_count’: 1, ’metad…
Add them. ⇒ [{‘output_type’: ‘display_data’, ‘metadata’: {‘is_ai_res’: …
plain text
What is 2+2? ⇒ [{‘output_type’: ‘display_data’, ‘metadata’: {‘is_ai_res’: …
Hello?

source

Dialog.save

def save(
    fname:NoneType=None
):

Write back to fname, or to the path_ stamped by read_ipynb

read_ipynb stamps the source path as path_ (the trailing-underscore working-attribute convention, staying clear of hosts’ own path properties), so load-edit-save needs no path threading. Saving an unedited dialog is a byte no-op:

before = (tstdir/'dlg.ipynb').read_text()
dlg.save()
test_eq((tstdir/'dlg.ipynb').read_text(), before)
assert isinstance(dlg.messages, Msgs)
with expect_fail(ValueError, 'path_'): Dialog('unread').save()

Metadata preservation

The library itself names no host fields: everything in cell metadata that a meta_attrs declaration doesn’t claim rides verbatim in Message.meta, and notebook-level metadata rides in Dialog.meta the same way. So a file written by any host survives a read/write round trip through the plain classes with every annotation intact:

m0 = dlg.messages[0]
m0.meta['my_app_flag'] = dict(level=3)
dlg.meta['my_app'] = dict(version=1)
dlg.write(tstdir)
dlg = read_ipynb(tstdir/'dlg')
test_eq(dlg.messages[0].meta['my_app_flag'], dict(level=3))
test_eq(dlg.meta['my_app'], dict(version=1))

Round-trip fidelity

For schema-valid files, read/write is byte-identical: every repo notebook (written by Jupyter and nbdev, not by us) reproduces exactly. So opening a dialog and saving it never creates diff noise, and any byte change in a file means a real content change.

for p in sorted(Path('.').glob('0*.ipynb')): test_eq(write_ipynb(read_ipynb(str(p))), p.read_text())

Schema-invalid files (hand-edited internals happen: this stray outputs key on a markdown cell is from a real file) aren’t preserved but healed, because Message has no slot for the invalid part. The content survives; the invalidity doesn’t:

bad_cells = [dict(cell_type='markdown', id='aaaa1111', metadata={}, source='hi', outputs=[])]
(tstdir/'bad.ipynb').write_text(json.dumps(dict(nbformat=4, nbformat_minor=5, metadata={}, cells=bad_cells)))
with expect_fail(NotebookValidationError): nbformat.validate(nbformat.read(tstdir/'bad.ipynb', as_version=4))
healed = write_ipynb(read_ipynb(tstdir/'bad'))
nbformat.validate(nbformat.reads(healed, as_version=4))
assert 'hi' in healed