from fastcore.test import *test
Run dialogs’ code messages through a live solveit,
nbdev-test-style
Testing a notebook with nbdev-test runs its cells under execnb, outside solveit: no dialog context, no current-message tracking, no dialog kernel. For notebooks written as solveit dialogs (like dialoghelper’s own), that tests a different thing than what runs in production. solveit-test instead asks a running solveit instance to execute a notebook exactly as the app does – the production runloop, a fresh dialog kernel, real current-message state – and reports the code messages whose outputs contain errors.
Running one dialog
test_dlg is the whole protocol: restart the dialog’s kernel (starting the dialog if needed, so every run begins from clean state), queue all its code messages in order – solveit’s own run-all semantics, including messages hidden from the AI – then poll each queued message’s run flag until the queue drains, and return the messages that ended up with error outputs. The run happens server-side, so each message executes with the dialog itself as context, exactly as when its cells were authored.
test_dlg
async def test_dlg(
dname:str, # Name/path of the dialog (relative to current dialog's folder, or absolute if starts with '/')
ids:list=None, # Message ids to run (default: all code messages, including those hidden from the AI)
timeout:int=600, # Max seconds to wait for the run to finish
poll:float=0.5, # Seconds between completion checks
save:bool=False, # Empty stored outputs of the messages to run first, so results reflect only this run?
)->list[dict]: # Run messages left with error outputs (empty means the dialog passed)Restart dname’s kernel, run its code messages through the solveit runloop, and return those that errored
To see it work we need a dialog on the live server, so we create a throwaway one with three code messages, the middle one broken. Note that the run continues past the failure – solveit’s run queue records the error and keeps going, so one bad cell doesn’t hide later ones:
tnm = f'/tmp_soltest_{os.urandom(4).hex()}'
await create_or_run_dialog(tnm, template=False)
for src in ('a=1', '1/0', 'print(a)'): await add_msg(src, msg_type='code', dname=tnm, placement='at_end')
errs = await test_dlg(tnm)
[m.id for m in errs]--------------------------------------------------------------------------- CancelledError Traceback (most recent call last) Cell In[15], line 2 1 tnm = f'/tmp_soltest_{os.urandom(4).hex()}' ----> 2 await create_or_run_dialog(tnm, template=False) 3 for src in ('a=1', '1/0', 'print(a)'): await add_msg(src, msg_type='code', dname=tnm, placement='at_end') 4 errs = await test_dlg(tnm) 5 [m.id for m in errs] File ~/aai-ws/fastaudit/fastaudit/core.py:64, in CallTracker.wrap.<locals>._fn(*args, **kwargs) 62 @wraps(fn) 63 async def _fn(*args, **kwargs): ---> 64 with self.track(fn, args, kwargs): return await fn(*args, **kwargs) File ~/aai-ws/dialoghelper/dialoghelper/core.py:889, in create_or_run_dialog(name, template) 887 "Create a new dialog, or set an existing one running" 888 name = find_dname(name).lstrip('/') --> 889 return await call_endpa('create_dialog_', name=name, template=template, json=True, required=False) File ~/aai-ws/dialoghelper/dialoghelper/core.py:134, in call_endpa(path, dname, json, raiseex, id, required, timeout, audit, chkerr, **data) 132 url, data, headers = _prep_endp(path, dname, json, id, data, required=required) 133 if audit: sys.audit("dialoghelper.endp", path, data) --> 134 return _handle_resp(await xposta(url, data=data, headers=headers, timeout=timeout), json, raiseex, chkerr) File ~/aai-ws/fastaudit/fastaudit/core.py:64, in CallTracker.wrap.<locals>._fn(*args, **kwargs) 62 @wraps(fn) 63 async def _fn(*args, **kwargs): ---> 64 with self.track(fn, args, kwargs): return await fn(*args, **kwargs) File ~/aai-ws/dialoghelper/dialoghelper/core.py:100, in xposta(url, **kwargs) 98 @allow 99 async def xposta(url, **kwargs): --> 100 async with AsyncClient() as c: return await c.post(url, **kwargs) File ~/aai-ws/.venv/lib/python3.13/site-packages/httpx/_client.py:1859, in AsyncClient.post(self, url, content, data, files, json, params, headers, cookies, auth, follow_redirects, timeout, extensions) 1838 async def post( 1839 self, 1840 url: URL | str, (...) 1852 extensions: RequestExtensions | None = None, 1853 ) -> Response: 1854 """ 1855 Send a `POST` request. 1856 1857 **Parameters**: See `httpx.request`. 1858 """ -> 1859 return await self.request( 1860 "POST", 1861 url, 1862 content=content, 1863 data=data, 1864 files=files, 1865 json=json, 1866 params=params, 1867 headers=headers, 1868 cookies=cookies, 1869 auth=auth, 1870 follow_redirects=follow_redirects, 1871 timeout=timeout, 1872 extensions=extensions, 1873 ) File ~/aai-ws/.venv/lib/python3.13/site-packages/httpx/_client.py:1540, in AsyncClient.request(self, method, url, content, data, files, json, params, headers, cookies, auth, follow_redirects, timeout, extensions) 1525 warnings.warn(message, DeprecationWarning, stacklevel=2) 1527 request = self.build_request( 1528 method=method, 1529 url=url, (...) 1538 extensions=extensions, 1539 ) -> 1540 return await self.send(request, auth=auth, follow_redirects=follow_redirects) File ~/aai-ws/.venv/lib/python3.13/site-packages/httpx/_client.py:1629, in AsyncClient.send(self, request, stream, auth, follow_redirects) 1625 self._set_timeout(request) 1627 auth = self._build_request_auth(request, auth) -> 1629 response = await self._send_handling_auth( 1630 request, 1631 auth=auth, 1632 follow_redirects=follow_redirects, 1633 history=[], 1634 ) 1635 try: 1636 if not stream: File ~/aai-ws/.venv/lib/python3.13/site-packages/httpx/_client.py:1657, in AsyncClient._send_handling_auth(self, request, auth, follow_redirects, history) 1654 request = await auth_flow.__anext__() 1656 while True: -> 1657 response = await self._send_handling_redirects( 1658 request, 1659 follow_redirects=follow_redirects, 1660 history=history, 1661 ) 1662 try: 1663 try: File ~/aai-ws/.venv/lib/python3.13/site-packages/httpx/_client.py:1694, in AsyncClient._send_handling_redirects(self, request, follow_redirects, history) 1691 for hook in self._event_hooks["request"]: 1692 await hook(request) -> 1694 response = await self._send_single_request(request) 1695 try: 1696 for hook in self._event_hooks["response"]: File ~/aai-ws/.venv/lib/python3.13/site-packages/httpx/_client.py:1730, in AsyncClient._send_single_request(self, request) 1725 raise RuntimeError( 1726 "Attempted to send an sync request with an AsyncClient instance." 1727 ) 1729 with request_context(request=request): -> 1730 response = await transport.handle_async_request(request) 1732 assert isinstance(response.stream, AsyncByteStream) 1733 response.request = request File ~/aai-ws/.venv/lib/python3.13/site-packages/httpx/_transports/default.py:394, in AsyncHTTPTransport.handle_async_request(self, request) 381 req = httpcore.Request( 382 method=request.method, 383 url=httpcore.URL( (...) 391 extensions=request.extensions, 392 ) 393 with map_httpcore_exceptions(): --> 394 resp = await self._pool.handle_async_request(req) 396 assert isinstance(resp.stream, typing.AsyncIterable) 398 return Response( 399 status_code=resp.status, 400 headers=resp.headers, 401 stream=AsyncResponseStream(resp.stream), 402 extensions=resp.extensions, 403 ) File ~/aai-ws/.venv/lib/python3.13/site-packages/httpcore/_async/connection_pool.py:256, in AsyncConnectionPool.handle_async_request(self, request) 253 closing = self._assign_requests_to_connections() 255 await self._close_connections(closing) --> 256 raise exc from None 258 # Return the response. Note that in this case we still have to manage 259 # the point at which the response is closed. 260 assert isinstance(response.stream, typing.AsyncIterable) File ~/aai-ws/.venv/lib/python3.13/site-packages/httpcore/_async/connection_pool.py:236, in AsyncConnectionPool.handle_async_request(self, request) 232 connection = await pool_request.wait_for_connection(timeout=timeout) 234 try: 235 # Send the request on the assigned connection. --> 236 response = await connection.handle_async_request( 237 pool_request.request 238 ) 239 except ConnectionNotAvailable: 240 # In some cases a connection may initially be available to 241 # handle a request, but then become unavailable. 242 # 243 # In this case we clear the connection and try again. 244 pool_request.clear_connection() File ~/aai-ws/.venv/lib/python3.13/site-packages/httpcore/_async/connection.py:103, in AsyncHTTPConnection.handle_async_request(self, request) 100 self._connect_failed = True 101 raise exc --> 103 return await self._connection.handle_async_request(request) File ~/aai-ws/.venv/lib/python3.13/site-packages/httpcore/_async/http11.py:136, in AsyncHTTP11Connection.handle_async_request(self, request) 134 async with Trace("response_closed", logger, request) as trace: 135 await self._response_closed() --> 136 raise exc File ~/aai-ws/.venv/lib/python3.13/site-packages/httpcore/_async/http11.py:106, in AsyncHTTP11Connection.handle_async_request(self, request) 95 pass 97 async with Trace( 98 "receive_response_headers", logger, request, kwargs 99 ) as trace: 100 ( 101 http_version, 102 status, 103 reason_phrase, 104 headers, 105 trailing_data, --> 106 ) = await self._receive_response_headers(**kwargs) 107 trace.return_value = ( 108 http_version, 109 status, 110 reason_phrase, 111 headers, 112 ) 114 network_stream = self._network_stream File ~/aai-ws/.venv/lib/python3.13/site-packages/httpcore/_async/http11.py:177, in AsyncHTTP11Connection._receive_response_headers(self, request) 174 timeout = timeouts.get("read", None) 176 while True: --> 177 event = await self._receive_event(timeout=timeout) 178 if isinstance(event, h11.Response): 179 break File ~/aai-ws/.venv/lib/python3.13/site-packages/httpcore/_async/http11.py:217, in AsyncHTTP11Connection._receive_event(self, timeout) 214 event = self._h11_state.next_event() 216 if event is h11.NEED_DATA: --> 217 data = await self._network_stream.read( 218 self.READ_NUM_BYTES, timeout=timeout 219 ) 221 # If we feed this case through h11 we'll raise an exception like: 222 # 223 # httpcore.RemoteProtocolError: can't handle event type (...) 227 # perspective. Instead we handle this case distinctly and treat 228 # it as a ConnectError. 229 if data == b"" and self._h11_state.their_state == h11.SEND_RESPONSE: File ~/aai-ws/.venv/lib/python3.13/site-packages/httpcore/_backends/anyio.py:35, in AnyIOStream.read(self, max_bytes, timeout) 33 with anyio.fail_after(timeout): 34 try: ---> 35 return await self._stream.receive(max_bytes=max_bytes) 36 except anyio.EndOfStream: # pragma: nocover 37 return b"" File ~/aai-ws/.venv/lib/python3.13/site-packages/anyio/_backends/_asyncio.py:1337, in SocketStream.receive(self, max_bytes) 1331 if ( 1332 not self._protocol.read_event.is_set() 1333 and not self._transport.is_closing() 1334 and not self._protocol.is_at_eof 1335 ): 1336 self._transport.resume_reading() -> 1337 await self._protocol.read_event.wait() 1338 self._transport.pause_reading() 1339 else: File ~/.local/share/uv/python/cpython-3.13.15-macos-aarch64-none/lib/python3.13/asyncio/locks.py:213, in Event.wait(self) 211 self._waiters.append(fut) 212 try: --> 213 await fut 214 return True 215 finally: CancelledError:
Exactly one message failed, and its output carries the traceback:
test_eq(len(errs), 1)
assert 'ZeroDivisionError' in str(errs[0].output)
errs[0].outputA dialog whose run leaves no error outputs passes. save=True empties the stored outputs of the messages about to run, so nothing from an earlier run can masquerade as this one’s result. And the scratch dialog cleans up fully – kernel stopped, file removed:
await del_msgs(errs[0].id, dname=tnm)
test_eq(await test_dlg(tnm, save=True), [])
await stop_dialog(tnm)
await rm_dialog(tnm)The command line
The solveit-test entrypoint maps each notebook path to its dialog name through the server’s own base path (so the notebooks must live under the tree the instance serves), runs each in turn, and reports nbdev-test-style. Which messages run is aidialog’s Dialog.select_msgs, the same selection Dialog.execute uses: code messages only (a prompt never runs), honoring the eval cascade — skip_exec: true frontmatter skips a whole notebook, #|eval: false (comment or meta form) skips a cell, and nbdev_export cells never run. A dialog that’s open in solveit gets its kernel restarted by its test run, so state you had in that session is lost – that’s the price of every run starting clean. Runs execute the notebooks’ code for real, mutating messages and outputs on disk, so run it on a clean checkout and review the diff: an example that doesn’t clean up after itself shows up there, which is itself worth knowing. Since only messages that execute write outputs, a message skipped mid-run (a crashed kernel, a queue hiccup) keeps its stored output, and the error report can’t tell it from a fresh failure. --save (like nbdev-test’s) empties the stored outputs of the messages about to run first, so outputs and report reflect this run alone.
test_nbs
async def test_nbs(
path:str='.', # An .ipynb file, or a directory of them, to test
timeout:int=600, # Max seconds to wait per dialog
keep:bool=False, # Leave dialog kernels running after their test?
n_workers:int=None, # Max dialogs tested concurrently (default: min(num_cpus(), 8))
save:bool=False, # Empty stored outputs of the messages to run first, so results reflect only this run?
)->dict: # Failures per notebook name: erroring message ids, or a repr'd exceptionTest each notebook under path as a dialog on the local solveit instance, printing progress nbdev-test-style
solveit_test
async def solveit_test(
path:str='.', # An .ipynb file, or a directory of them, to test
timeout:int=600, # Max seconds to wait per dialog
keep:bool=False, # Leave dialog kernels running after their test?
n_workers:int=None, # Max dialogs tested concurrently (default: min(num_cpus(), 8))
save:bool=False, # Empty stored outputs of the messages to run first, so results reflect only this run?
):Run notebooks as dialogs on the local solveit instance, nbdev-test-style; they must live under its data path