from fastcore.test import *test
Run dialogs’ code messages through a live solveit,
nbdev-test-style
solveit-test tests notebooks through a running solveit instance. It uses solveit’s runloop and dialog kernels, with current-message tracking and dialog context. nbdev-test uses execnb outside solveit, where these features aren’t available. Use solveit-test for notebooks that depend on them, such as dialoghelper’s own.
Running one dialog
test_dlg restarts the dialog’s kernel, starting the dialog if needed. It queues code messages in order and waits for them to finish. It returns the messages with error outputs.
By default, it runs all code messages, including those hidden from the AI. Pass ids to select messages explicitly. Each message runs in its dialog’s context on the server.
test_dlg
async def test_dlg(
dname:str, # Name/path of the dialog (relative to current dialog's folder, or absolute if starts with '/')
ids:list=None, # Message ids to run (default: all code messages, including those hidden from the AI)
timeout:int=600, # Max seconds to wait for the run to finish
poll:float=0.5, # Seconds between completion checks
save:bool=False, # Empty stored outputs of the messages to run first, so results reflect only this run?
)->list[dict]: # Run messages left with error outputs (empty means the dialog passed)Restart dname’s kernel, run its code messages through the solveit runloop, and return those that errored
Create a temporary dialog on the live server with an error in its second message. The run continues after the error.
tnm = f'/tmp_soltest_{os.urandom(4).hex()}.ipynb'
await create_or_run_dialog(tnm, template=False)
for src in ('a=1', '1/0', 'print(a)'): await add_msg(src, msg_type='code', dname=tnm, placement='at_end')
errs = await test_dlg(tnm)
[m.id for m in errs]--------------------------------------------------------------------------- CancelledError Traceback (most recent call last) Cell In[15], line 2 1 tnm = f'/tmp_soltest_{os.urandom(4).hex()}' ----> 2 await create_or_run_dialog(tnm, template=False) 3 for src in ('a=1', '1/0', 'print(a)'): await add_msg(src, msg_type='code', dname=tnm, placement='at_end') 4 errs = await test_dlg(tnm) 5 [m.id for m in errs] File ~/aai-ws/fastaudit/fastaudit/core.py:64, in CallTracker.wrap.<locals>._fn(*args, **kwargs) 62 @wraps(fn) 63 async def _fn(*args, **kwargs): ---> 64 with self.track(fn, args, kwargs): return await fn(*args, **kwargs) File ~/aai-ws/dialoghelper/dialoghelper/core.py:889, in create_or_run_dialog(name, template) 887 "Create a new dialog, or set an existing one running" 888 name = find_dname(name).lstrip('/') --> 889 return await call_endpa('create_dialog_', name=name, template=template, json=True, required=False) File ~/aai-ws/dialoghelper/dialoghelper/core.py:134, in call_endpa(path, dname, json, raiseex, id, required, timeout, audit, chkerr, **data) 132 url, data, headers = _prep_endp(path, dname, json, id, data, required=required) 133 if audit: sys.audit("dialoghelper.endp", path, data) --> 134 return _handle_resp(await xposta(url, data=data, headers=headers, timeout=timeout), json, raiseex, chkerr) File ~/aai-ws/fastaudit/fastaudit/core.py:64, in CallTracker.wrap.<locals>._fn(*args, **kwargs) 62 @wraps(fn) 63 async def _fn(*args, **kwargs): ---> 64 with self.track(fn, args, kwargs): return await fn(*args, **kwargs) File ~/aai-ws/dialoghelper/dialoghelper/core.py:100, in xposta(url, **kwargs) 98 @allow 99 async def xposta(url, **kwargs): --> 100 async with AsyncClient() as c: return await c.post(url, **kwargs) File ~/aai-ws/.venv/lib/python3.13/site-packages/httpx/_client.py:1859, in AsyncClient.post(self, url, content, data, files, json, params, headers, cookies, auth, follow_redirects, timeout, extensions) 1838 async def post( 1839 self, 1840 url: URL | str, (...) 1852 extensions: RequestExtensions | None = None, 1853 ) -> Response: 1854 """ 1855 Send a `POST` request. 1856 1857 **Parameters**: See `httpx.request`. 1858 """ -> 1859 return await self.request( 1860 "POST", 1861 url, 1862 content=content, 1863 data=data, 1864 files=files, 1865 json=json, 1866 params=params, 1867 headers=headers, 1868 cookies=cookies, 1869 auth=auth, 1870 follow_redirects=follow_redirects, 1871 timeout=timeout, 1872 extensions=extensions, 1873 ) File ~/aai-ws/.venv/lib/python3.13/site-packages/httpx/_client.py:1540, in AsyncClient.request(self, method, url, content, data, files, json, params, headers, cookies, auth, follow_redirects, timeout, extensions) 1525 warnings.warn(message, DeprecationWarning, stacklevel=2) 1527 request = self.build_request( 1528 method=method, 1529 url=url, (...) 1538 extensions=extensions, 1539 ) -> 1540 return await self.send(request, auth=auth, follow_redirects=follow_redirects) File ~/aai-ws/.venv/lib/python3.13/site-packages/httpx/_client.py:1629, in AsyncClient.send(self, request, stream, auth, follow_redirects) 1625 self._set_timeout(request) 1627 auth = self._build_request_auth(request, auth) -> 1629 response = await self._send_handling_auth( 1630 request, 1631 auth=auth, 1632 follow_redirects=follow_redirects, 1633 history=[], 1634 ) 1635 try: 1636 if not stream: File ~/aai-ws/.venv/lib/python3.13/site-packages/httpx/_client.py:1657, in AsyncClient._send_handling_auth(self, request, auth, follow_redirects, history) 1654 request = await auth_flow.__anext__() 1656 while True: -> 1657 response = await self._send_handling_redirects( 1658 request, 1659 follow_redirects=follow_redirects, 1660 history=history, 1661 ) 1662 try: 1663 try: File ~/aai-ws/.venv/lib/python3.13/site-packages/httpx/_client.py:1694, in AsyncClient._send_handling_redirects(self, request, follow_redirects, history) 1691 for hook in self._event_hooks["request"]: 1692 await hook(request) -> 1694 response = await self._send_single_request(request) 1695 try: 1696 for hook in self._event_hooks["response"]: File ~/aai-ws/.venv/lib/python3.13/site-packages/httpx/_client.py:1730, in AsyncClient._send_single_request(self, request) 1725 raise RuntimeError( 1726 "Attempted to send an sync request with an AsyncClient instance." 1727 ) 1729 with request_context(request=request): -> 1730 response = await transport.handle_async_request(request) 1732 assert isinstance(response.stream, AsyncByteStream) 1733 response.request = request File ~/aai-ws/.venv/lib/python3.13/site-packages/httpx/_transports/default.py:394, in AsyncHTTPTransport.handle_async_request(self, request) 381 req = httpcore.Request( 382 method=request.method, 383 url=httpcore.URL( (...) 391 extensions=request.extensions, 392 ) 393 with map_httpcore_exceptions(): --> 394 resp = await self._pool.handle_async_request(req) 396 assert isinstance(resp.stream, typing.AsyncIterable) 398 return Response( 399 status_code=resp.status, 400 headers=resp.headers, 401 stream=AsyncResponseStream(resp.stream), 402 extensions=resp.extensions, 403 ) File ~/aai-ws/.venv/lib/python3.13/site-packages/httpcore/_async/connection_pool.py:256, in AsyncConnectionPool.handle_async_request(self, request) 253 closing = self._assign_requests_to_connections() 255 await self._close_connections(closing) --> 256 raise exc from None 258 # Return the response. Note that in this case we still have to manage 259 # the point at which the response is closed. 260 assert isinstance(response.stream, typing.AsyncIterable) File ~/aai-ws/.venv/lib/python3.13/site-packages/httpcore/_async/connection_pool.py:236, in AsyncConnectionPool.handle_async_request(self, request) 232 connection = await pool_request.wait_for_connection(timeout=timeout) 234 try: 235 # Send the request on the assigned connection. --> 236 response = await connection.handle_async_request( 237 pool_request.request 238 ) 239 except ConnectionNotAvailable: 240 # In some cases a connection may initially be available to 241 # handle a request, but then become unavailable. 242 # 243 # In this case we clear the connection and try again. 244 pool_request.clear_connection() File ~/aai-ws/.venv/lib/python3.13/site-packages/httpcore/_async/connection.py:103, in AsyncHTTPConnection.handle_async_request(self, request) 100 self._connect_failed = True 101 raise exc --> 103 return await self._connection.handle_async_request(request) File ~/aai-ws/.venv/lib/python3.13/site-packages/httpcore/_async/http11.py:136, in AsyncHTTP11Connection.handle_async_request(self, request) 134 async with Trace("response_closed", logger, request) as trace: 135 await self._response_closed() --> 136 raise exc File ~/aai-ws/.venv/lib/python3.13/site-packages/httpcore/_async/http11.py:106, in AsyncHTTP11Connection.handle_async_request(self, request) 95 pass 97 async with Trace( 98 "receive_response_headers", logger, request, kwargs 99 ) as trace: 100 ( 101 http_version, 102 status, 103 reason_phrase, 104 headers, 105 trailing_data, --> 106 ) = await self._receive_response_headers(**kwargs) 107 trace.return_value = ( 108 http_version, 109 status, 110 reason_phrase, 111 headers, 112 ) 114 network_stream = self._network_stream File ~/aai-ws/.venv/lib/python3.13/site-packages/httpcore/_async/http11.py:177, in AsyncHTTP11Connection._receive_response_headers(self, request) 174 timeout = timeouts.get("read", None) 176 while True: --> 177 event = await self._receive_event(timeout=timeout) 178 if isinstance(event, h11.Response): 179 break File ~/aai-ws/.venv/lib/python3.13/site-packages/httpcore/_async/http11.py:217, in AsyncHTTP11Connection._receive_event(self, timeout) 214 event = self._h11_state.next_event() 216 if event is h11.NEED_DATA: --> 217 data = await self._network_stream.read( 218 self.READ_NUM_BYTES, timeout=timeout 219 ) 221 # If we feed this case through h11 we'll raise an exception like: 222 # 223 # httpcore.RemoteProtocolError: can't handle event type (...) 227 # perspective. Instead we handle this case distinctly and treat 228 # it as a ConnectError. 229 if data == b"" and self._h11_state.their_state == h11.SEND_RESPONSE: File ~/aai-ws/.venv/lib/python3.13/site-packages/httpcore/_backends/anyio.py:35, in AnyIOStream.read(self, max_bytes, timeout) 33 with anyio.fail_after(timeout): 34 try: ---> 35 return await self._stream.receive(max_bytes=max_bytes) 36 except anyio.EndOfStream: # pragma: nocover 37 return b"" File ~/aai-ws/.venv/lib/python3.13/site-packages/anyio/_backends/_asyncio.py:1337, in SocketStream.receive(self, max_bytes) 1331 if ( 1332 not self._protocol.read_event.is_set() 1333 and not self._transport.is_closing() 1334 and not self._protocol.is_at_eof 1335 ): 1336 self._transport.resume_reading() -> 1337 await self._protocol.read_event.wait() 1338 self._transport.pause_reading() 1339 else: File ~/.local/share/uv/python/cpython-3.13.15-macos-aarch64-none/lib/python3.13/asyncio/locks.py:213, in Event.wait(self) 211 self._waiters.append(fut) 212 try: --> 213 await fut 214 return True 215 finally: CancelledError:
Check that the failed message contains the traceback.
test_eq(len(errs), 1)
assert 'ZeroDivisionError' in str(errs[0].output)
errs[0].outputRemove the failing message and run again. save=True clears stored outputs before the run. After checking the result, stop the kernel and remove the temporary dialog.
await del_msgs(errs[0].id, dname=tnm)
test_eq(await test_dlg(tnm, save=True), [])
await stop_dialog(tnm)
await rm_dialog(tnm)The command line
solveit-test requires a running solveit instance. The notebooks must be under the directory that instance serves. The command maps file paths to dialog names and tests notebooks concurrently.
The command uses aidialog’s Dialog.select_msgs, matching Dialog.execute. It selects code messages, never prompts, with these exclusions:
skip_exec: truein frontmatter skips the notebook.eval: falsein a cell directive or metadata skips the cell.nbdev_exportcells never run.
Testing restarts each dialog’s kernel, including dialogs you have open in solveit. You lose their existing kernel state. Runs change messages and outputs on disk. Use a clean checkout and review the diff afterward.
A kernel crash or queue problem can prevent a selected message from executing. Its stored output can still contain an error from an earlier run. The report cannot distinguish that error from a new one. Use --save to clear the selected messages’ outputs before running them.
test_nbs
async def test_nbs(
path:str='.', # An .ipynb file, or a directory of them, to test
timeout:int=600, # Max seconds to wait per dialog
keep:bool=False, # Leave dialog kernels running after their test?
n_workers:int=None, # Max dialogs tested concurrently (default: min(num_cpus(), 8))
save:bool=False, # Empty stored outputs of the messages to run first, so results reflect only this run?
)->dict: # Failures per notebook name: erroring message ids, or a repr'd exceptionTest each notebook under path as a dialog on the local solveit instance, printing progress nbdev-test-style
solveit_test
async def solveit_test(
path:str='.', # An .ipynb file, or a directory of them, to test
timeout:int=600, # Max seconds to wait per dialog
keep:bool=False, # Leave dialog kernels running after their test?
n_workers:int=None, # Max dialogs tested concurrently (default: min(num_cpus(), 8))
save:bool=False, # Empty stored outputs of the messages to run first, so results reflect only this run?
):Run notebooks as dialogs on the local solveit instance, nbdev-test-style; they must live under its data path