API

API for ipykernel-helper

from fastcore.test import *
from fastcore.utils import *

from pprint import pprint

InteractiveShell helpers

s = "Some long string that will be truncated"
print(_safe_repr(s, max_len=20))

Some long string tha…

o = dict(name="Example", data=[1,2,3,4,5] * 5, nested={"a": 1, "b": 2, "c": [3, 4, 5] * 10})
print(_safe_repr(o, max_len=40))

{'name': 'Example', 'data': [1, 2, 3, 4,…

InteractiveShell.user_items


def user_items(
    max_len:int=200, xtra_skip:tuple=()
):

Get user-defined vars & funcs from namespace.

# ipy = get_ipython()
# _vs,_fs = ipy.user_items()
# pprint(_vs)
# print('---')
# pprint(_fs)

InteractiveShell.ranked_complete


def ranked_complete(
    code, line_no:NoneType=None, col_no:NoneType=None
):

Call self as a function.

from random import random

ipy = get_ipython()

def range_ex(
    a:str # some param
):
    "some func docstring"
    ...
ipy.ranked_complete('rang')

[{'text': 'range',
  'type': 'class',
  'signature': '',
  'start': 0,
  'end': 4,
  'mod': None,
  'rank': 5},
 {'text': 'range_ex',
  'type': 'function',
  'signature': '(a: str)',
  'start': 0,
  'end': 4,
  'mod': '__main__',
  'rank': 2},
 {'text': 'range_of',
  'type': 'function',
  'signature': '(a, b=None, step=None)',
  'start': 0,
  'end': 4,
  'mod': 'fastcore.basics',
  'rank': 5}]

res = ipy.ranked_complete('a="foo"\na.', 2, 3)
res[:2]

[{'text': 'capitalize',
  'type': 'function',
  'signature': '() -> str',
  'start': 10,
  'end': 10,
  'mod': None,
  'rank': 5},
 {'text': 'casefold',
  'type': 'function',
  'signature': '() -> str',
  'start': 10,
  'end': 10,
  'mod': None,
  'rank': 5}]

class Foo:
    def __dir__(self): return ['bar', 'baz', '_secret', 'quux']
    bar = 42
    baz = "hello"

f = Foo()
[o['text'] for o in ipy.ranked_complete('f.')]

['bar', 'baz', 'quux']

InteractiveShell.get_vars


def get_vars(
    vs:list, literal:bool=True
):

Get variables from namespace.

x, y, fp = 3, 4, open('./00_core.ipynb')
def add(a,b): return a+b

test_eq(ipy.get_vars(['x', 'y', 'fp']).values(), [x,y,str(fp)])
test_eq(ipy.get_vars(['x', 'y', 'fp'], False).values(), [str(x),str(y),str(fp)])

InteractiveShell.eval_exprs


def eval_exprs(
    vs:list, literal:bool=True
):

Evaluate expressions in namespace.

print(ipy.eval_exprs(['1/0'])['1/0'])

<error type="ZeroDivisionError" desc="division by zero">
Traceback (most recent call last):
  File "/var/folders/51/b2_szf2945n072c0vj2cyty40000gn/T/ipykernel_57170/2788349185.py", line 6, in eval_exprs
    try: res[v] = _maybe_eval(eval(v, ns)) if literal else str(eval(v, ns))
                              ^^^^^^^^^^^
  File "<string>", line 1, in <module>
ZeroDivisionError: division by zero
</error>

test_eq(ipy.eval_exprs(['x', 'y']), {'x': 3, 'y': 4})
test_eq(ipy.eval_exprs(['add(1,2)']), {'add(1,2)': 3})
test_eq(ipy.eval_exprs(['x+y']), {'x+y': 7})
test_eq(ipy.eval_exprs(['[x, y]']), {'[x, y]': [3, 4]})
test(ipy.eval_exprs(['undefined_var'])['undefined_var'],"NameError: name 'undefined_var' is not defined",operator.contains)

InteractiveShell.get_schemas


def get_schemas(
    fs:list
):

Get schemas from namespace.

ipy.get_schemas(['range_ex'])

{'range_ex': {'type': 'function',
  'function': {'name': 'range_ex',
   'description': 'some func docstring',
   'parameters': {'type': 'object',
    'properties': {'a': {'type': 'string', 'description': 'some param'}},
    'required': ['a']}}}}

Errors are passed back as strings:

def add(a:int,b:int): return a + b
ipy.get_schemas(['add'])

{'add': '`add`: Docstring missing!.'}

ipy.get_schemas(['div'])

{'div': '`div` not found. Did you run it?'}

Dotted names (like obj.method) are supported for getting schemas from object attributes:

class Calculator:
    def add(self, a:int, b:int) -> int:
        "Add two numbers"
        return a + b

calc = Calculator()
ipy.get_schemas(['calc.add'])

{'calc.add': {'type': 'function',
  'function': {'name': 'calc-add',
   'description': 'Add two numbers\n\nReturns:\n- type: integer',
   'parameters': {'type': 'object',
    'properties': {'a': {'type': 'integer', 'description': ''},
     'b': {'type': 'integer', 'description': ''}},
    'required': ['a', 'b']}}}}

InteractiveShell.xpush


def xpush(
    interactive:bool=False, kw:VAR_KEYWORD
):

Like push, but with kwargs

ipy.push(dict(a=2))
a

# ipykernel_helper version uses `**kwargs`
ipy.xpush(a=3)
a

The main benefits of using ipy.push(dict(a=2)) over directly executing code are:

Bulk variable assignment - You can set multiple variables at once with a single command
Programmatic variable injection - It provides a way to inject variables into the namespace from another context or function
No execution history - Variables are added without creating an entry in the execution history
No side effects - It’s a “pure” namespace modification without executing any code that might have side effects

There are several interesting functions in the IPython interpreter object that are useful for notebook development and interactive computing:

reset/reset_selective - Clear variables from the namespace (either all or selectively)
run_cell/run_cell_async - Execute code in a cell programmatically
set_next_input - Programmatically set the content of the next cell
system/system_raw/system_piped - Execute shell commands with different output handling
run_line_magic/run_cell_magic - Execute IPython magics programmatically
set_custom_exc - Set custom exception handlers

Signatures

res = ipy._sig_jedi('truncstr("a",', 1, 13)
r = res[0]
test_eq(r['params'][r['idx']]['name'], 'maxlen')
res

[{'label': 'def truncstr',
  'typ': 'function',
  'mod': 'fastcore.xtras',
  'doc': "truncstr(s: 'str', maxlen: 'int', suf: 'str'='…', space='', sizevar: 'str'=None)\n\nTruncate `s` to length `maxlen`, adding suffix `suf` if truncated",
  'idx': 1,
  'params': [{'name': 's', 'desc': "param s: 'str'"},
   {'name': 'maxlen', 'desc': "param maxlen: 'int'"},
   {'name': 'suf', 'desc': "param suf: 'str'='…'"},
   {'name': 'space', 'desc': "param space=''"},
   {'name': 'sizevar', 'desc': "param sizevar: 'str'=None"}]}]

_param_idx('print(1, 2, ', 12), _param_idx('print(foo(1,2), ', 16), _param_idx('print("a,b", ', 13)

(2, 1, 1)

s = 'truncstr("a",'
res = ipy._sig_dyn(s, 1, len(s))
r = res[0]
test_eq(r['params'][r['idx']]['name'], 'maxlen')
res

[{'label': 'truncstr',
  'typ': 'function',
  'mod': 'fastcore.xtras',
  'doc': 'Truncate `s` to length `maxlen`, adding suffix `suf` if truncated',
  'idx': 1,
  'params': [{'name': 's', 'desc': "s: 'str'"},
   {'name': 'maxlen', 'desc': "maxlen: 'int'"},
   {'name': 'suf', 'desc': "suf: 'str' = '…'"},
   {'name': 'space', 'desc': "space=''"},
   {'name': 'sizevar', 'desc': "sizevar: 'str' = None"}]}]

InteractiveShell.sig_help


def sig_help(
    code, line_no:NoneType=None, col_no:NoneType=None
):

Get signature help for code at cursor position using dynamic analysis or jedi as a backup.

class _DynObj:
    def __getattr__(self, name):
        def _inner(x, y=1): ...
        _inner.__name__ = name
        return _inner

_dyn = _DynObj()

# Dynamic path: Jedi can't resolve __getattr__:
res = ipy.sig_help('_dyn.foo(', 1, 10)
test_eq(res[0]['label'], '_dyn.foo')
test_eq(len(res[0]['params']), 2)
res

[{'label': '_dyn.foo',
  'typ': 'function',
  'mod': '__main__',
  'doc': '',
  'idx': 0,
  'params': [{'name': 'x', 'desc': 'x'}, {'name': 'y', 'desc': 'y=1'}]}]

Displaying MIME data

cts = '#### A heading\n\nThis is **bold**.'
md_bundle = { 'text/markdown': cts }
ipy.display_pub.publish(data=md_bundle)

A heading

This is bold.

InteractiveShell.publish


def publish(
    data:str='', subtype:str='plain', mimetype:str='text', meta:NoneType=None, update:bool=False, kw:VAR_KEYWORD
):

Call self as a function.

ipy.publish(cts, 'markdown', foo='bar')

A heading

This is bold.

ipy.publish(HTML('<b>hi</b> there'))

hi there

ipy.publish({'text/plain':'hi there'})

hi there

transient


def transient(
    data:str='', subtype:str='plain', mimetype:str='text', meta:NoneType=None, update:bool=False, kw:VAR_KEYWORD
):

Call self as a function.

transient('hi there', foo='bar')

hi there

transient('*hi* **there**', subtype='markdown')

hi there

run_cmd


def run_cmd(
    cmd, data:str='', meta:NoneType=None, update:bool=False, kw:VAR_KEYWORD
):

Call self as a function.

read_url et al

_md = 'An alt text with escape chars\n![\[Uncaptioned image\]](https://www.example.org)'

Running the following will crash solveit:

# DON'T RUN!
# md_bundle = { 'text/markdown': md}
# ipy.display_pub.publish(data=md_bundle)

This happens because EscapeSequence isn’t handled correctly inside FrankenRenderer.render_image and raises an exception. We fix this by replacing the escape characters in the image alt-text:

cts = _absolutify_imgs(_md, '')
md_bundle = { 'text/markdown': cts }
ipy.display_pub.publish(data=md_bundle)

get_md


def get_md(
    html, url:str='', mmode:NoneType=None, ignore_links:bool=False, ignore_images:bool=False, mark_code:bool=True
):

Convert HTML to markdown with absolute image URLs and optional math mode

scrape_url


def scrape_url(
    url
):

Call self as a function.

scrape_url('http://www.example.org').encoding

'utf-8'

gh_blob_to_raw


def gh_blob_to_raw(
    url
):

Convert github.com/user/repo/blob/… URL to raw.githubusercontent.com URL

org = 'answerdotai'
reponm = 'dialoghelper'
repopre = f'https://github.com/{org}/{reponm}'

parse_gh_url(repopre), parse_gh_url(repopre + '/tree/main/dialoghelper')

({'owner': 'answerdotai',
  'repo': 'dialoghelper',
  'typ': None,
  'ref': None,
  'path': None},
 {'owner': 'answerdotai',
  'repo': 'dialoghelper',
  'typ': 'tree',
  'ref': 'main',
  'path': 'dialoghelper'})

read_gh_repo


def read_gh_repo(
    owner:str, repo:str, ref:str=None, path:str=''
):

Read GitHub repo info: description, file list, and README

read_url


def read_url(
    url:str, # URL to read
    as_md:bool=True, # Convert HTML to markdown
    extract_section:bool=True, # Extract section matching URL fragment or selector
    selector:str=None, # CSS selector to extract specific content
    ai_img:bool=False, # Add #ai suffix to image URLs
):

Read url from web

print(read_url('https://www.example.org'))

# Example Domain

This domain is for use in documentation examples without needing permission. Avoid use in operations.

[Learn more](https://iana.org/domains/example)

print(read_url('https://www.example.org', as_md=False, selector='body'))

<html><body><div><h1>Example Domain</h1><p>This domain is for use in documentation examples without needing permission. Avoid use in operations.</p><p><a href="https://iana.org/domains/example">Learn more</a></p></div></body></html>

print(read_url('https://caddyserver.com/docs/running#unit-files'))

### Unit Files

We provide two different systemd unit files that you can choose between, depending on your use case:

  * [**`caddy.service`**](https://github.com/caddyserver/dist/blob/master/init/caddy.service) if you configure Caddy with a [Caddyfile](/docs/caddyfile). If you prefer to use a different config adapter or a JSON config file, you may override the `ExecStart` and `ExecReload` commands.

  * [**`caddy-api.service`**](https://github.com/caddyserver/dist/blob/master/init/caddy-api.service) if you configure Caddy solely through its [API](/docs/api). This service uses the [`--resume`](/docs/command-line#caddy-run) option which will start Caddy using the `autosave.json` which is [persisted](/docs/json/admin/config/) by default.




They are very similar, but differ in the `ExecStart` and `ExecReload` commands to accommodate the workflows.

If you need to switch between the services, you should disable and stop the previous one before enabling and starting the other. For example, to switch from the `caddy` service to the `caddy-api` service:
```
sudo systemctl disable --now caddy
sudo systemctl enable --now caddy-api
```

html2text removes new lines so we only use get_md on html content and static text/markdown content is returned as is with new lines preserved:

print('\n'.join(read_url('https://fastht.ml/docs/llms.txt').splitlines()[:5]))

# FastHTML

> FastHTML is a python library which brings together Starlette, Uvicorn, HTMX, and fastcore's `FT` "FastTags" into a library for creating server-rendered hypermedia applications. The `FastHTML` class itself inherits from `Starlette`, and adds decorator-based routing with many additions, Beforeware, automatic `FT` to HTML rendering, and much more.

Things to remember when writing FastHTML apps:

Github repos display name, description, top level directory, and readme:

print(read_url(repopre)[:300])

# AnswerDotAI/dialoghelper
Helper functions for solveit dialogs

## / Files
- 📁 .github
- .gitignore
- CHANGELOG.md
- LICENSE
- MANIFEST.in
- README.md
- 📁 dialoghelper
- 📁 nbs
- pyproject.toml

## README
# dialoghelper

A Python library for programmatic dialog manipulation in [Solveit](https://solv

Github paths show the same as repos, but for that path:

print(read_url(repopre + '/tree/main/dialoghelper')[:300])

# AnswerDotAI/dialoghelper
Helper functions for solveit dialogs

## /dialoghelper Files
- __init__.py
- _modidx.py
- capture.py
- core.py
- db_dc.py
- dialoghelper
- exhash.py
- screenshot.js
- solve_auth.py
- stdtools.py
- tmux.py
- tracetools.py

Github files are redirected to the raw file:

print(read_url(repopre + '/blob/main/dialoghelper/core.py')[:300])

# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_core.ipynb.

# %% auto #0
__all__ = ['dname_doc', 'md_cls_d', 'dh_settings', 'Placements', 'mermaid_url', 'msg_insert_line', 'file_insert_line',
           'msg_str_replace', 'file_str_replace', 'msg_strs_replace', 'file_strs_replace', 'msg_repla

read_url renders latex based on the solveit user’s katex setting (USE_KATEX)

os.environ['USE_KATEX']='dollar'
print(read_url('https://arxiv.org/html/1706.03762v7',selector='#S3\.SS2\.SSS1')[:700])

####  3.2.1 Scaled Dot-Product Attention

We call our particular attention "Scaled Dot-Product Attention" (Figure 2). The input consists of queries and keys of dimension $d_{k}$, and values of dimension $d_{v}$. We compute the dot products of the query with all keys, divide each by $\sqrt{d_{k}}$, and apply a softmax function to obtain the weights on the values.

In practice, we compute the attention function on a set of queries simultaneously, packed together into a matrix $Q$. The keys and values are also packed together into matrices $K$ and $V$. We compute the matrix of outputs as:

| $$\mathrm{Attention}(Q,K,V)=\mathrm{softmax}(\frac{QK^{T}}{\sqrt{d_{k}}})V$$ |  | (1)  
---|---|---|---

os.environ['USE_KATEX']='1'
print(read_url('https://arxiv.org/html/1706.03762v7',selector='#S3\.SS2\.SSS1')[:700])

####  3.2.1 Scaled Dot-Product Attention

We call our particular attention "Scaled Dot-Product Attention" (Figure 2). The input consists of queries and keys of dimension \(d_{k}\), and values of dimension \(d_{v}\). We compute the dot products of the query with all keys, divide each by \(\sqrt{d_{k}}\), and apply a softmax function to obtain the weights on the values.

In practice, we compute the attention function on a set of queries simultaneously, packed together into a matrix \(Q\). The keys and values are also packed together into matrices \(K\) and \(V\). We compute the matrix of outputs as:

| $$\mathrm{Attention}(Q,K,V)=\mathrm{softmax}(\frac{QK^{T}}{\sqrt{d_{k}}})V$$ |  | (1)  
---|

Relative image paths are automatically corrected as well:

print(read_url('https://arxiv.org/html/1706.03762v7',selector='#Sx1')[:700])

## Attention Visualizations

![Refer to caption](https://arxiv.org/html/1706.03762v7/x1.png) Figure 3: An example of the attention mechanism following long-distance dependencies in the encoder self-attention in layer 5 of 6. Many of the attention heads attend to a distant dependency of the verb ‘making’, completing the phrase ‘making…more difficult’. Attentions here shown only for the word ‘making’. Different colors represent different heads. Best viewed in color.

![Refer to caption](https://arxiv.org/html/1706.03762v7/x2.png)

![Refer to caption](https://arxiv.org/html/1706.03762v7/x3.png)

Figure 4: Two attention heads, also in layer 5 of 6, apparently involved in anaphora resolution. Top

print(read_url('https://arxiv.org/html/1706.03762v7',selector='#Sx1',ai_img=True)[:700])

## Attention Visualizations

![Refer to caption](https://arxiv.org/html/1706.03762v7/x1.png#ai) Figure 3: An example of the attention mechanism following long-distance dependencies in the encoder self-attention in layer 5 of 6. Many of the attention heads attend to a distant dependency of the verb ‘making’, completing the phrase ‘making…more difficult’. Attentions here shown only for the word ‘making’. Different colors represent different heads. Best viewed in color.

![Refer to caption](https://arxiv.org/html/1706.03762v7/x2.png#ai)

![Refer to caption](https://arxiv.org/html/1706.03762v7/x3.png#ai)

Figure 4: Two attention heads, also in layer 5 of 6, apparently involved in anaphora resolu

Other helpers

fix_editable_priority


def fix_editable_priority(
    
):

Call self as a function.

DisplayObject.repr


def __repr__(
    
):

Return repr(self).

ZMQShellDisplayHook.finish_displayhook


def finish_displayhook(
    
):

Finish up all displayhook activities.

Extension

flexiclass?

def flexiclass(
    cls, # The class to convert
)->dataclass:

Convert `cls` into a `dataclass` like `make_nullable`. Converts in place and also returns the result.

File: ~/aai-ws/fastcore/fastcore/xtras.py

Type: function

flexiclass??

def flexiclass(
        cls # The class to convert
    ) -> dataclass:
    "Convert `cls` into a `dataclass` like `make_nullable`. Converts in place and also returns the result."
    if is_dataclass(cls): return make_nullable(cls)
    for k,v in get_annotations_ex(cls)[0].items():
        if not hasattr(cls,k) or getattr(cls,k) is MISSING:
            setattr(cls, k, field(default=UNSET))
    return dataclass(cls, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False)

File: ~/aai-ws/fastcore/fastcore/xtras.py

def f(a:int=0 # aa
): pass

@delegates(f)
def g(
    b:int, # bb
    **kwargs
)->int: # Returns the meaning of life
    "The g function"
    # nothing to see here
    pass

g?

def g(
    b:int, # bb
    a:int=0, # aa
)->int: # Returns the meaning of life

The g function

File: /var/folders/51/b2_szf2945n072c0vj2cyty40000gn/T/ipykernel_57170/3905198878.py

Type: function

g??

@delegates(f)
def g(
    b:int, # bb
    **kwargs
)->int: # Returns the meaning of life
    "The g function"
    # nothing to see here
    pass

File: /var/folders/51/b2_szf2945n072c0vj2cyty40000gn/T/ipykernel_57170/3905198878.py

Workaround for error where non str evalue causes a crash:

SyntaxTB.structured_traceback


def structured_traceback(
    etype, # Type of the exception raised.
    evalue, # Data stored in the exception
    etb, # If list: List of frames, see class docstring for details.
If Traceback: Traceback of the exception.
    tb_offset:NoneType=None, # Number of frames in the traceback to skip.  If not given, the
instance evalue is used (set in constructor).
    context:int=5, # Number of lines of context information to print.
):

Return a color formatted string with the traceback info.

Workaround for error where non str __file__ causes a crash:

load_ipython_extension


def load_ipython_extension(
    ip
):

Call self as a function.

InteractiveShell.run_cell_magic


async def run_cell_magic(
    magic_name, line, cell
):

Call self as a function.

InteractiveShell helpers

InteractiveShell.user_items

InteractiveShell.ranked_complete

InteractiveShell.get_vars

InteractiveShell.eval_exprs

InteractiveShell.get_schemas

InteractiveShell.xpush

Signatures

InteractiveShell.sig_help

Displaying MIME data

A heading

InteractiveShell.publish

A heading

transient

run_cmd

read_url et al

get_md

scrape_url

gh_blob_to_raw

read_gh_repo

read_url

Other helpers

fix_editable_priority

DisplayObject.__repr__

ZMQShellDisplayHook.finish_displayhook

Extension

SyntaxTB.structured_traceback

load_ipython_extension

InteractiveShell.run_cell_magic

DisplayObject.repr