Download helpers

from IPython.display import Markdown,HTML
from fastcore.test import *

source

clean_md

def clean_md(
    text, rm_comments:bool=True, rm_details:bool=True
):

Remove comments and <details> sections from text

source

read_md

def read_md(
    url, rm_comments:bool=True, rm_details:bool=True, *, params:QueryParamTypes | None=None,
    headers:HeaderTypes | None=None, cookies:CookieTypes | None=None, auth:AuthTypes | None=None,
    proxy:ProxyTypes | None=None, follow_redirects:bool=False, verify:ssl.SSLContext | str | bool=True,
    timeout:TimeoutTypes=Timeout(timeout=5.0), trust_env:bool=True
):

Read text from url and clean with clean_docs

mdurl = 'https://claudette.answer.ai/index.html.md'
md = read_md(mdurl)
# Markdown(md)

source

html2md

def html2md(
    s:str, # HTML to convert
    ignore_links:bool=True, # Drop links, keeping their text
    ignore_images:bool=True, # Drop images
    base_url:str='', # URL that relative link and image URLs resolve against
):

Convert s from HTML to markdown, with code blocks fenced

html2md fences code blocks, and resolves relative link and image URLs against base_url when it keeps them:

s = '<p>See <a href="b.html">this</a> <img alt="fig" src="f.png"></p><pre><code>  x = 1\n  y = 2</code></pre>'
md = html2md(s, ignore_links=False, ignore_images=False, base_url='https://ex.org/a/')
assert '[this](https://ex.org/a/b.html)' in md and '![fig](https://ex.org/a/f.png)' in md
assert '````\nx = 1\ny = 2\n````' in md
md

source

read_html

def read_html(
    url, # URL to read
    sel:NoneType=None, # Read only outerHTML of CSS selector `sel`
    rm_comments:bool=True, # Removes HTML comments
    rm_details:bool=True, # Removes `<details>` tags
    multi:bool=False, # Get all matches to `sel` or first one
    wrap_tag:NoneType=None, # If multi, each selection wrapped with <wrap_tag>content</wrap_tag>
    ignore_links:bool=True
):

Get url, optionally selecting CSS selector sel, and convert to clean markdown

# test single class selector
listings = read_html('https://www.answer.ai/', sel='.listing-description')
assert len(listings) < 500

# Test multi class selector
listings = read_html('https://www.answer.ai/', sel='.listing-description', multi=True)
assert len(listings) > 1000 # returns more than single so selecting multi

# Test multi_wrap_tag
listings = read_html('https://www.answer.ai/', sel='.listing-description', multi=True, wrap_tag='document')
assert '<document>' in listings and '</document>' in listings

read_html('https://www.answer.ai/', sel='.listing-description', ignore_links=False)

'[ How data quality, proxy measures, and missing domain expertise can undermine AI science results ](./posts/2026-03-17-risks-life-sciences/index.html)\n\n'

# test tag css selectors
assert len(read_html('https://www.answer.ai/', sel='div.listing-description', multi=True)) > 1000
assert len(read_html('https://www.answer.ai/', sel='div', multi=True)) > 1000

htmlurl = 'https://hypermedia.systems/hypermedia-a-reintroduction/'
hmd = read_html(htmlurl)
assert len(hmd) > 100
# Markdown(hmd)

read_url reads a page the way a browser would, through scrape_url, which gets past Cloudflare’s bot checks. It converts HTML to markdown with html2md, keeping links and images with absolute URLs. A URL fragment, such as #unit-files, selects just that section, and selector selects elements by CSS selector instead. Content that isn’t HTML, such as an llms.txt file, comes back unchanged.

source

scrape_url

def scrape_url(
    url
):

Get url with a Cloudflare-aware scraper, decoding text without a declared encoding as UTF-8

html2text escapes every backslash that precedes punctuation. In TeX, that turns \{ into \\{. To keep each formula intact, _tex_math replaces it with a placeholder before conversion. read_url puts the TeX back afterwards.

source

read_url

def read_url(
    url:str, # URL to read
    as_md:bool=True, # Convert HTML to markdown
    extract_section:bool=True, # Extract section matching URL fragment or selector
    selector:str=None, # CSS selector to extract specific content
):

Read url, converting HTML to markdown with absolute link and image URLs

read_url('https://www.example.org')

A fragment selects the section that starts at that id, up to the next element of the same kind:

sec = read_url('https://caddyserver.com/docs/running#unit-files')
assert sec.startswith('### Unit Files')
sec[:300]

Formulas in MathML that carry their TeX source, as in arXiv’s HTML papers, come back as TeX, written $...$ inline and $$...$$ for display math:

att = read_url('https://arxiv.org/html/1706.03762v7#S3.SS2.SSS1')
assert r'$$\mathrm{Attention}(Q,K,V)' in att and r'\(d_{k}\)' in att
Markdown(att)

Plain text, such as an llms.txt file, keeps its own line breaks:

read_url('https://fastht.ml/docs/llms.txt').splitlines()[:3]

source

get_llmstxt

def get_llmstxt(
    url, optional:bool=False, n_workers:NoneType=None
):

Get llms.txt file from and expand it with llms_txt.create_ctx()

# print(get_llmstxt('https://llmstxt.org/llms.txt'))

source

split_url

def split_url(
    url
):

Split url into base, path, and file name, normalising name to ‘/’ if empty

urls = ('https://claudette.answer.ai/path/', 'https://claudette.answer.ai/', 'https://llmstxt.org', 'https://llmstxt.org/')

[split_url(o) for o in urls]

[('https://claudette.answer.ai', '', '/path'),
 ('https://claudette.answer.ai', '/', ''),
 ('https://llmstxt.org', '/', ''),
 ('https://llmstxt.org', '/', '')]

source

find_docs

def find_docs(
    url
):

If available, return LLM-friendly llms.txt context or markdown file location from url

fl_url = 'https://answerdotai.github.io/fastlite'

find_docs(fl_url)

'https://answerdotai.github.io/fastlite/llms.txt'

for o in urls: print(find_docs(o))

https://claudette.answer.ai/llms.txt
https://claudette.answer.ai/llms.txt
https://llmstxt.org/llms.txt
https://llmstxt.org/llms.txt

suffixes = ["/", "/tmp", "/tmp/tmp/"]
for suff in suffixes:
    for o in urls:  test_eq(find_docs(o), find_docs(o+suff))

test_eq(find_docs("https://github.com"), "https://github.com/llms.txt")
test_eq(find_docs("https://github.com/AnswerDotAI"), "https://github.com/llms.txt")
test_eq(find_docs("https://github.com/AnswerDotAI/"), "https://github.com/llms.txt")

source

read_docs

def read_docs(
    url, optional:bool=False, n_workers:NoneType=None, rm_comments:bool=True, rm_details:bool=True
):

If available, return LLM-friendly llms.txt context or markdown file response for url

Other Formats

clean_md

read_md

html2md

read_html

scrape_url

read_url

get_llmstxt

split_url

find_docs

read_docs