llms.txt API Reference¶

def parse(text: str) -> LlmsTxt:
    """Parse llms.txt content into structured data.

    Args:
        text: Raw text content of an llms.txt file.

    Returns:
        Parsed ``LlmsTxt`` object.

    Raises:
        LlmsTxtError: If the required H1 title is missing.
    """
    text = text.replace("\r\n", "\n").strip()
    if not text:
        raise LlmsTxtError("empty input")

    # Extract H1 title
    h1_match = _H1_RE.search(text)
    if not h1_match:
        raise LlmsTxtError("missing required H1 title")
    title = h1_match.group(1).strip()

    # Split on H2 headers: [preamble, name1, body1, name2, body2, ...]
    parts = _H2_SPLIT_RE.split(text)
    preamble = parts[0]

    # Remove the H1 line from preamble before parsing description/details
    preamble = preamble[h1_match.end() :]

    description, details = _parse_preamble(preamble)

    # Parse sections
    sections: dict[str, list[FileEntry]] = {}
    for i in range(1, len(parts), 2):
        section_name = parts[i].strip()
        section_body = parts[i + 1] if i + 1 < len(parts) else ""
        sections[section_name] = _parse_links(section_body)

    # Separate "Optional" section
    optional = sections.pop("Optional", [])

    return LlmsTxt(
        title=title,
        description=description,
        details=details,
        sections=sections,
        optional=optional,
    )

def find_candidates(url: str, doc: LlmsTxt | None = None) -> list[FileEntry]:
    """Find candidate markdown resources for a given URL.

    When *doc* is provided, searches all sections and optional entries for
    URLs that relate to *url* (exact match > extension variation > path
    prefix).  If no match is found (or *doc* is ``None``), falls back to
    heuristic URL generation based on common per-page ``.md`` conventions.

    Args:
        url: The page URL to look up.
        doc: An optional parsed ``LlmsTxt`` object to search in.

    Returns:
        List of ``FileEntry`` candidates, ordered by match quality.
    """
    base = _strip_url(url)
    base_path = _url_path(url).rstrip("/")

    # ── Search llms.txt entries ──
    if doc is not None:
        all_entries: list[FileEntry] = []
        for entries in doc.sections.values():
            all_entries.extend(entries)
        all_entries.extend(doc.optional)

        exact: list[FileEntry] = []
        extension: list[FileEntry] = []
        prefix: list[FileEntry] = []

        for entry in all_entries:
            entry_base = _strip_url(entry.url)
            entry_path = _url_path(entry.url).rstrip("/")

            if entry_base == base:
                exact.append(entry)
                continue

            if entry_path == base_path + ".md" or entry_path == base_path + ".html.md":
                extension.append(entry)
                continue
            if base_path == entry_path + ".md" or base_path == entry_path + ".html.md":
                extension.append(entry)
                continue

            if entry_path.startswith(base_path + "/") or base_path.startswith(
                entry_path + "/"
            ):
                prefix.append(entry)

        results = exact + extension + prefix
        if results:
            return results

    # ── Fallback: heuristic URL candidates ──
    return [FileEntry(name="", url=u) for u in _candidate_md_urls(url)]

def discover(url: str, *, timeout: int = 10) -> DiscoveryResult:
    """Probe a site for ``/llms.txt`` and ``/llms-full.txt``.

    Given any URL, extracts the root (``{scheme}://{netloc}``) and attempts to
    fetch both ``/llms.txt`` and ``/llms-full.txt``.  If the input URL already
    points to one of these files, it is still fetched (along with its sibling).

    Args:
        url: Any URL belonging to the target site.
        timeout: HTTP request timeout in seconds (per request).

    Returns:
        A ``DiscoveryResult`` with the raw content of whichever files were
        found (fields are ``None`` when the file does not exist or could not
        be fetched).

    Example::

        result = discover("https://example.com/docs/guide")
        content = result.llms_full_txt or result.llms_txt
        if content:
            doc = parse(content)
    """
    parsed = urllib.parse.urlparse(url)
    root = f"{parsed.scheme}://{parsed.netloc}"

    llms_txt = _fetch_text(f"{root}/llms.txt", timeout)
    llms_full_txt = _fetch_text(f"{root}/llms-full.txt", timeout)

    return DiscoveryResult(
        llms_txt=llms_txt,
        llms_full_txt=llms_full_txt,
        source_url=root,
    )