llms.txt generator for Astro blogs - zero dependencies

[slug].txt.ts

import type { GetStaticPaths } from "astro";
import { siteConfig } from "@/site-config";
import { llmsPost } from "@utils/llms";
import { formatUrl } from "@utils/path";
import { type BlogPost, getAllPosts } from "@utils/posts";

export const getStaticPaths: GetStaticPaths = async () => {
  const posts = await getAllPosts();
  return posts.map((post) => ({
    params: { slug: post.slug },
    props: { post },
  }));
};

interface Props {
  post: BlogPost;
}

export const GET = ({ props }: { props: Props }) => {
  const { post } = props;

  return llmsPost({
    post,
    site: siteConfig.url,
    link: formatUrl(post.slug),
  });
};

llms-full.txt.ts

import type { APIRoute } from "astro";
import { siteConfig } from "@/site-config";
import { llmsFullTxt, postsToLlmsFullItems } from "@utils/llms";
import { formatUrl } from "@utils/path";
import { getAllPosts } from "@utils/posts";

export const GET: APIRoute = async () => {
  const posts = await getAllPosts();

  return llmsFullTxt({
    name: siteConfig.name,
    description: siteConfig.description,
    author: siteConfig.author,
    site: siteConfig.url,
    items: postsToLlmsFullItems(posts, formatUrl),
  });
};

llms.ts

import type { BlogPost } from "./posts";

interface LlmsItem {
  title: string;
  description: string;
  link: string;
}

interface LlmsFullItem extends LlmsItem {
  pubDate: Date;
  category: string;
  body: string;
}

interface LlmsTxtConfig {
  name: string;
  description: string;
  site: string;
  items: LlmsItem[];
  optional?: LlmsItem[];
}

interface LlmsFullTxtConfig {
  name: string;
  description: string;
  author: string;
  site: string;
  items: LlmsFullItem[];
}

interface LlmsPostConfig {
  post: BlogPost;
  site: string;
  link: string;
}

const MDX_PATTERNS = [
  /^import\s+.+from\s+['"].+['"];?\s*$/gm,
  /<[A-Z][a-zA-Z]*[^>]*>[\s\S]*?<\/[A-Z][a-zA-Z]*>/g,
  /<[A-Z][a-zA-Z]*[^>]*\/>/g,
] as const;

function stripMdx(content: string): string {
  return MDX_PATTERNS.reduce((text, pattern) => text.replace(pattern, ""), content).trim();
}

function formatDate(date: Date): string {
  return date.toISOString().split("T")[0];
}

function doc(...sections: (string | string[])[]): Response {
  const content = sections
    .flat()
    .join("\n")
    .replace(/\n{3,}/g, "\n\n")
    .trim();

  return new Response(content + "\n", {
    headers: { "Content-Type": "text/plain; charset=utf-8" },
  });
}

function header(name: string, description: string): string[] {
  return [`# ${name}`, "", `> ${description}`];
}

function linkList(title: string, items: LlmsItem[], site: string): string[] {
  return [
    "",
    `## ${title}`,
    ...items.map((item) => `- [${item.title}](${site}${item.link}): ${item.description}`),
  ];
}

function postMeta(site: string, link: string, pubDate: Date, category: string): string[] {
  return [`URL: ${site}${link}`, `Published: ${formatDate(pubDate)}`, `Category: ${category}`];
}

export function llmsTxt(config: LlmsTxtConfig): Response {
  const sections = [
    header(config.name, config.description),
    linkList("Posts", config.items, config.site),
  ];

  if (config.optional?.length) {
    sections.push(linkList("Optional", config.optional, config.site));
  }

  return doc(...sections);
}

export function llmsFullTxt(config: LlmsFullTxtConfig): Response {
  const head = [
    ...header(config.name, config.description),
    "",
    `Author: ${config.author}`,
    `Site: ${config.site}`,
    "",
    "---",
  ];

  const posts = config.items.flatMap((item) => [
    "",
    `## ${item.title}`,
    "",
    ...postMeta(config.site, item.link, item.pubDate, item.category),
    "",
    `> ${item.description}`,
    "",
    stripMdx(item.body),
    "",
    "---",
  ]);

  return doc(head, posts);
}

export function llmsPost(config: LlmsPostConfig): Response {
  const { post, site, link } = config;
  const { title, description, pubDate, category } = post.data;

  return doc(
    `# ${title}`,
    "",
    `> ${description}`,
    "",
    ...postMeta(site, link, pubDate, category),
    "",
    stripMdx(post.body ?? ""),
  );
}

export function postsToLlmsItems(
  posts: BlogPost[],
  formatUrl: (slug: string) => string,
): LlmsItem[] {
  return posts.map((post) => ({
    title: post.data.title,
    description: post.data.description,
    link: formatUrl(post.slug),
  }));
}

export function postsToLlmsFullItems(
  posts: BlogPost[],
  formatUrl: (slug: string) => string,
): LlmsFullItem[] {
  return posts.map((post) => ({
    ...postsToLlmsItems([post], formatUrl)[0],
    pubDate: post.data.pubDate,
    category: post.data.category,
    body: post.body ?? "",
  }));
}

llms.txt.ts

import type { APIRoute } from "astro";
import { siteConfig } from "@/site-config";
import { llmsTxt, postsToLlmsItems } from "@utils/llms";
import { getAllPosts } from "@utils/posts";

const formatLlmsUrl = (slug: string) => `/llms/${slug}.txt`;

export const GET: APIRoute = async () => {
  const posts = await getAllPosts();

  return llmsTxt({
    name: siteConfig.name,
    description: siteConfig.description,
    site: siteConfig.url,
    items: postsToLlmsItems(posts, formatLlmsUrl),
    optional: [
      { title: "About", link: "/about", description: "About the author" },
      { title: "RSS Feed", link: "/rss.xml", description: "Subscribe to updates" },
      {
        title: "Full Content",
        link: "/llms-full.txt",
        description: "Complete post content for deeper context",
      },
    ],
  });
};