Add cache headers, real 404s, social meta, JSON-LD, robots and sitemap

src/lib/components/DiscussionsFeed.svelte

@@ -11,6 +11,10 @@
     type Author,
   } from "$lib/components/ThreadItem.svelte";
   import SortToggle from "$lib/components/SortToggle.svelte";
+  import Meta from "$lib/components/Meta.svelte";
+  import { groupStore } from "$lib/group.svelte";
+  import { groupsStore } from "$lib/groups.svelte";
+  import { MODE } from "$lib/config";
   import type { NostrUser } from "@nostr/gadgets/metadata";
   import { auth, openLogin } from "$lib/auth.svelte";
   import { openDraft } from "$lib/draft.svelte";
@@ -45,6 +49,12 @@
     if (urlSort) sortPref.value = urlSort;
   });
 
+  const about = $derived(
+    MODE === "full"
+      ? (groupsStore.list.find((g) => g.id === groupId)?.about ?? "")
+      : (groupStore.data?.about ?? ""),
+  );
+
   const flags = $derived(getGroupFlags(groupId));
   const member = $derived(membershipOf(groupId));
 
@@ -123,9 +133,7 @@
   );
 </script>
 
-<svelte:head>
-  <title>{title}</title>
-</svelte:head>
+<Meta {title} description={about} />
 
 <div class="mx-auto max-w-6xl">
   <div class="flex flex-wrap items-center justify-between gap-2 py-2">

src/lib/components/Meta.svelte

@@ -0,0 +1,57 @@
+<script lang="ts">
+  import { page } from "$app/state";
+  import { TITLE } from "$lib/config";
+  import { groupStore } from "$lib/group.svelte";
+  import { GROUP_ID, MODE } from "$lib/config";
+
+  type Props = {
+    title: string;
+    description?: string;
+    type?: "website" | "article";
+    image?: string;
+    jsonLd?: Record<string, unknown>;
+  };
+
+  let {
+    title,
+    description = "",
+    type = "website",
+    image,
+    jsonLd,
+  }: Props = $props();
+
+  // Escape "<" so the JSON can never close the script tag early.
+  const ld = $derived(
+    jsonLd ? JSON.stringify(jsonLd).replace(/</g, "\\u003c") : "",
+  );
+
+  const siteName = $derived(
+    TITLE || (MODE === "simple" ? (groupStore.data?.name ?? GROUP_ID) : ""),
+  );
+  const canonical = $derived(page.url.origin + page.url.pathname);
+</script>
+
+<svelte:head>
+  <title>{title}</title>
+  <link rel="canonical" href={canonical} />
+  {#if description}
+    <meta name="description" content={description} />
+    <meta property="og:description" content={description} />
+  {/if}
+  <meta property="og:title" content={title} />
+  <meta property="og:type" content={type} />
+  <meta property="og:url" content={canonical} />
+  {#if siteName}
+    <meta property="og:site_name" content={siteName} />
+  {/if}
+  {#if image}
+    <meta property="og:image" content={image} />
+  {/if}
+  <meta
+    name="twitter:card"
+    content={image ? "summary_large_image" : "summary"}
+  />
+  {#if ld}
+    {@html `<script type="application/ld+json">${ld}</script>`}
+  {/if}
+</svelte:head>

src/lib/config.ts

@@ -21,6 +21,11 @@ if (MODE === "simple" && !GROUP_ID) {
 // Server-side rendering: opt-in, and only meaningful when the build targets the
 // Node adapter (svelte.config.js reads the same variable).
 export const SSR_ENABLED = env.PUBLIC_SSR === "yes";
+// Server-rendered pages and snapshots are anonymous, so a shared cache may
+// hold them: fresh for 5 minutes, served stale for an hour while revalidating.
+// Browsers always revalidate (max-age=0) so a login shows its content at once.
+export const CACHE_CONTROL =
+  "public, max-age=0, s-maxage=300, stale-while-revalidate=3600";
 // Requires a relay with NIP-50 support.
 export const SEARCH_ENABLED = env.PUBLIC_SEARCH === "yes";
 export const LABELS = (env.PUBLIC_LABELS ?? "")

src/lib/forum/snapshot.ts

@@ -1,4 +1,5 @@
 import { browser } from "$app/environment";
+import { error } from "@sveltejs/kit";
 import type { NostrUser } from "$lib/gadgets";
 import type { GroupMetadata, GroupSummary } from "./groups";
 import type { Resource } from "./resources";
@@ -37,14 +38,22 @@ type Fetch = typeof fetch;
 // HTML; in the browser it is consumed only while hydrating that HTML. Client
 // navigations get null, leaving the stores to fetch live (and authenticated)
 // data as they always did — the same path a client-only build takes.
+// A 404 from the endpoint becomes a real 404 page on the server, so crawlers
+// never index an empty shell; the client keeps its own not-found handling.
 export async function snapshot<T>(
   fetch: Fetch,
   url: string,
 ): Promise<T | null> {
   if (browser && !hasInlinedResponse(url)) return null;
+  let res: Response;
+  try {
+    res = await fetch(url);
+  } catch {
+    return null;
+  }
+  if (res.status === 404 && !browser) error(404, "Not found");
+  if (!res.ok) return null;
   try {
-    const res = await fetch(url);
-    if (!res.ok) return null;
     return (await res.json()) as T;
   } catch {
     return null;

src/lib/seo.ts

@@ -0,0 +1,17 @@
+// Plain-text excerpt of a markdown body for description meta tags: the first
+// paragraph that isn't a heading or a bare media URL, trimmed to `max` chars.
+export function excerpt(markdown: string, max = 160): string {
+  const lines = markdown.split("\n").map((l) => l.trim());
+  const line = lines.find(
+    (l) => l && !l.startsWith("#") && !/^https?:\/\/\S+$/.test(l),
+  );
+  if (!line) return "";
+  const text = line
+    .replace(/!\[[^\]]*\]\([^)]*\)/g, "") // images
+    .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") // links → label
+    .replace(/[*_`~]+/g, "")
+    .replace(/nostr:([a-z0-9]{12})[a-z0-9]*/g, "@$1…") // keep a short handle
+    .replace(/\s+/g, " ")
+    .trim();
+  return text.length > max ? text.slice(0, max - 1).trimEnd() + "…" : text;
+}

src/routes/+layout.ts

@@ -1,8 +1,11 @@
-import { SSR_ENABLED } from "$lib/config";
+import { browser } from "$app/environment";
+import { CACHE_CONTROL, SSR_ENABLED } from "$lib/config";
 import { snapshot, type ForumShell } from "$lib/forum/snapshot";
 
 export const ssr = SSR_ENABLED;
 
-export async function load({ fetch }) {
+// Runs for every page, so the cache policy for rendered HTML is set here.
+export async function load({ fetch, setHeaders }) {
+  if (!browser) setHeaders({ "cache-control": CACHE_CONTROL });
   return { shell: await snapshot<ForumShell>(fetch, "/api/shell") };
 }

src/routes/+page.svelte

@@ -7,10 +7,26 @@
   import { partialsStore } from "$lib/partials.svelte";
   import { MODE, GROUP_ID, SEARCH_ENABLED } from "$lib/config";
   import SearchInline from "$lib/components/SearchInline.svelte";
+  import Meta from "$lib/components/Meta.svelte";
+  import { groupStore } from "$lib/group.svelte";
+  import { excerpt } from "$lib/seo";
   import type { NostrUser } from "@nostr/gadgets/metadata";
 
   const partial = $derived(partialsStore.get("home"));
 
+  const description = $derived(
+    partial
+      ? excerpt(partial.content)
+      : MODE === "simple"
+        ? (groupStore.data?.about ?? "")
+        : "",
+  );
+  // A partial's leading image doubles as the social preview.
+  const image = $derived(
+    partial?.content.match(/^https?:\/\/\S+\.(?:png|jpe?g|webp|gif)$/im)?.[0] ??
+      groupStore.data?.picture,
+  );
+
   // Full mode: load per-room activity + recent threads once the rooms are known.
   $effect(() => {
     if (MODE !== "full") return;
@@ -35,9 +51,7 @@
   }
 </script>
 
-<svelte:head>
-  <title>{MODE === "full" ? "Rooms" : "Discussions"}</title>
-</svelte:head>
+<Meta title={MODE === "full" ? "Rooms" : "Discussions"} {description} {image} />
 
 {#snippet pic(a: { name: string; picture?: string })}
   {#if a.picture}

src/routes/api/contacts/+server.ts

@@ -1,9 +1,11 @@
 import { json } from "@sveltejs/kit";
+import { CACHE_CONTROL } from "$lib/config";
 import { fetchProfiles } from "$lib/forum/profiles";
 import { loadShell } from "$lib/ssr/shell";
 import { profileQuery } from "$lib/ssr/relay";
 
-export async function GET() {
+export async function GET({ setHeaders }) {
+  setHeaders({ "cache-control": CACHE_CONTROL });
   const shell = await loadShell();
   return json({ profiles: await fetchProfiles(profileQuery, shell.admins) });
 }

src/routes/api/overview/+server.ts

@@ -1,5 +1,5 @@
 import { json } from "@sveltejs/kit";
-import { MODE, GROUP_ID } from "$lib/config";
+import { MODE, GROUP_ID, CACHE_CONTROL } from "$lib/config";
 import { fetchGroups } from "$lib/forum/groups";
 import { fetchOverview, overviewPubkeys } from "$lib/forum/overview";
 import { fetchProfiles } from "$lib/forum/profiles";
@@ -8,7 +8,8 @@ import { forumQuery, profileQuery } from "$lib/ssr/relay";
 
 // Room activity and the latest discussions: every room in full mode, the
 // single group in simple mode (mirrors LatestDiscussions' own room set).
-export async function GET() {
+export async function GET({ setHeaders }) {
+  setHeaders({ "cache-control": CACHE_CONTROL });
   const roomIds =
     MODE === "full"
       ? (await fetchGroups(forumQuery)).map((g) => g.id)

src/routes/api/shell/+server.ts

@@ -1,6 +1,8 @@
 import { json } from "@sveltejs/kit";
+import { CACHE_CONTROL } from "$lib/config";
 import { loadShell } from "$lib/ssr/shell";
 
-export async function GET() {
+export async function GET({ setHeaders }) {
+  setHeaders({ "cache-control": CACHE_CONTROL });
   return json(await loadShell());
 }

src/routes/api/thread/[id]/+server.ts

@@ -1,10 +1,11 @@
 import { error, json } from "@sveltejs/kit";
-import { GROUP_ID } from "$lib/config";
+import { GROUP_ID, CACHE_CONTROL } from "$lib/config";
 import { fetchThread, threadAuthors } from "$lib/forum/thread";
 import { fetchProfiles } from "$lib/forum/profiles";
 import { forumQuery, profileQuery } from "$lib/ssr/relay";
 
-export async function GET({ params }) {
+export async function GET({ params, setHeaders }) {
+  setHeaders({ "cache-control": CACHE_CONTROL });
   if (!/^[0-9a-f]{64}$/.test(params.id)) error(404, "Not found");
   const thread = await fetchThread(forumQuery, params.id, GROUP_ID);
   if (!thread) error(404, "Not found");

src/routes/api/threads/[group]/+server.ts

@@ -1,5 +1,5 @@
 import { error, json } from "@sveltejs/kit";
-import { MODE, GROUP_ID } from "$lib/config";
+import { MODE, GROUP_ID, CACHE_CONTROL } from "$lib/config";
 import { fetchGroups } from "$lib/forum/groups";
 import {
   fetchThreadPage,
@@ -12,7 +12,8 @@ import { forumQuery, profileQuery } from "$lib/ssr/relay";
 
 // First page of a group's listing. Unknown rooms are a 404 so crawlers don't
 // index empty shells; private rooms are simply empty to the anonymous server.
-export async function GET({ params, url }) {
+export async function GET({ params, url, setHeaders }) {
+  setHeaders({ "cache-control": CACHE_CONTROL });
   const groupId = params.group;
   if (MODE === "simple") {
     if (groupId !== GROUP_ID) error(404, "Not found");

src/routes/contacts/+page.svelte

@@ -16,6 +16,7 @@
   import { partialsStore } from "$lib/partials.svelte";
   import PostContent from "$lib/components/PostContent.svelte";
   import { page } from "$app/state";
+  import Meta from "$lib/components/Meta.svelte";
 
   const partial = $derived(partialsStore.get("contacts"));
 
@@ -86,10 +87,7 @@
   }
 </script>
 
-<svelte:head>
-  <title>Contacts</title>
-  <meta name="description" content="Admins and contacts for this community." />
-</svelte:head>
+<Meta title="Contacts" description="Admins and contacts for this community." />
 
 <div class="mx-auto max-w-6xl">
   <h1 class="text-accent py-2 text-[1.65rem]">Contacts</h1>

src/routes/resource/[slug]/+page.svelte

@@ -3,6 +3,8 @@
   import { resourcesStore } from "$lib/resources.svelte";
   import { adminPubkeys } from "$lib/admins.svelte";
   import PostContent from "$lib/components/PostContent.svelte";
+  import Meta from "$lib/components/Meta.svelte";
+  import { excerpt } from "$lib/seo";
 
   const slug = $derived(page.params.slug ?? "");
   const resource = $derived(resourcesStore.list.find((r) => r.slug === slug));
@@ -10,23 +12,10 @@
   const ready = $derived(resourcesStore.loaded && adminPubkeys.loaded);
   const notFound = $derived(ready && !resource);
 
-  // First non-empty line of the body, trimmed of markdown markers, for SEO.
-  const description = $derived(
-    (resource?.content ?? "")
-      .replace(/^#+\s*/gm, "")
-      .split("\n")
-      .map((l) => l.trim())
-      .find(Boolean)
-      ?.slice(0, 160) ?? "",
-  );
+  const description = $derived(excerpt(resource?.content ?? ""));
 </script>
 
-<svelte:head>
-  <title>{resource?.title ?? "Resource"}</title>
-  {#if description}
-    <meta name="description" content={description} />
-  {/if}
-</svelte:head>
+<Meta title={resource?.title ?? "Resource"} {description} type="article" />
 
 <div class="mx-auto max-w-6xl">
   <a
@@ -56,8 +45,12 @@
       <PostContent content={resource.content} headingOffset={0} />
     </div>
   {:else if notFound}
-    <p class="py-12 text-center text-neutral-400 dark:text-neutral-500">Resource not found.</p>
+    <p class="py-12 text-center text-neutral-400 dark:text-neutral-500">
+      Resource not found.
+    </p>
   {:else}
-    <p class="py-12 text-center text-neutral-400 dark:text-neutral-500">Loading…</p>
+    <p class="py-12 text-center text-neutral-400 dark:text-neutral-500">
+      Loading…
+    </p>
   {/if}
 </div>

src/routes/resource/[slug]/+page.ts

@@ -1,7 +1,17 @@
+import { browser } from "$app/environment";
+import { error } from "@sveltejs/kit";
 import { snapshot, type OverviewSnapshot } from "$lib/forum/snapshot";
 
-// Articles show the latest-discussions panel next to the text.
-export async function load({ fetch }) {
+// Articles show the latest-discussions panel next to the text. An unknown
+// slug is a real 404 on the server; the client keeps its in-page message.
+export async function load({ fetch, params, parent }) {
+  if (!browser) {
+    const { shell } = await parent();
+    const known = shell?.resources.some(
+      (r) => r.slug === params.slug && shell.admins.includes(r.pubkey),
+    );
+    if (shell && !known) error(404, "Not found");
+  }
   const overview = await snapshot<OverviewSnapshot>(fetch, "/api/overview");
   return { overview, profiles: overview?.profiles ?? null };
 }

src/routes/robots.txt/+server.ts

@@ -0,0 +1,13 @@
+import { SSR_ENABLED } from "$lib/config";
+
+// Static builds bake it in at build time (no sitemap to point at); the server
+// build answers live so the sitemap URL carries the real origin.
+export const prerender = !SSR_ENABLED;
+
+export function GET({ url }) {
+  const lines = ["User-agent: *", "Disallow:"];
+  if (SSR_ENABLED) lines.push(`Sitemap: ${url.origin}/sitemap.xml`);
+  return new Response(lines.join("\n") + "\n", {
+    headers: { "content-type": "text/plain" },
+  });
+}

src/routes/sitemap.xml/+server.ts

@@ -0,0 +1,56 @@
+import { CACHE_CONTROL, MODE, GROUP_ID } from "$lib/config";
+import { fetchGroups } from "$lib/forum/groups";
+import { forumQuery } from "$lib/ssr/relay";
+import { loadShell } from "$lib/ssr/shell";
+
+const THREADS_PER_ROOM = 1000;
+
+type Entry = { path: string; lastmod?: number };
+
+function xml(origin: string, entries: Entry[]): string {
+  const items = entries.map((e) => {
+    const loc = `<loc>${origin}${e.path}</loc>`;
+    const mod = e.lastmod
+      ? `<lastmod>${new Date(e.lastmod * 1000).toISOString()}</lastmod>`
+      : "";
+    return `<url>${loc}${mod}</url>`;
+  });
+  return (
+    '<?xml version="1.0" encoding="UTF-8"?>' +
+    '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">' +
+    items.join("") +
+    "</urlset>"
+  );
+}
+
+// Public pages only: rooms the anonymous server can see, their threads, and
+// the admin-authored resources.
+export async function GET({ url, setHeaders }) {
+  const entries: Entry[] = [{ path: "/" }, { path: "/contacts" }];
+  const shell = await loadShell();
+  const admins = new Set(shell.admins);
+  for (const r of shell.resources) {
+    if (admins.has(r.pubkey))
+      entries.push({ path: `/resource/${r.slug}`, lastmod: r.createdAt });
+  }
+  const roomIds =
+    MODE === "full"
+      ? (await fetchGroups(forumQuery))
+          .filter((g) => !g.flags.includes("private"))
+          .map((g) => g.id)
+      : [GROUP_ID];
+  for (const id of roomIds) {
+    if (MODE === "full") entries.push({ path: `/room/${id}` });
+    const ops = await forumQuery({
+      kinds: [11],
+      "#h": [id],
+      limit: THREADS_PER_ROOM,
+    });
+    for (const e of ops)
+      entries.push({ path: `/thread/${e.id}`, lastmod: e.created_at });
+  }
+  setHeaders({ "cache-control": CACHE_CONTROL });
+  return new Response(xml(url.origin, entries), {
+    headers: { "content-type": "application/xml" },
+  });
+}

src/routes/thread/[id]/+page.svelte

@@ -30,6 +30,8 @@
   import MessageEditor from "$lib/components/MessageEditor.svelte";
   import PostContent from "$lib/components/PostContent.svelte";
   import Tag from "$lib/components/Tag.svelte";
+  import Meta from "$lib/components/Meta.svelte";
+  import { excerpt } from "$lib/seo";
   import type { NostrUser } from "$lib/gadgets";
 
   type Author = { pubkey: string; name: string; picture?: string };
@@ -120,6 +122,25 @@
   const detail = $derived(threadDetailStore.detail);
   const profiles = $derived(threadDetailStore.profiles);
 
+  // Structured data for crawlers: the OP as a forum posting.
+  const jsonLd = $derived(
+    detail
+      ? {
+          "@context": "https://schema.org",
+          "@type": "DiscussionForumPosting",
+          headline: detail.title,
+          text: detail.op.content,
+          url: page.url.origin + page.url.pathname,
+          datePublished: new Date(detail.op.createdAt * 1000).toISOString(),
+          author: {
+            "@type": "Person",
+            name: resolveAuthor(detail.op.pubkey, profiles).name,
+          },
+          commentCount: detail.replies.length,
+        }
+      : undefined,
+  );
+
   // Replying needs membership regardless of flags. Only gate a confirmed guest;
   // while membership resolves the editor stays and submit awaits the check, so a
   // member never flashes "Join to reply".
@@ -322,9 +343,12 @@
   });
 </script>
 
-<svelte:head>
-  <title>{detail?.title ?? "Thread"}</title>
-</svelte:head>
+<Meta
+  title={detail?.title ?? "Thread"}
+  description={detail ? excerpt(detail.op.content) : ""}
+  type="article"
+  {jsonLd}
+/>
 
 {#snippet avatar(author: Author)}
   {#if author.picture}