🔍 Improve SEO

apps/web/app/routes/robots[.]txt.ts

@@ -0,0 +1,50 @@
+import { LoaderFunctionArgs } from '@remix-run/cloudflare';
+
+enum PolicyType {
+  UserAgent = 'User-agent',
+  Allow = 'Allow',
+  Disallow = 'Disallow',
+  Sitemap = 'Sitemap',
+  CrawlDelay = 'Crawl-delay',
+}
+
+type Policy = {
+  type: PolicyType;
+  value: string;
+};
+
+function getRobotsText(policies: Policy[]): string {
+  let result = '';
+  for (const { type, value } of policies) {
+    result += `${type}: ${value}\n`;
+  }
+
+  return result;
+}
+
+export async function loader({ request }: LoaderFunctionArgs) {
+  const url = new URL(request.url);
+  const baseUrl = `${url.protocol}//${url.host}`;
+  const robots = getRobotsText([
+    {
+      type: PolicyType.UserAgent,
+      value: '*',
+    },
+    {
+      type: PolicyType.Allow,
+      value: '/',
+    },
+    {
+      type: PolicyType.Sitemap,
+      value: `${baseUrl}/sitemap.xml`,
+    },
+  ]);
+
+  return new Response(robots, {
+    headers: {
+      'Content-Type': 'text/plain',
+      'Cache-Control': 'public, max-age=86400, must-revalidate',
+      'Content-Length': new TextEncoder().encode(robots).byteLength.toString(),
+    },
+  });
+}

apps/web/app/routes/sitemap[.]xml.ts

@@ -0,0 +1,21 @@
+import { LoaderFunctionArgs } from '@remix-run/cloudflare';
+
+export async function loader({ request }: LoaderFunctionArgs) {
+  const url = new URL(request.url);
+  const baseUrl = `${url.protocol}//${url.host}`;
+
+  const sitemap = `<?xml version="1.0" encoding="UTF-8"?>
+    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+      <url><loc>${baseUrl}</loc></url>
+      <url><loc>${baseUrl}/web</loc></url>
+    </urlset>
+  `;
+
+  return new Response(sitemap, {
+    status: 200,
+    headers: {
+      'Content-Type': 'application/xml',
+      'Cache-Control': 'public, max-age=86400, must-revalidate',
+    },
+  });
+}

apps/web/public/_headers

@@ -1,3 +1,6 @@
+https://:project.pages.dev/*
+  X-Robots-Tag: noindex
+
 /favicon.ico
   Cache-Control: public, max-age=3600, s-maxage=3600
 /assets/*