From af73d5c8ce73ab15b9557277e5970aa2282d179e Mon Sep 17 00:00:00 2001 From: ktym4a Date: Sun, 28 Jan 2024 01:11:07 +0700 Subject: [PATCH 01/10] Add option to prefix sitemap --- .changeset/eighty-falcons-tease.md | 5 ++ packages/integrations/sitemap/src/index.ts | 31 +++++--- packages/integrations/sitemap/src/schema.ts | 2 + .../integrations/sitemap/test/prefix.test.js | 71 +++++++++++++++++++ 4 files changed, 100 insertions(+), 9 deletions(-) create mode 100644 .changeset/eighty-falcons-tease.md create mode 100644 packages/integrations/sitemap/test/prefix.test.js diff --git a/.changeset/eighty-falcons-tease.md b/.changeset/eighty-falcons-tease.md new file mode 100644 index 000000000000..49d31276a528 --- /dev/null +++ b/.changeset/eighty-falcons-tease.md @@ -0,0 +1,5 @@ +--- +"@astrojs/sitemap": patch +--- + +Added option to prefix sitemap (default is still sitemap-\*) diff --git a/packages/integrations/sitemap/src/index.ts b/packages/integrations/sitemap/src/index.ts index c254fb662bcf..c8648dafa9ea 100644 --- a/packages/integrations/sitemap/src/index.ts +++ b/packages/integrations/sitemap/src/index.ts @@ -1,12 +1,14 @@ import type { AstroConfig, AstroIntegration } from 'astro'; -import path from 'node:path'; +import path, { resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import type { EnumChangefreq, LinkItem as LinkItemBase, SitemapItemLoose } from 'sitemap'; -import { simpleSitemapAndIndex } from 'sitemap'; +import { SitemapAndIndexStream, SitemapStream, streamToPromise } from 'sitemap'; import { ZodError } from 'zod'; import { generateSitemap } from './generate-sitemap.js'; import { validateOptions } from './validate-options.js'; +import { createWriteStream } from 'fs'; +import { Readable } from 'node:stream'; export { EnumChangefreq as ChangeFreqEnum } from 'sitemap'; export type ChangeFreq = `${EnumChangefreq}`; @@ -33,6 +35,8 @@ export type SitemapOptions = lastmod?: Date; priority?: number; + prefix?: string; + // called for each sitemap item just before to save them on disk, sync or async serialize?(item: SitemapItem): SitemapItem | Promise | undefined; } @@ -44,7 +48,6 @@ function formatConfigErrorMessage(err: ZodError) { } const PKG_NAME = '@astrojs/sitemap'; -const OUTFILE = 'sitemap-index.xml'; const STATUS_CODE_PAGES = new Set(['404', '500']); function isStatusCodePage(pathname: string): boolean { @@ -77,7 +80,8 @@ const createPlugin = (options?: SitemapOptions): AstroIntegration => { const opts = validateOptions(config.site, options); - const { filter, customPages, serialize, entryLimit } = opts; + const { filter, customPages, serialize, entryLimit, prefix = 'sitemap-' } = opts; + const OUTFILE = `${prefix}index.xml`; let finalSiteUrl: URL; if (config.site) { @@ -166,13 +170,22 @@ const createPlugin = (options?: SitemapOptions): AstroIntegration => { } } const destDir = fileURLToPath(dir); - await simpleSitemapAndIndex({ - hostname: finalSiteUrl.href, - destinationDir: destDir, - sourceData: urlData, + + const sms = new SitemapAndIndexStream({ limit: entryLimit, - gzip: false, + getSitemapStream: (i) => { + const sitemapStream = new SitemapStream({ hostname: finalSiteUrl.href }); + const fileName = `${prefix}${i}.xml`; + + const ws = sitemapStream.pipe(createWriteStream(resolve(destDir + fileName))); + + return [new URL(fileName, finalSiteUrl.href).toString(), sitemapStream, ws]; + }, }); + + sms.pipe(createWriteStream(resolve(resolve(destDir + OUTFILE)))); + await streamToPromise(Readable.from(urlData).pipe(sms)); + sms.end(); logger.info(`\`${OUTFILE}\` created at \`${path.relative(process.cwd(), destDir)}\``); } catch (err) { if (err instanceof ZodError) { diff --git a/packages/integrations/sitemap/src/schema.ts b/packages/integrations/sitemap/src/schema.ts index a7682e881a05..81272ddb975c 100644 --- a/packages/integrations/sitemap/src/schema.ts +++ b/packages/integrations/sitemap/src/schema.ts @@ -34,6 +34,8 @@ export const SitemapOptionsSchema = z changefreq: z.nativeEnum(ChangeFreq).optional(), lastmod: z.date().optional(), priority: z.number().min(0).max(1).optional(), + + prefix: z.string().optional(), }) .strict() .default(SITEMAP_CONFIG_DEFAULTS); diff --git a/packages/integrations/sitemap/test/prefix.test.js b/packages/integrations/sitemap/test/prefix.test.js new file mode 100644 index 000000000000..f1ac644352d9 --- /dev/null +++ b/packages/integrations/sitemap/test/prefix.test.js @@ -0,0 +1,71 @@ +import { loadFixture, readXML } from './test-utils.js'; +import { expect } from 'chai'; +import { sitemap } from './fixtures/static/deps.mjs'; + +describe('Prefix support', () => { + /** @type {import('./test-utils.js').Fixture} */ + let fixture; + let prefix = 'test-'; + + describe('Static', () => { + before(async () => { + fixture = await loadFixture({ + root: './fixtures/static/', + integrations: [ + sitemap(), + sitemap({ + prefix, + }), + ], + }); + await fixture.build(); + }); + + it('Content is same', async () => { + const data = await readXML(fixture.readFile('/sitemap-0.xml')); + const prefixData = await readXML(fixture.readFile(`/${prefix}0.xml`)); + expect(prefixData).to.deep.equal(data); + }); + + it('Index file load correct sitemap', async () => { + const data = await readXML(fixture.readFile('/sitemap-index.xml')); + const sitemapUrl = data.sitemapindex.sitemap[0].loc[0]; + expect(sitemapUrl).to.equal('http://example.com/sitemap-0.xml'); + + const prefixData = await readXML(fixture.readFile(`/${prefix}index.xml`)); + const prefixSitemapUrl = prefixData.sitemapindex.sitemap[0].loc[0]; + expect(prefixSitemapUrl).to.equal(`http://example.com/${prefix}0.xml`); + }); + }); + + describe('SSR', () => { + before(async () => { + fixture = await loadFixture({ + root: './fixtures/ssr/', + integrations: [ + sitemap(), + sitemap({ + prefix, + }), + ], + }); + await fixture.build(); + }); + + it('Content is same', async () => { + const data = await readXML(fixture.readFile('/client/sitemap-0.xml')); + const prefixData = await readXML(fixture.readFile(`/client/${prefix}0.xml`)); + expect(prefixData).to.deep.equal(data); + }); + + it('Index file load correct sitemap', async () => { + const data = await readXML(fixture.readFile('/client/sitemap-index.xml')); + const sitemapUrl = data.sitemapindex.sitemap[0].loc[0]; + expect(sitemapUrl).to.equal('http://example.com/sitemap-0.xml'); + + const prefixData = await readXML(fixture.readFile(`/client/${prefix}index.xml`)); + const prefixSitemapUrl = prefixData.sitemapindex.sitemap[0].loc[0]; + expect(prefixSitemapUrl).to.equal(`http://example.com/${prefix}0.xml`); + }); + }); +}); From 013bc8e0ead7ba5e80fdf216395576d16a3f1587 Mon Sep 17 00:00:00 2001 From: ktym4a Date: Sun, 28 Jan 2024 01:28:25 +0700 Subject: [PATCH 02/10] Fix call resolve twice --- packages/integrations/sitemap/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/integrations/sitemap/src/index.ts b/packages/integrations/sitemap/src/index.ts index c8648dafa9ea..39acca0ff97e 100644 --- a/packages/integrations/sitemap/src/index.ts +++ b/packages/integrations/sitemap/src/index.ts @@ -183,7 +183,7 @@ const createPlugin = (options?: SitemapOptions): AstroIntegration => { }, }); - sms.pipe(createWriteStream(resolve(resolve(destDir + OUTFILE)))); + sms.pipe(createWriteStream(resolve(destDir + OUTFILE))); await streamToPromise(Readable.from(urlData).pipe(sms)); sms.end(); logger.info(`\`${OUTFILE}\` created at \`${path.relative(process.cwd(), destDir)}\``); From b596506a242d05ffc112af2aaa008f29aa6a7f9a Mon Sep 17 00:00:00 2001 From: ktym4a Date: Sun, 28 Jan 2024 01:44:34 +0700 Subject: [PATCH 03/10] let to const --- packages/integrations/sitemap/test/prefix.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/integrations/sitemap/test/prefix.test.js b/packages/integrations/sitemap/test/prefix.test.js index f1ac644352d9..ee1e3c88280c 100644 --- a/packages/integrations/sitemap/test/prefix.test.js +++ b/packages/integrations/sitemap/test/prefix.test.js @@ -5,7 +5,7 @@ import { sitemap } from './fixtures/static/deps.mjs'; describe('Prefix support', () => { /** @type {import('./test-utils.js').Fixture} */ let fixture; - let prefix = 'test-'; + const prefix = 'test-'; describe('Static', () => { before(async () => { From 4c6a69eca0f220662078c07c706233b8d88ae1b7 Mon Sep 17 00:00:00 2001 From: ktym4a Date: Tue, 13 Feb 2024 19:54:31 +0700 Subject: [PATCH 04/10] Apply suggestions from code review Co-authored-by: Emanuele Stoppa --- packages/integrations/sitemap/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/integrations/sitemap/src/index.ts b/packages/integrations/sitemap/src/index.ts index 39acca0ff97e..4c049df9170b 100644 --- a/packages/integrations/sitemap/src/index.ts +++ b/packages/integrations/sitemap/src/index.ts @@ -7,7 +7,7 @@ import { ZodError } from 'zod'; import { generateSitemap } from './generate-sitemap.js'; import { validateOptions } from './validate-options.js'; -import { createWriteStream } from 'fs'; +import { createWriteStream } from 'node:fs'; import { Readable } from 'node:stream'; export { EnumChangefreq as ChangeFreqEnum } from 'sitemap'; From 1f192b1d8a5f68b9d5eb5f3dd9340bcc5dcf1fee Mon Sep 17 00:00:00 2001 From: ktym4a Date: Tue, 13 Feb 2024 20:05:37 +0700 Subject: [PATCH 05/10] change changeset patch to minor --- .changeset/eighty-falcons-tease.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/eighty-falcons-tease.md b/.changeset/eighty-falcons-tease.md index 49d31276a528..5bea6a70f640 100644 --- a/.changeset/eighty-falcons-tease.md +++ b/.changeset/eighty-falcons-tease.md @@ -1,5 +1,5 @@ --- -"@astrojs/sitemap": patch +"@astrojs/sitemap": minor --- Added option to prefix sitemap (default is still sitemap-\*) From 76d7c3fa7f7ef4b8a5858b0ad2f74902dae26a96 Mon Sep 17 00:00:00 2001 From: ktym4a Date: Tue, 13 Feb 2024 20:11:35 +0700 Subject: [PATCH 06/10] use node:test --- packages/integrations/sitemap/test/prefix.test.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/integrations/sitemap/test/prefix.test.js b/packages/integrations/sitemap/test/prefix.test.js index ee1e3c88280c..fdd538d0f3cd 100644 --- a/packages/integrations/sitemap/test/prefix.test.js +++ b/packages/integrations/sitemap/test/prefix.test.js @@ -1,6 +1,7 @@ import { loadFixture, readXML } from './test-utils.js'; -import { expect } from 'chai'; import { sitemap } from './fixtures/static/deps.mjs'; +import assert from 'node:assert/strict'; +import { before, describe, it } from 'node:test'; describe('Prefix support', () => { /** @type {import('./test-utils.js').Fixture} */ @@ -24,17 +25,17 @@ describe('Prefix support', () => { it('Content is same', async () => { const data = await readXML(fixture.readFile('/sitemap-0.xml')); const prefixData = await readXML(fixture.readFile(`/${prefix}0.xml`)); - expect(prefixData).to.deep.equal(data); + assert.deepEqual(prefixData, data); }); it('Index file load correct sitemap', async () => { const data = await readXML(fixture.readFile('/sitemap-index.xml')); const sitemapUrl = data.sitemapindex.sitemap[0].loc[0]; - expect(sitemapUrl).to.equal('http://example.com/sitemap-0.xml'); + assert.strictEqual(sitemapUrl, 'http://example.com/sitemap-0.xml'); const prefixData = await readXML(fixture.readFile(`/${prefix}index.xml`)); const prefixSitemapUrl = prefixData.sitemapindex.sitemap[0].loc[0]; - expect(prefixSitemapUrl).to.equal(`http://example.com/${prefix}0.xml`); + assert.strictEqual(prefixSitemapUrl, `http://example.com/${prefix}0.xml`); }); }); @@ -55,17 +56,17 @@ describe('Prefix support', () => { it('Content is same', async () => { const data = await readXML(fixture.readFile('/client/sitemap-0.xml')); const prefixData = await readXML(fixture.readFile(`/client/${prefix}0.xml`)); - expect(prefixData).to.deep.equal(data); + assert.deepEqual(prefixData, data); }); it('Index file load correct sitemap', async () => { const data = await readXML(fixture.readFile('/client/sitemap-index.xml')); const sitemapUrl = data.sitemapindex.sitemap[0].loc[0]; - expect(sitemapUrl).to.equal('http://example.com/sitemap-0.xml'); + assert.strictEqual(sitemapUrl, 'http://example.com/sitemap-0.xml'); const prefixData = await readXML(fixture.readFile(`/client/${prefix}index.xml`)); const prefixSitemapUrl = prefixData.sitemapindex.sitemap[0].loc[0]; - expect(prefixSitemapUrl).to.equal(`http://example.com/${prefix}0.xml`); + assert.strictEqual(prefixSitemapUrl, `http://example.com/${prefix}0.xml`); }); }); }); From c0cd345e5318db7d5d15281783257666cc001a65 Mon Sep 17 00:00:00 2001 From: ktym4a Date: Tue, 13 Feb 2024 21:44:07 +0700 Subject: [PATCH 07/10] Update changeset --- .changeset/eighty-falcons-tease.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.changeset/eighty-falcons-tease.md b/.changeset/eighty-falcons-tease.md index 5bea6a70f640..00906eac0a21 100644 --- a/.changeset/eighty-falcons-tease.md +++ b/.changeset/eighty-falcons-tease.md @@ -2,4 +2,23 @@ "@astrojs/sitemap": minor --- -Added option to prefix sitemap (default is still sitemap-\*) +Adds a new configuration option `prefix` that allows you to change the default `sitemap-*.xml` file name. + +By default, running `astro build` creates both `sitemap-index.xml` and `sitemap-0.xml` in your output directory. + +To change the names of these files (e.g. to `astrosite-index.xml` and `astrosite-0.xml`), set the `prefix` option in your `sitemap` integration configuration: + +``` +import { defineConfig } from 'astro/config'; +import sitemap from '@astrojs/sitemap'; +export default defineConfig({ + site: 'https://example.com', + integrations: [ + sitemap({ + prefix: 'astrosite-', + }), + ], +}); +``` + +This option is useful when you submit a sitemap to Google Search Console and the Status becomes `Cloudn't fetch` From 66f52bdf96f9356870ded96e9349396a3c6c888f Mon Sep 17 00:00:00 2001 From: ktym4a Date: Tue, 13 Feb 2024 23:02:56 +0700 Subject: [PATCH 08/10] Add regex validation for prefix --- packages/integrations/sitemap/src/schema.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/integrations/sitemap/src/schema.ts b/packages/integrations/sitemap/src/schema.ts index 81272ddb975c..b2cb1eea248b 100644 --- a/packages/integrations/sitemap/src/schema.ts +++ b/packages/integrations/sitemap/src/schema.ts @@ -35,7 +35,12 @@ export const SitemapOptionsSchema = z lastmod: z.date().optional(), priority: z.number().min(0).max(1).optional(), - prefix: z.string().optional(), + prefix: z + .string() + .regex(/^[a-zA-Z\-]+$/gm, { + message: 'Only English alphabet symbols, hyphen and underscore allowed', + }) + .optional(), }) .strict() .default(SITEMAP_CONFIG_DEFAULTS); From 1889d67f7963281cbe8750e3d446fe1eb41743a6 Mon Sep 17 00:00:00 2001 From: ktym4a Date: Wed, 14 Feb 2024 00:17:42 +0700 Subject: [PATCH 09/10] Update .changeset/eighty-falcons-tease.md Co-authored-by: Sarah Rainsberger --- .changeset/eighty-falcons-tease.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/eighty-falcons-tease.md b/.changeset/eighty-falcons-tease.md index 00906eac0a21..7ea106f1e86b 100644 --- a/.changeset/eighty-falcons-tease.md +++ b/.changeset/eighty-falcons-tease.md @@ -21,4 +21,4 @@ export default defineConfig({ }); ``` -This option is useful when you submit a sitemap to Google Search Console and the Status becomes `Cloudn't fetch` +This option is useful when Google Search Console is unable to fetch your default sitemap files, but can read renamed files. From 8d96f1ae7e684601dab77726a2669b1747c146d0 Mon Sep 17 00:00:00 2001 From: ktym4a Date: Wed, 14 Feb 2024 01:50:09 +0700 Subject: [PATCH 10/10] Update prefix regex in SitemapOptionsSchema --- packages/integrations/sitemap/src/schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/integrations/sitemap/src/schema.ts b/packages/integrations/sitemap/src/schema.ts index b2cb1eea248b..da629fc0c690 100644 --- a/packages/integrations/sitemap/src/schema.ts +++ b/packages/integrations/sitemap/src/schema.ts @@ -37,7 +37,7 @@ export const SitemapOptionsSchema = z prefix: z .string() - .regex(/^[a-zA-Z\-]+$/gm, { + .regex(/^[a-zA-Z\-_]+$/gm, { message: 'Only English alphabet symbols, hyphen and underscore allowed', }) .optional(),