Merge pull request #229 from jaredwray/adding-in-robots.txt-support

adding in robots.txt support

src/builder.ts

@@ -47,6 +47,8 @@ export class WritrBuilder {
 		// build the sitemap (/sitemap.xml)
 
 		// build the robots.txt (/robots.txt)
+		await this.buildRobotsPage(this.options);
+
 		console.log('build');
 	}
 
@@ -114,4 +116,14 @@ export class WritrBuilder {
 
 		return result;
 	}
+
+	public async buildRobotsPage(options: WritrOptions): Promise<void> {
+		const {sitePath} = options;
+		const {outputPath} = options;
+		const robotsPath = `${outputPath}/robots.txt`;
+
+		await fs.ensureDir(outputPath);
+
+		await (await fs.pathExists(`${sitePath}/robots.txt`) ? fs.copy(`${sitePath}/robots.txt`, robotsPath) : fs.writeFile(robotsPath, 'User-agent: *\nDisallow:'));
+	}
 }

test/builder.test.ts

@@ -1,4 +1,5 @@
 import {expect, it, describe} from 'vitest';
+import * as fs from 'fs-extra';
 import {WritrBuilder} from '../src/builder.js';
 import {WritrOptions} from '../src/options.js';
 
@@ -16,12 +17,18 @@ describe('WritrBuilder', () => {
 	it('should build', async () => {
 		const builder = new WritrBuilder();
 		const consoleLog = console.log;
+		const options = new WritrOptions();
+		options.outputPath = 'test/temp-build-test';
 		let consoleMessage = '';
 		console.log = message => {
 			consoleMessage = message as string;
 		};
 
-		await builder.build();
+		try {
+			await builder.build();
+		} finally {
+			await fs.remove(builder.options.outputPath);
+		}
 
 		expect(consoleMessage).toBe('build');
 
@@ -99,4 +106,34 @@ describe('WritrBuilder', () => {
 			expect(error.message).toBe('No template path found');
 		}
 	});
+	it('should build the robots.txt (/robots.txt)', async () => {
+		const builder = new WritrBuilder();
+		const options = new WritrOptions();
+		options.sitePath = 'test/fixtures/single-page-site';
+		options.outputPath = 'test/temp-robots-test';
+
+		await fs.remove(options.outputPath);
+		try {
+			await builder.buildRobotsPage(options);
+			const robots = await fs.readFile(`${options.outputPath}/robots.txt`, 'utf8');
+			expect(robots).toBe('User-agent: *\nDisallow:');
+		} finally {
+			await fs.remove(options.outputPath);
+		}
+	});
+	it('should copy the robots.txt (/robots.txt)', async () => {
+		const builder = new WritrBuilder();
+		const options = new WritrOptions();
+		options.sitePath = 'test/fixtures/multi-page-site';
+		options.outputPath = 'test/temp-robots-test-copy';
+
+		await fs.remove(options.outputPath);
+		try {
+			await builder.buildRobotsPage(options);
+			const robots = await fs.readFile(`${options.outputPath}/robots.txt`, 'utf8');
+			expect(robots).toBe('User-agent: *\nDisallow: /meow');
+		} finally {
+			await fs.remove(options.outputPath);
+		}
+	});
 });

test/fixtures/multi-page-site/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /meow