Merge pull request #360 from ekalinin/fix-misc

drop deprecated items, added pubilc path option for simplesitemap, drop node 10

.github/workflows/nodejs.yml

@@ -9,7 +9,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [10.x, 12.x, 14.x]
+        node-version: [12.x, 14.x, 16.x]
 
     steps:
     - uses: actions/checkout@v1

.travis.yml

@@ -1,8 +1,8 @@
 language: node_js
 node_js:
-  - "10"
   - "12"
   - "14"
+  - "16"
 install:
   - npm ci
 script:

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## 7.0.0
+
+### [BREAKING]
+
+- dropped support for Node 10, added support for Node 16
+- removed deprecated createSitemapsAndIndex. use SitemapAndIndexStream or simpleSitemapAndIndex
+- dropped deprecated `getSitemapStream` option for SitemapAndIndexStream that does not return a write stream
+- fixed invalid documentation for #357
+
+### non-breaking
+
+- Added option to simplesitemap `publicBasePath`: allows the user to set the location of sitemap files hosted on the site fixes [#359]
+- bumped dependencies
+
 ## 6.4.0
 
 - added support for content_loc parsing #347 and uploader info attr

api.md

@@ -87,11 +87,11 @@ const sms = new SitemapAndIndexStream({
     const sitemapStream = new SitemapStream();
     const path = `./sitemap-${i}.xml`;
 
-    sitemapStream
+    const ws = sitemapStream
       .pipe(createGzip()) // compress the output of the sitemap
       .pipe(createWriteStream(resolve(path + '.gz'))); // write it to sitemap-NUMBER.xml
 
-    return [new URL(path, 'https://example.com/subdir/').toString(), sitemapStream];
+    return [new URL(path, 'https://example.com/subdir/').toString(), sitemapStream, ws];
   },
 });
 
@@ -103,23 +103,6 @@ lineSeparatedURLsToSitemapOptions(
 .pipe(createWriteStream(resolve('./sitemap-index.xml.gz')));
 ```
 
-## createSitemapsAndIndex
-
-Create several sitemaps and an index automatically from a list of urls. __deprecated__
-
-```js
-const { createSitemapsAndIndex } = require('sitemap')
-createSitemapsAndIndex({
-  urls: [/* list of urls */],
-  targetFolder: 'absolute path to target folder',
-  hostname: 'http://example.com',
-  cacheTime: 600,
-  sitemapName: 'sitemap',
-  sitemapSize: 50000, // number of urls to allow in each sitemap
-  gzip: true, // whether to gzip the files
-})
-```
-
 ## SitemapIndexStream
 
 Writes a sitemap index when given a stream urls.

cli.ts

@@ -12,6 +12,7 @@ import { SitemapStream } from './lib/sitemap-stream';
 import { SitemapAndIndexStream } from './lib/sitemap-index-stream';
 import { URL } from 'url';
 import { createGzip, Gzip } from 'zlib';
+import { WriteStream } from 'node:fs';
 /* eslint-disable-next-line @typescript-eslint/no-var-requires */
 const arg = require('arg');
 
@@ -112,16 +113,17 @@ Use XMLLib to validate your sitemap (requires xmllib)
   }
   const sms = new SitemapAndIndexStream({
     limit,
-    getSitemapStream: (i: number): [string, SitemapStream] => {
+    getSitemapStream: (i: number): [string, SitemapStream, WriteStream] => {
       const sm = new SitemapStream();
       const path = `./sitemap-${i}.xml`;
 
+      let ws: WriteStream;
       if (argv['--gzip']) {
-        sm.pipe(createGzip()).pipe(createWriteStream(path));
+        ws = sm.pipe(createGzip()).pipe(createWriteStream(path));
       } else {
-        sm.pipe(createWriteStream(path));
+        ws = sm.pipe(createWriteStream(path));
       }
-      return [new URL(path, baseURL).toString(), sm];
+      return [new URL(path, baseURL).toString(), sm, ws];
     },
   });
   let oStream: SitemapAndIndexStream | Gzip = lineSeparatedURLsToSitemapOptions(

index.ts

@@ -11,7 +11,6 @@ export {
   IndexTagNames,
   SitemapIndexStream,
   SitemapIndexStreamOptions,
-  createSitemapsAndIndex,
   SitemapAndIndexStream,
   SitemapAndIndexStreamOptions,
 } from './lib/sitemap-index-stream';

lib/sitemap-index-stream.ts

@@ -1,16 +1,5 @@
-import { promisify } from 'util';
-import { URL } from 'url';
-import { stat, createWriteStream } from 'fs';
-import { createGzip } from 'zlib';
-import {
-  Transform,
-  TransformOptions,
-  TransformCallback,
-  Writable,
-} from 'stream';
+import { Transform, TransformOptions, TransformCallback } from 'stream';
 import { IndexItem, SitemapItemLoose, ErrorLevel } from './types';
-import { UndefinedTargetFolder } from './errors';
-import { chunk } from './utils';
 import { SitemapStream, stylesheetInclude } from './sitemap-stream';
 import { element, otag, ctag } from './sitemap-xml';
 import { WriteStream } from 'fs';
@@ -21,7 +10,6 @@ export enum IndexTagNames {
   lastmod = 'lastmod',
 }
 
-const statPromise = promisify(stat);
 const xmlDec = '<?xml version="1.0" encoding="UTF-8"?>';
 
 const sitemapIndexTagStart =
@@ -79,118 +67,25 @@ export class SitemapIndexStream extends Transform {
   }
 }
 
-/**
- * Shortcut for `new SitemapIndex (...)`.
- * Create several sitemaps and an index automatically from a list of urls
- *
- * @deprecated Use SitemapAndIndexStream
- * @param   {Object}        conf
- * @param   {String|Array}  conf.urls
- * @param   {String}        conf.targetFolder where do you want the generated index and maps put
- * @param   {String}        conf.hostname required for index file, will also be used as base url for sitemap items
- * @param   {String}        conf.sitemapName what do you want to name the files it generats
- * @param   {Number}        conf.sitemapSize maximum number of entries a sitemap should have before being split
- * @param   {Boolean}       conf.gzip whether to gzip the files (defaults to true)
- * @return  {SitemapIndex}
- */
-export async function createSitemapsAndIndex({
-  urls,
-  targetFolder,
-  hostname,
-  sitemapName = 'sitemap',
-  sitemapSize = 50000,
-  gzip = true,
-  xslUrl,
-}: {
-  urls: (string | SitemapItemLoose)[];
-  targetFolder: string;
-  hostname?: string;
-  sitemapName?: string;
-  sitemapSize?: number;
-  gzip?: boolean;
-  xslUrl?: string;
-}): Promise<boolean> {
-  const indexStream = new SitemapIndexStream({ xslUrl });
-
-  try {
-    const stats = await statPromise(targetFolder);
-    if (!stats.isDirectory()) {
-      throw new UndefinedTargetFolder();
-    }
-  } catch (e) {
-    throw new UndefinedTargetFolder();
-  }
-
-  const indexWS = createWriteStream(
-    targetFolder + '/' + sitemapName + '-index.xml'
-  );
-  indexStream.pipe(indexWS);
-  const smPromises = chunk(urls, sitemapSize).map(
-    (chunk: (string | SitemapItemLoose)[], idx): Promise<boolean> => {
-      return new Promise((resolve, reject): void => {
-        const extension = '.xml' + (gzip ? '.gz' : '');
-        const filename = sitemapName + '-' + idx + extension;
-        indexStream.write(new URL(filename, hostname).toString());
-
-        const ws = createWriteStream(targetFolder + '/' + filename);
-        const sms = new SitemapStream({ hostname, xslUrl });
-        let pipe: Writable;
-        if (gzip) {
-          pipe = sms.pipe(createGzip()).pipe(ws);
-        } else {
-          pipe = sms.pipe(ws);
-        }
-        chunk.forEach((smi) => sms.write(smi));
-        sms.end();
-        pipe.on('finish', () => resolve(true));
-        pipe.on('error', (e) => reject(e));
-      });
-    }
-  );
-  return Promise.all(smPromises).then(() => {
-    indexStream.end();
-    return true;
-  });
-}
-
 type getSitemapStream = (
   i: number
 ) => [IndexItem | string, SitemapStream, WriteStream];
-/** @deprecated */
-type getSitemapStreamDeprecated = (
-  i: number
-) => [IndexItem | string, SitemapStream];
 
 export interface SitemapAndIndexStreamOptions
   extends SitemapIndexStreamOptions {
   level?: ErrorLevel;
   limit?: number;
   getSitemapStream: getSitemapStream;
 }
-export interface SitemapAndIndexStreamOptionsDeprecated
-  extends SitemapIndexStreamOptions {
-  level?: ErrorLevel;
-  limit?: number;
-  getSitemapStream: getSitemapStreamDeprecated;
-}
 // const defaultSIStreamOpts: SitemapAndIndexStreamOptions = {};
 export class SitemapAndIndexStream extends SitemapIndexStream {
   private i: number;
-  private getSitemapStream: getSitemapStream | getSitemapStreamDeprecated;
+  private getSitemapStream: getSitemapStream;
   private currentSitemap: SitemapStream;
   private currentSitemapPipeline?: WriteStream;
   private idxItem: IndexItem | string;
   private limit: number;
-  /**
-   * @deprecated this version does not properly wait for everything to write before resolving
-   * pass a 3rd param in your return from getSitemapStream that is the writeable stream
-   * to remove this warning
-   */
-  constructor(opts: SitemapAndIndexStreamOptionsDeprecated);
-  constructor(opts: SitemapAndIndexStreamOptions);
-  constructor(
-    opts: SitemapAndIndexStreamOptions | SitemapAndIndexStreamOptionsDeprecated
-  ) {
+  constructor(opts: SitemapAndIndexStreamOptions) {
     opts.objectMode = true;
     super(opts);
     this.i = 0;

lib/sitemap-simple.ts

@@ -5,14 +5,26 @@ import {
 } from '../index';
 import { createGzip } from 'zlib';
 import { createWriteStream, createReadStream, promises } from 'fs';
-import { resolve } from 'path';
+import { normalize, resolve } from 'path';
 import { Readable, pipeline as pline } from 'stream';
 import { SitemapItemLoose } from './types';
 import { promisify } from 'util';
 import { URL } from 'url';
 import { WriteStream } from 'fs';
 
 const pipeline = promisify(pline);
+/**
+ *
+ * @param {object} options -
+ * @param {string} options.hostname - The hostname for all URLs
+ * @param {string} [options.sitemapHostname] - The hostname for the sitemaps if different than hostname
+ * @param {SitemapItemLoose[] | string | Readable | string[]} options.sourceData - The urls you want to make a sitemap out of.
+ * @param {string} options.destinationDir - where to write the sitemaps and index
+ * @param {string} [options.publicBasePath] - where the sitemaps are relative to the hostname. Defaults to root.
+ * @param {number} [options.limit] - how many URLs to write before switching to a new file. Defaults to 50k
+ * @param {boolean} [options.gzip] - whether to compress the written files. Defaults to true
+ * @returns {Promise<void>} an empty promise that resolves when everything is done
+ */
 export const simpleSitemapAndIndex = async ({
   hostname,
   sitemapHostname = hostname, // if different
@@ -23,11 +35,13 @@ export const simpleSitemapAndIndex = async ({
   destinationDir,
   limit = 50000,
   gzip = true,
+  publicBasePath = './',
 }: {
   hostname: string;
   sitemapHostname?: string;
   sourceData: SitemapItemLoose[] | string | Readable | string[];
   destinationDir: string;
+  publicBasePath?: string;
   limit?: number;
   gzip?: boolean;
 }): Promise<void> => {
@@ -40,6 +54,10 @@ export const simpleSitemapAndIndex = async ({
       });
       const path = `./sitemap-${i}.xml`;
       const writePath = resolve(destinationDir, path + (gzip ? '.gz' : ''));
+      if (!publicBasePath.endsWith('/')) {
+        publicBasePath += '/';
+      }
+      const publicPath = normalize(publicBasePath + path);
 
       let pipeline: WriteStream;
       if (gzip) {
@@ -51,7 +69,10 @@ export const simpleSitemapAndIndex = async ({
       }
 
       return [
-        new URL(`${path}${gzip ? '.gz' : ''}`, sitemapHostname).toString(),
+        new URL(
+          `${publicPath}${gzip ? '.gz' : ''}`,
+          sitemapHostname
+        ).toString(),
         sitemapStream,
         pipeline,
       ];