diff --git a/README.md b/README.md index 3e074c5..c002873 100755 --- a/README.md +++ b/README.md @@ -85,3 +85,6 @@ parser.on('error', error => { console.error(error) }) parser.start() ``` + +## Environment +* `NODEWARC_WRITE_GZIPPED` - writes gzipped records to WARC outputs. \ No newline at end of file diff --git a/index.d.ts b/index.d.ts new file mode 100644 index 0000000..6af1c38 --- /dev/null +++ b/index.d.ts @@ -0,0 +1,1753 @@ +/** Declaration file generated by dts-gen */ + +export = node_warc; + +declare class node_warc { + constructor(...args: any[]); + + parseWARC(...args: any[]): void; + + start(...args: any[]): void; + + static prefixed: boolean; + +} + +declare namespace node_warc { + class ElectronRequestCap { + constructor(...args: any[]); + + attach(...args: any[]): void; + + maybeNetworkMessage(...args: any[]): void; + + withNavigationManager(...args: any[]): void; + + } + + class ElectronWARCWriter { + constructor(...args: any[]); + + generateWarcEntry(...args: any[]): void; + + static prefixed: boolean; + + } + + class EventEmitter { + constructor(); + + addListener(event: any, fn: any, context: any): any; + + emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + eventNames(): any; + + listenerCount(event: any): any; + + listeners(event: any): any; + + off(event: any, fn: any, context: any, once: any): any; + + on(event: any, fn: any, context: any): any; + + once(event: any, fn: any, context: any): any; + + removeAllListeners(event: any): any; + + removeListener(event: any, fn: any, context: any, once: any): any; + + static EventEmitter: any; + + static prefixed: boolean; + + } + + class PuppeteerRequestCap { + constructor(...args: any[]); + + clear(...args: any[]): void; + + iterateRequests(...args: any[]): any[]; + + requestWillBeSent(...args: any[]): void; + + requests(...args: any[]): void; + + startCapturing(...args: any[]): void; + + stopCapturing(...args: any[]): void; + + } + + class PuppeteerWARCWriter extends WARCWriterBase { + constructor(...args: any[]); + + generateWarcEntry(...args: any[]): void; + + static prefixed: boolean; + + } + + class RemoteChromeRequestCap { + constructor(...args: any[]); + + withNavigationManager(...args: any[]): void; + + } + + class RemoteChromeWARCWriter extends WARCWriterBase { + constructor(...args: any[]); + + generateWarcEntry(...args: any[]): void; + + static prefixed: boolean; + + } + + class WARCGzParser { + constructor(...args: any[]); + + parseWARC(...args: any[]): void; + + start(...args: any[]): void; + + static prefixed: boolean; + + } + + class WARCParser { + constructor(...args: any[]); + + parseWARC(...args: any[]): void; + + start(...args: any[]): void; + + static prefixed: boolean; + + } + + class WARCWriterBase { + constructor(...args: any[]); + + end(...args: any[]): void; + + initWARC(...args: any[]): void; + + writeRecordBlock(...args: any[]): void; + + writeRecordChunks(...args: any[]): void; + + writeRequestRecord(...args: any[]): void; + + writeResponseRecord(...args: any[]): void; + + writeWarcInfoRecord(...args: any[]): void; + + writeWarcMetadataOutlinks(...args: any[]): void; + + static prefixed: boolean; + + } + + namespace ElectronRequestCap { + namespace prototype { + function attach(...args: any[]): void; + + function clear(...args: any[]): void; + + function entries(...args: any[]): void; + + function forEach(...args: any[]): void; + + function get(...args: any[]): void; + + function has(...args: any[]): void; + + function iterateRequests(...args: any[]): void; + + function keys(...args: any[]): void; + + function loadingFailed(...args: any[]): void; + + function loadingFinished(...args: any[]): void; + + function maybeNetworkMessage(...args: any[]): void; + + function requestWillBeSent(...args: any[]): void; + + function responseReceived(...args: any[]): void; + + function size(...args: any[]): void; + + function startCapturing(...args: any[]): void; + + function stopCapturing(...args: any[]): void; + + function values(...args: any[]): void; + + function withNavigationManager(...args: any[]): void; + + namespace iterateRequests { + namespace prototype { + // Too-deep object hierarchy from node_warc.ElectronRequestCap.prototype.iterateRequests.prototype + const next: any; + + // Too-deep object hierarchy from node_warc.ElectronRequestCap.prototype.iterateRequests.prototype + const return: any; + + // Too-deep object hierarchy from node_warc.ElectronRequestCap.prototype.iterateRequests.prototype + const throw: any; + + } + + } + + } + + } + + namespace ElectronWARCWriter { + class EventEmitter { + constructor(); + + addListener(event: any, fn: any, context: any): any; + + emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + eventNames(): any; + + listenerCount(event: any): any; + + listeners(event: any): any; + + off(event: any, fn: any, context: any, once: any): any; + + on(event: any, fn: any, context: any): any; + + once(event: any, fn: any, context: any): any; + + removeAllListeners(event: any): any; + + removeListener(event: any, fn: any, context: any, once: any): any; + + static EventEmitter: any; + + static prefixed: boolean; + + } + + namespace EventEmitter { + namespace prototype { + function addListener(event: any, fn: any, context: any): any; + + function emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + function eventNames(): any; + + function listenerCount(event: any): any; + + function listeners(event: any): any; + + function off(event: any, fn: any, context: any, once: any): any; + + function on(event: any, fn: any, context: any): any; + + function once(event: any, fn: any, context: any): any; + + function removeAllListeners(event: any): any; + + function removeListener(event: any, fn: any, context: any, once: any): any; + + namespace addListener { + // Too-deep object hierarchy from node_warc.ElectronWARCWriter.EventEmitter.prototype.addListener + const prototype: any; + + } + + namespace emit { + // Too-deep object hierarchy from node_warc.ElectronWARCWriter.EventEmitter.prototype.emit + const prototype: any; + + } + + namespace eventNames { + // Too-deep object hierarchy from node_warc.ElectronWARCWriter.EventEmitter.prototype.eventNames + const prototype: any; + + } + + namespace listenerCount { + // Too-deep object hierarchy from node_warc.ElectronWARCWriter.EventEmitter.prototype.listenerCount + const prototype: any; + + } + + namespace listeners { + // Too-deep object hierarchy from node_warc.ElectronWARCWriter.EventEmitter.prototype.listeners + const prototype: any; + + } + + namespace off { + // Too-deep object hierarchy from node_warc.ElectronWARCWriter.EventEmitter.prototype.off + const prototype: any; + + } + + namespace on { + // Too-deep object hierarchy from node_warc.ElectronWARCWriter.EventEmitter.prototype.on + const prototype: any; + + } + + namespace once { + // Too-deep object hierarchy from node_warc.ElectronWARCWriter.EventEmitter.prototype.once + const prototype: any; + + } + + namespace removeAllListeners { + // Too-deep object hierarchy from node_warc.ElectronWARCWriter.EventEmitter.prototype.removeAllListeners + const prototype: any; + + } + + namespace removeListener { + // Too-deep object hierarchy from node_warc.ElectronWARCWriter.EventEmitter.prototype.removeListener + const prototype: any; + + } + + } + + } + + namespace prototype { + function addListener(event: any, fn: any, context: any): any; + + function emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + function end(...args: any[]): void; + + function eventNames(): any; + + function generateWarcEntry(...args: any[]): void; + + function initWARC(...args: any[]): void; + + function listenerCount(event: any): any; + + function listeners(event: any): any; + + function off(event: any, fn: any, context: any, once: any): any; + + function on(event: any, fn: any, context: any): any; + + function once(event: any, fn: any, context: any): any; + + function removeAllListeners(event: any): any; + + function removeListener(event: any, fn: any, context: any, once: any): any; + + function writeRecordBlock(...args: any[]): void; + + function writeRecordChunks(...args: any[]): void; + + function writeRequestRecord(...args: any[]): void; + + function writeResponseRecord(...args: any[]): void; + + function writeWarcInfoRecord(...args: any[]): void; + + function writeWarcMetadataOutlinks(...args: any[]): void; + + namespace addListener { + const prototype: { + }; + + } + + namespace emit { + const prototype: { + }; + + } + + namespace eventNames { + const prototype: { + }; + + } + + namespace listenerCount { + const prototype: { + }; + + } + + namespace listeners { + const prototype: { + }; + + } + + namespace off { + const prototype: { + }; + + } + + namespace on { + const prototype: { + }; + + } + + namespace once { + const prototype: { + }; + + } + + namespace removeAllListeners { + const prototype: { + }; + + } + + namespace removeListener { + const prototype: { + }; + + } + + } + + } + + namespace EventEmitter { + namespace prototype { + function addListener(event: any, fn: any, context: any): any; + + function emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + function eventNames(): any; + + function listenerCount(event: any): any; + + function listeners(event: any): any; + + function off(event: any, fn: any, context: any, once: any): any; + + function on(event: any, fn: any, context: any): any; + + function once(event: any, fn: any, context: any): any; + + function removeAllListeners(event: any): any; + + function removeListener(event: any, fn: any, context: any, once: any): any; + + namespace addListener { + const prototype: { + }; + + } + + namespace emit { + const prototype: { + }; + + } + + namespace eventNames { + const prototype: { + }; + + } + + namespace listenerCount { + const prototype: { + }; + + } + + namespace listeners { + const prototype: { + }; + + } + + namespace off { + const prototype: { + }; + + } + + namespace on { + const prototype: { + }; + + } + + namespace once { + const prototype: { + }; + + } + + namespace removeAllListeners { + const prototype: { + }; + + } + + namespace removeListener { + const prototype: { + }; + + } + + } + + } + + namespace PuppeteerRequestCap { + namespace prototype { + function clear(...args: any[]): void; + + function iterateRequests(...args: any[]): void; + + function requestWillBeSent(...args: any[]): void; + + function requests(...args: any[]): void; + + function startCapturing(...args: any[]): void; + + function stopCapturing(...args: any[]): void; + + namespace iterateRequests { + namespace prototype { + // Too-deep object hierarchy from node_warc.PuppeteerRequestCap.prototype.iterateRequests.prototype + const next: any; + + // Too-deep object hierarchy from node_warc.PuppeteerRequestCap.prototype.iterateRequests.prototype + const return: any; + + // Too-deep object hierarchy from node_warc.PuppeteerRequestCap.prototype.iterateRequests.prototype + const throw: any; + + } + + } + + } + + } + + namespace PuppeteerWARCWriter { + class EventEmitter { + constructor(); + + addListener(event: any, fn: any, context: any): any; + + emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + eventNames(): any; + + listenerCount(event: any): any; + + listeners(event: any): any; + + off(event: any, fn: any, context: any, once: any): any; + + on(event: any, fn: any, context: any): any; + + once(event: any, fn: any, context: any): any; + + removeAllListeners(event: any): any; + + removeListener(event: any, fn: any, context: any, once: any): any; + + static EventEmitter: any; + + static prefixed: boolean; + + } + + namespace EventEmitter { + namespace prototype { + function addListener(event: any, fn: any, context: any): any; + + function emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + function eventNames(): any; + + function listenerCount(event: any): any; + + function listeners(event: any): any; + + function off(event: any, fn: any, context: any, once: any): any; + + function on(event: any, fn: any, context: any): any; + + function once(event: any, fn: any, context: any): any; + + function removeAllListeners(event: any): any; + + function removeListener(event: any, fn: any, context: any, once: any): any; + + namespace addListener { + // Too-deep object hierarchy from node_warc.PuppeteerWARCWriter.EventEmitter.prototype.addListener + const prototype: any; + + } + + namespace emit { + // Too-deep object hierarchy from node_warc.PuppeteerWARCWriter.EventEmitter.prototype.emit + const prototype: any; + + } + + namespace eventNames { + // Too-deep object hierarchy from node_warc.PuppeteerWARCWriter.EventEmitter.prototype.eventNames + const prototype: any; + + } + + namespace listenerCount { + // Too-deep object hierarchy from node_warc.PuppeteerWARCWriter.EventEmitter.prototype.listenerCount + const prototype: any; + + } + + namespace listeners { + // Too-deep object hierarchy from node_warc.PuppeteerWARCWriter.EventEmitter.prototype.listeners + const prototype: any; + + } + + namespace off { + // Too-deep object hierarchy from node_warc.PuppeteerWARCWriter.EventEmitter.prototype.off + const prototype: any; + + } + + namespace on { + // Too-deep object hierarchy from node_warc.PuppeteerWARCWriter.EventEmitter.prototype.on + const prototype: any; + + } + + namespace once { + // Too-deep object hierarchy from node_warc.PuppeteerWARCWriter.EventEmitter.prototype.once + const prototype: any; + + } + + namespace removeAllListeners { + // Too-deep object hierarchy from node_warc.PuppeteerWARCWriter.EventEmitter.prototype.removeAllListeners + const prototype: any; + + } + + namespace removeListener { + // Too-deep object hierarchy from node_warc.PuppeteerWARCWriter.EventEmitter.prototype.removeListener + const prototype: any; + + } + + } + + } + + namespace prototype { + function addListener(event: any, fn: any, context: any): any; + + function emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + function end(...args: any[]): void; + + function eventNames(): any; + + function generateWarcEntry(...args: any[]): void; + + function initWARC(...args: any[]): void; + + function listenerCount(event: any): any; + + function listeners(event: any): any; + + function off(event: any, fn: any, context: any, once: any): any; + + function on(event: any, fn: any, context: any): any; + + function once(event: any, fn: any, context: any): any; + + function removeAllListeners(event: any): any; + + function removeListener(event: any, fn: any, context: any, once: any): any; + + function writeRecordBlock(...args: any[]): void; + + function writeRecordChunks(...args: any[]): void; + + function writeRequestRecord(...args: any[]): void; + + function writeResponseRecord(...args: any[]): void; + + function writeWarcInfoRecord(...args: any[]): void; + + function writeWarcMetadataOutlinks(...args: any[]): void; + + namespace addListener { + const prototype: { + }; + + } + + namespace emit { + const prototype: { + }; + + } + + namespace eventNames { + const prototype: { + }; + + } + + namespace listenerCount { + const prototype: { + }; + + } + + namespace listeners { + const prototype: { + }; + + } + + namespace off { + const prototype: { + }; + + } + + namespace on { + const prototype: { + }; + + } + + namespace once { + const prototype: { + }; + + } + + namespace removeAllListeners { + const prototype: { + }; + + } + + namespace removeListener { + const prototype: { + }; + + } + + } + + } + + namespace RemoteChromeRequestCap { + namespace prototype { + function clear(...args: any[]): void; + + function entries(...args: any[]): void; + + function forEach(...args: any[]): void; + + function get(...args: any[]): void; + + function has(...args: any[]): void; + + function iterateRequests(...args: any[]): void; + + function keys(...args: any[]): void; + + function loadingFailed(...args: any[]): void; + + function loadingFinished(...args: any[]): void; + + function requestWillBeSent(...args: any[]): void; + + function responseReceived(...args: any[]): void; + + function size(...args: any[]): void; + + function startCapturing(...args: any[]): void; + + function stopCapturing(...args: any[]): void; + + function values(...args: any[]): void; + + function withNavigationManager(...args: any[]): void; + + namespace iterateRequests { + namespace prototype { + // Too-deep object hierarchy from node_warc.RemoteChromeRequestCap.prototype.iterateRequests.prototype + const next: any; + + // Too-deep object hierarchy from node_warc.RemoteChromeRequestCap.prototype.iterateRequests.prototype + const return: any; + + // Too-deep object hierarchy from node_warc.RemoteChromeRequestCap.prototype.iterateRequests.prototype + const throw: any; + + } + + } + + } + + } + + namespace RemoteChromeWARCWriter { + class EventEmitter { + constructor(); + + addListener(event: any, fn: any, context: any): any; + + emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + eventNames(): any; + + listenerCount(event: any): any; + + listeners(event: any): any; + + off(event: any, fn: any, context: any, once: any): any; + + on(event: any, fn: any, context: any): any; + + once(event: any, fn: any, context: any): any; + + removeAllListeners(event: any): any; + + removeListener(event: any, fn: any, context: any, once: any): any; + + static EventEmitter: any; + + static prefixed: boolean; + + } + + namespace EventEmitter { + namespace prototype { + function addListener(event: any, fn: any, context: any): any; + + function emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + function eventNames(): any; + + function listenerCount(event: any): any; + + function listeners(event: any): any; + + function off(event: any, fn: any, context: any, once: any): any; + + function on(event: any, fn: any, context: any): any; + + function once(event: any, fn: any, context: any): any; + + function removeAllListeners(event: any): any; + + function removeListener(event: any, fn: any, context: any, once: any): any; + + namespace addListener { + // Too-deep object hierarchy from node_warc.RemoteChromeWARCWriter.EventEmitter.prototype.addListener + const prototype: any; + + } + + namespace emit { + // Too-deep object hierarchy from node_warc.RemoteChromeWARCWriter.EventEmitter.prototype.emit + const prototype: any; + + } + + namespace eventNames { + // Too-deep object hierarchy from node_warc.RemoteChromeWARCWriter.EventEmitter.prototype.eventNames + const prototype: any; + + } + + namespace listenerCount { + // Too-deep object hierarchy from node_warc.RemoteChromeWARCWriter.EventEmitter.prototype.listenerCount + const prototype: any; + + } + + namespace listeners { + // Too-deep object hierarchy from node_warc.RemoteChromeWARCWriter.EventEmitter.prototype.listeners + const prototype: any; + + } + + namespace off { + // Too-deep object hierarchy from node_warc.RemoteChromeWARCWriter.EventEmitter.prototype.off + const prototype: any; + + } + + namespace on { + // Too-deep object hierarchy from node_warc.RemoteChromeWARCWriter.EventEmitter.prototype.on + const prototype: any; + + } + + namespace once { + // Too-deep object hierarchy from node_warc.RemoteChromeWARCWriter.EventEmitter.prototype.once + const prototype: any; + + } + + namespace removeAllListeners { + // Too-deep object hierarchy from node_warc.RemoteChromeWARCWriter.EventEmitter.prototype.removeAllListeners + const prototype: any; + + } + + namespace removeListener { + // Too-deep object hierarchy from node_warc.RemoteChromeWARCWriter.EventEmitter.prototype.removeListener + const prototype: any; + + } + + } + + } + + namespace prototype { + function addListener(event: any, fn: any, context: any): any; + + function emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + function end(...args: any[]): void; + + function eventNames(): any; + + function generateWarcEntry(...args: any[]): void; + + function initWARC(...args: any[]): void; + + function listenerCount(event: any): any; + + function listeners(event: any): any; + + function off(event: any, fn: any, context: any, once: any): any; + + function on(event: any, fn: any, context: any): any; + + function once(event: any, fn: any, context: any): any; + + function removeAllListeners(event: any): any; + + function removeListener(event: any, fn: any, context: any, once: any): any; + + function writeRecordBlock(...args: any[]): void; + + function writeRecordChunks(...args: any[]): void; + + function writeRequestRecord(...args: any[]): void; + + function writeResponseRecord(...args: any[]): void; + + function writeWarcInfoRecord(...args: any[]): void; + + function writeWarcMetadataOutlinks(...args: any[]): void; + + namespace addListener { + const prototype: { + }; + + } + + namespace emit { + const prototype: { + }; + + } + + namespace eventNames { + const prototype: { + }; + + } + + namespace listenerCount { + const prototype: { + }; + + } + + namespace listeners { + const prototype: { + }; + + } + + namespace off { + const prototype: { + }; + + } + + namespace on { + const prototype: { + }; + + } + + namespace once { + const prototype: { + }; + + } + + namespace removeAllListeners { + const prototype: { + }; + + } + + namespace removeListener { + const prototype: { + }; + + } + + } + + } + + namespace WARCGzParser { + class EventEmitter { + constructor(); + + addListener(event: any, fn: any, context: any): any; + + emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + eventNames(): any; + + listenerCount(event: any): any; + + listeners(event: any): any; + + off(event: any, fn: any, context: any, once: any): any; + + on(event: any, fn: any, context: any): any; + + once(event: any, fn: any, context: any): any; + + removeAllListeners(event: any): any; + + removeListener(event: any, fn: any, context: any, once: any): any; + + static EventEmitter: any; + + static prefixed: boolean; + + } + + namespace EventEmitter { + namespace prototype { + function addListener(event: any, fn: any, context: any): any; + + function emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + function eventNames(): any; + + function listenerCount(event: any): any; + + function listeners(event: any): any; + + function off(event: any, fn: any, context: any, once: any): any; + + function on(event: any, fn: any, context: any): any; + + function once(event: any, fn: any, context: any): any; + + function removeAllListeners(event: any): any; + + function removeListener(event: any, fn: any, context: any, once: any): any; + + namespace addListener { + // Too-deep object hierarchy from node_warc.WARCGzParser.EventEmitter.prototype.addListener + const prototype: any; + + } + + namespace emit { + // Too-deep object hierarchy from node_warc.WARCGzParser.EventEmitter.prototype.emit + const prototype: any; + + } + + namespace eventNames { + // Too-deep object hierarchy from node_warc.WARCGzParser.EventEmitter.prototype.eventNames + const prototype: any; + + } + + namespace listenerCount { + // Too-deep object hierarchy from node_warc.WARCGzParser.EventEmitter.prototype.listenerCount + const prototype: any; + + } + + namespace listeners { + // Too-deep object hierarchy from node_warc.WARCGzParser.EventEmitter.prototype.listeners + const prototype: any; + + } + + namespace off { + // Too-deep object hierarchy from node_warc.WARCGzParser.EventEmitter.prototype.off + const prototype: any; + + } + + namespace on { + // Too-deep object hierarchy from node_warc.WARCGzParser.EventEmitter.prototype.on + const prototype: any; + + } + + namespace once { + // Too-deep object hierarchy from node_warc.WARCGzParser.EventEmitter.prototype.once + const prototype: any; + + } + + namespace removeAllListeners { + // Too-deep object hierarchy from node_warc.WARCGzParser.EventEmitter.prototype.removeAllListeners + const prototype: any; + + } + + namespace removeListener { + // Too-deep object hierarchy from node_warc.WARCGzParser.EventEmitter.prototype.removeListener + const prototype: any; + + } + + } + + } + + namespace prototype { + function addListener(event: any, fn: any, context: any): any; + + function emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + function eventNames(): any; + + function listenerCount(event: any): any; + + function listeners(event: any): any; + + function off(event: any, fn: any, context: any, once: any): any; + + function on(event: any, fn: any, context: any): any; + + function once(event: any, fn: any, context: any): any; + + function parseWARC(...args: any[]): void; + + function removeAllListeners(event: any): any; + + function removeListener(event: any, fn: any, context: any, once: any): any; + + function start(...args: any[]): void; + + namespace addListener { + const prototype: { + }; + + } + + namespace emit { + const prototype: { + }; + + } + + namespace eventNames { + const prototype: { + }; + + } + + namespace listenerCount { + const prototype: { + }; + + } + + namespace listeners { + const prototype: { + }; + + } + + namespace off { + const prototype: { + }; + + } + + namespace on { + const prototype: { + }; + + } + + namespace once { + const prototype: { + }; + + } + + namespace removeAllListeners { + const prototype: { + }; + + } + + namespace removeListener { + const prototype: { + }; + + } + + } + + } + + namespace WARCParser { + class EventEmitter { + constructor(); + + addListener(event: any, fn: any, context: any): any; + + emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + eventNames(): any; + + listenerCount(event: any): any; + + listeners(event: any): any; + + off(event: any, fn: any, context: any, once: any): any; + + on(event: any, fn: any, context: any): any; + + once(event: any, fn: any, context: any): any; + + removeAllListeners(event: any): any; + + removeListener(event: any, fn: any, context: any, once: any): any; + + static EventEmitter: any; + + static prefixed: boolean; + + } + + namespace EventEmitter { + namespace prototype { + function addListener(event: any, fn: any, context: any): any; + + function emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + function eventNames(): any; + + function listenerCount(event: any): any; + + function listeners(event: any): any; + + function off(event: any, fn: any, context: any, once: any): any; + + function on(event: any, fn: any, context: any): any; + + function once(event: any, fn: any, context: any): any; + + function removeAllListeners(event: any): any; + + function removeListener(event: any, fn: any, context: any, once: any): any; + + namespace addListener { + // Too-deep object hierarchy from node_warc.WARCParser.EventEmitter.prototype.addListener + const prototype: any; + + } + + namespace emit { + // Too-deep object hierarchy from node_warc.WARCParser.EventEmitter.prototype.emit + const prototype: any; + + } + + namespace eventNames { + // Too-deep object hierarchy from node_warc.WARCParser.EventEmitter.prototype.eventNames + const prototype: any; + + } + + namespace listenerCount { + // Too-deep object hierarchy from node_warc.WARCParser.EventEmitter.prototype.listenerCount + const prototype: any; + + } + + namespace listeners { + // Too-deep object hierarchy from node_warc.WARCParser.EventEmitter.prototype.listeners + const prototype: any; + + } + + namespace off { + // Too-deep object hierarchy from node_warc.WARCParser.EventEmitter.prototype.off + const prototype: any; + + } + + namespace on { + // Too-deep object hierarchy from node_warc.WARCParser.EventEmitter.prototype.on + const prototype: any; + + } + + namespace once { + // Too-deep object hierarchy from node_warc.WARCParser.EventEmitter.prototype.once + const prototype: any; + + } + + namespace removeAllListeners { + // Too-deep object hierarchy from node_warc.WARCParser.EventEmitter.prototype.removeAllListeners + const prototype: any; + + } + + namespace removeListener { + // Too-deep object hierarchy from node_warc.WARCParser.EventEmitter.prototype.removeListener + const prototype: any; + + } + + } + + } + + namespace prototype { + function addListener(event: any, fn: any, context: any): any; + + function emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + function eventNames(): any; + + function listenerCount(event: any): any; + + function listeners(event: any): any; + + function off(event: any, fn: any, context: any, once: any): any; + + function on(event: any, fn: any, context: any): any; + + function once(event: any, fn: any, context: any): any; + + function parseWARC(...args: any[]): void; + + function removeAllListeners(event: any): any; + + function removeListener(event: any, fn: any, context: any, once: any): any; + + function start(...args: any[]): void; + + namespace addListener { + const prototype: { + }; + + } + + namespace emit { + const prototype: { + }; + + } + + namespace eventNames { + const prototype: { + }; + + } + + namespace listenerCount { + const prototype: { + }; + + } + + namespace listeners { + const prototype: { + }; + + } + + namespace off { + const prototype: { + }; + + } + + namespace on { + const prototype: { + }; + + } + + namespace once { + const prototype: { + }; + + } + + namespace removeAllListeners { + const prototype: { + }; + + } + + namespace removeListener { + const prototype: { + }; + + } + + } + + } + + namespace WARCWriterBase { + class EventEmitter { + constructor(); + + addListener(event: any, fn: any, context: any): any; + + emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + eventNames(): any; + + listenerCount(event: any): any; + + listeners(event: any): any; + + off(event: any, fn: any, context: any, once: any): any; + + on(event: any, fn: any, context: any): any; + + once(event: any, fn: any, context: any): any; + + removeAllListeners(event: any): any; + + removeListener(event: any, fn: any, context: any, once: any): any; + + static EventEmitter: any; + + static prefixed: boolean; + + } + + namespace EventEmitter { + namespace prototype { + function addListener(event: any, fn: any, context: any): any; + + function emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + function eventNames(): any; + + function listenerCount(event: any): any; + + function listeners(event: any): any; + + function off(event: any, fn: any, context: any, once: any): any; + + function on(event: any, fn: any, context: any): any; + + function once(event: any, fn: any, context: any): any; + + function removeAllListeners(event: any): any; + + function removeListener(event: any, fn: any, context: any, once: any): any; + + namespace addListener { + // Too-deep object hierarchy from node_warc.WARCWriterBase.EventEmitter.prototype.addListener + const prototype: any; + + } + + namespace emit { + // Too-deep object hierarchy from node_warc.WARCWriterBase.EventEmitter.prototype.emit + const prototype: any; + + } + + namespace eventNames { + // Too-deep object hierarchy from node_warc.WARCWriterBase.EventEmitter.prototype.eventNames + const prototype: any; + + } + + namespace listenerCount { + // Too-deep object hierarchy from node_warc.WARCWriterBase.EventEmitter.prototype.listenerCount + const prototype: any; + + } + + namespace listeners { + // Too-deep object hierarchy from node_warc.WARCWriterBase.EventEmitter.prototype.listeners + const prototype: any; + + } + + namespace off { + // Too-deep object hierarchy from node_warc.WARCWriterBase.EventEmitter.prototype.off + const prototype: any; + + } + + namespace on { + // Too-deep object hierarchy from node_warc.WARCWriterBase.EventEmitter.prototype.on + const prototype: any; + + } + + namespace once { + // Too-deep object hierarchy from node_warc.WARCWriterBase.EventEmitter.prototype.once + const prototype: any; + + } + + namespace removeAllListeners { + // Too-deep object hierarchy from node_warc.WARCWriterBase.EventEmitter.prototype.removeAllListeners + const prototype: any; + + } + + namespace removeListener { + // Too-deep object hierarchy from node_warc.WARCWriterBase.EventEmitter.prototype.removeListener + const prototype: any; + + } + + } + + } + + namespace prototype { + function addListener(event: any, fn: any, context: any): any; + + function emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + function end(...args: any[]): void; + + function eventNames(): any; + + function initWARC(...args: any[]): void; + + function listenerCount(event: any): any; + + function listeners(event: any): any; + + function off(event: any, fn: any, context: any, once: any): any; + + function on(event: any, fn: any, context: any): any; + + function once(event: any, fn: any, context: any): any; + + function removeAllListeners(event: any): any; + + function removeListener(event: any, fn: any, context: any, once: any): any; + + function writeRecordBlock(...args: any[]): void; + + function writeRecordChunks(...args: any[]): void; + + function writeRequestRecord(...args: any[]): void; + + function writeResponseRecord(...args: any[]): void; + + function writeWarcInfoRecord(...args: any[]): void; + + function writeWarcMetadataOutlinks(...args: any[]): void; + + namespace addListener { + const prototype: { + }; + + } + + namespace emit { + const prototype: { + }; + + } + + namespace eventNames { + const prototype: { + }; + + } + + namespace listenerCount { + const prototype: { + }; + + } + + namespace listeners { + const prototype: { + }; + + } + + namespace off { + const prototype: { + }; + + } + + namespace on { + const prototype: { + }; + + } + + namespace once { + const prototype: { + }; + + } + + namespace removeAllListeners { + const prototype: { + }; + + } + + namespace removeListener { + const prototype: { + }; + + } + + } + + } + + namespace prototype { + function addListener(event: any, fn: any, context: any): any; + + function emit(event: any, a1: any, a2: any, a3: any, a4: any, a5: any, ...args: any[]): any; + + function eventNames(): any; + + function listenerCount(event: any): any; + + function listeners(event: any): any; + + function off(event: any, fn: any, context: any, once: any): any; + + function on(event: any, fn: any, context: any): any; + + function once(event: any, fn: any, context: any): any; + + function parseWARC(...args: any[]): void; + + function removeAllListeners(event: any): any; + + function removeListener(event: any, fn: any, context: any, once: any): any; + + function start(...args: any[]): void; + + namespace addListener { + const prototype: { + }; + + } + + namespace emit { + const prototype: { + }; + + } + + namespace eventNames { + const prototype: { + }; + + } + + namespace listenerCount { + const prototype: { + }; + + } + + namespace listeners { + const prototype: { + }; + + } + + namespace off { + const prototype: { + }; + + } + + namespace on { + const prototype: { + }; + + } + + namespace once { + const prototype: { + }; + + } + + namespace removeAllListeners { + const prototype: { + }; + + } + + namespace removeListener { + const prototype: { + }; + + } + + } + +} diff --git a/lib/writers/warcWriterBase.js b/lib/writers/warcWriterBase.js index a470588..6395efc 100755 --- a/lib/writers/warcWriterBase.js +++ b/lib/writers/warcWriterBase.js @@ -1,4 +1,7 @@ const fs = require('fs-extra') +/** @ignore */ +const zlib = require('zlib'); +/** @ignore */ const Path = require('path') const uuid = require('uuid/v1') const EventEmitter = require('eventemitter3') @@ -245,6 +248,11 @@ class WARCWriterBase extends EventEmitter { */ writeRecordBlock (recordBuffer) { return new Promise((resolve, reject) => { + if (process.env.NODEWARC_WRITE_GZIPPED) { + // we're in gzipped mode - GZip the buffer + recordBuffer = zlib.gzipSync(recordBuffer) + } + if (!this._warcOutStream.write(recordBuffer, 'utf8')) { this._warcOutStream.once('drain', resolve) } else {