feat: added debugging

README.md

@@ -63,6 +63,7 @@ const { getHistoricRates } = require('dukascopy-node');
 * [Downloading data with cache](https://github.com/Leo4815162342/dukascopy-node/wiki/Downloading-data-with-cache)
 * [Downloading data with custom batching](https://github.com/Leo4815162342/dukascopy-node/wiki/Downloading-data-with-custom-batching)
 * [Usage with typescript](https://github.com/Leo4815162342/dukascopy-node/wiki/Usage-with-typescript)
+* [Debugging](https://github.com/Leo4815162342/dukascopy-node/wiki/Debugging)
 
 
 ## Config object
@@ -165,6 +166,7 @@ Options:
 
 ***
 
+
 ## Instruments
 
 * [Bonds 📊 (3)](#bnd_cfd)

dev-utils/test-call.ts

@@ -4,24 +4,23 @@ import { getHistoricRates, Config } from '../src';
 (async () => {
   try {
     const config: Config = {
-      instrument: 'btcusd',
+      instrument: 'eurusd',
       dates: {
-        from: '2018-01-01',
-        to: '2019-01-01'
+        from: '2022-04-22',
+        to: '2022-04-23'
       },
-      timeframe: 'd1',
-      format: 'json',
-      useCache: false
+      timeframe: 'm1',
+      format: 'array',
+      useCache: false,
+      ignoreFlats: false
     };
-    console.log('fetching started');
-    console.log(JSON.stringify(config));
+    // console.log('fetching started');
+    // console.log(JSON.stringify(config));
     const d1 = +new Date();
     const quotes = await getHistoricRates(config);
     const d2 = +new Date();
-    console.log('fetching success');
+    // console.log('fetching success');
     // console.log(JSON.stringify(quotes));
-    console.log(`size: ${quotes.length}`);
-    console.log(JSON.stringify(quotes, null, 2));
     console.log(`time ms: ${d2 - d1}`);
   } catch (error) {
     console.log('error', error);

examples/with-debugging/README.md

@@ -0,0 +1,17 @@
+## With debugging
+
+The library ships with a built-in [`debug`](https://www.npmjs.com/package/debug) module.
+
+To enable debugging add `DEBUG=dukascopy-node:*` environment variable to your execution code.
+
+For CLI calls, add `-d` flag (or alternatively prepend `DEBUG=dukascopy-node:cli:*`):
+
+```bash
+dukascopy-cli -i usdjpy -from 2022-03-21 -to 2022-03-22 -t m1 -d
+```
+
+```bash
+DEBUG=dukascopy-node:cli:* dukascopy-cli -i usdjpy -from 2022-03-21 -to 2022-03-22 -t m1
+```
+
+![example of debugging output](https://github.com/Leo4815162342/dukascopy-node/blob/feat/debug_module/examples/with-debugging/with-debugging.png?raw=true)

package.json

@@ -45,6 +45,7 @@
 		"chalk": "3.0.0",
 		"cli-progress": "3.10.0",
 		"commander": "5.0.0",
+		"debug": "4.3.4",
 		"fastest-validator": "1.10.0",
 		"fs-extra": "10.1.0",
 		"lzma-purejs": "0.9.3",
@@ -53,6 +54,7 @@
 	},
 	"devDependencies": {
 		"@types/cli-progress": "3.9.2",
+		"@types/debug": "4.1.7",
 		"@types/fs-extra": "9.0.13",
 		"@types/node": "17.0.25",
 		"@types/node-fetch": "2.6.1",

src/buffer-fetcher/index.ts

@@ -1,7 +1,6 @@
 import fetch from 'node-fetch';
 import { CacheManagerBase } from '../cache-manager';
 import { splitArrayInChunks, wait } from '../utils/general';
-
 import { NotifyFn, BufferFetcherInput, BufferObject } from './types';
 
 export class BufferFetcher {
@@ -35,10 +34,12 @@ export class BufferFetcher {
     return await Promise.all(
       urls.map(async url => {
         let buffer: Buffer;
+        let isCacheHit = false;
 
         if (this.cacheManager) {
           const bufferFromCache = await this.cacheManager.readItemFromCache(url);
           if (bufferFromCache) {
+            isCacheHit = true;
             buffer = bufferFromCache;
           } else {
             buffer = await this.fetchBuffer(url);
@@ -47,7 +48,7 @@ export class BufferFetcher {
           buffer = await this.fetchBuffer(url);
         }
 
-        this.notifyOnItemFetchFn && this.notifyOnItemFetchFn(url);
+        this.notifyOnItemFetchFn && this.notifyOnItemFetchFn(url, buffer, isCacheHit);
 
         return { url, buffer };
       })

src/buffer-fetcher/types.ts

@@ -5,7 +5,7 @@ export interface BufferObject {
   buffer: Buffer;
 }
 
-export type NotifyFn = (downloadedUrl: string) => void;
+export type NotifyFn = (url: string, buffer: Buffer, isCacheHit: boolean) => void;
 
 export interface BufferFetcherInput {
   batchSize?: number;

src/cli/config.ts

@@ -8,7 +8,7 @@ import { Price } from '../config/price-types';
 import { Timeframe } from '../config/timeframes';
 
 program
-  .option('-d, --debug', 'Output extra debugging')
+  .option('-d, --debug', 'Output extra debugging', false)
   .option('-s, --silent', 'Hides the search config in the CLI output', false)
   .requiredOption('-i, --instrument <value>', 'Trading instrument')
   .requiredOption('-from, --date-from <value>', 'From date (yyyy-mm-dd)')
@@ -35,12 +35,10 @@ program.parse(process.argv);
 
 const options = program.opts();
 
-// eslint-disable-next-line no-console
-if (program.debug) console.log(options);
-
 export interface CliConfig extends Config {
   dir: string;
   silent: boolean;
+  debug: boolean;
 }
 
 const cliConfig: CliConfig = {
@@ -62,14 +60,16 @@ const cliConfig: CliConfig = {
   useCache: options.cache,
   cacheFolderPath: options.cachePath,
   retryCount: options.retries,
-  pauseBetweenRetriesMs: options.retryPause
+  pauseBetweenRetriesMs: options.retryPause,
+  debug: options.debug
 };
 
 const cliSchema: InputSchema<CliConfig> = {
   ...schema,
   ...{
     dir: { type: 'string', required: true } as RuleString,
-    silent: { type: 'boolean', required: false } as RuleBoolean
+    silent: { type: 'boolean', required: false } as RuleBoolean,
+    debug: { type: 'boolean', required: false } as RuleBoolean
   }
 };
 

src/cli/index.ts

@@ -12,10 +12,16 @@ import { BufferFetcher } from '../buffer-fetcher';
 import { CacheManager } from '../cache-manager';
 import { processData } from '../processor';
 import { formatOutput } from '../output-formatter';
+import { formatBytes } from '../utils/formatBytes';
 import chalk from 'chalk';
+import debug from 'debug';
+
 import { Output } from '../output-formatter/types';
+import { Timeframe } from '../config/timeframes';
+
+const DEBUG_NAMESPACE = 'dukascopy-node:cli';
 
-const {
+let {
   instrument,
   dates: { from: fromDate, to: toDate },
   timeframe,
@@ -29,9 +35,19 @@ const {
   useCache,
   cacheFolderPath,
   dir,
-  silent
+  silent,
+  debug: isDebugActive
 } = input;
 
+if (isDebugActive) {
+  debug.enable(`${DEBUG_NAMESPACE}:*`);
+} else {
+  if (process.env.DEBUG) {
+    isDebugActive = true;
+    debug.enable(process.env.DEBUG);
+  }
+}
+
 const fileName = `${instrument}-${timeframe}${
   timeframe === 'tick' ? '' : '-' + priceType
 }-${fromDate.toISOString().slice(0, 10)}-${toDate.toISOString().slice(0, 10)}.${
@@ -43,6 +59,12 @@ const filePath = resolve(folderPath, fileName);
 // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
 (async () => {
   try {
+    debug(`${DEBUG_NAMESPACE}:config`)('%O', {
+      input,
+      isValid,
+      validationErrors
+    });
+
     if (isValid) {
       const [startDate, endDate] = normaliseDates({
         instrument,
@@ -52,21 +74,35 @@ const filePath = resolve(folderPath, fileName);
         utcOffset
       });
 
-      silent ? printDivider() : printHeader(input, startDate, endDate);
+      if (!isDebugActive) {
+        silent ? printDivider() : printHeader(input, startDate, endDate);
+      }
 
       const urls = generateUrls({ instrument, timeframe, priceType, startDate, endDate });
 
+      debug(`${DEBUG_NAMESPACE}:urls`)(`Generated ${urls.length} urls`);
+      debug(`${DEBUG_NAMESPACE}:urls`)(`%O`, urls);
+
       let val = 0;
 
-      progressBar.start(urls.length, val);
+      if (!isDebugActive) {
+        progressBar.start(urls.length, val);
+      }
 
       const bufferFetcher = new BufferFetcher({
         batchSize,
         pauseBetweenBatchesMs,
         cacheManager: useCache ? new CacheManager({ cacheFolderPath }) : undefined,
-        notifyOnItemFetchFn: (): void => {
-          val += 1;
-          progressBar.update(val);
+        notifyOnItemFetchFn: (url, buffer, isCacheHit): void => {
+          debug(`${DEBUG_NAMESPACE}:fetcher`)(
+            url,
+            `| ${formatBytes(buffer.length)} |`,
+            `${isCacheHit ? 'cache' : 'network'}`
+          );
+          if (!isDebugActive) {
+            val += 1;
+            progressBar.update(val);
+          }
         }
       });
 
@@ -90,6 +126,12 @@ const filePath = resolve(folderPath, fileName);
           ([timestamp]) => timestamp && timestamp >= startDateMs && timestamp < endDateMs
         );
 
+        debug(`${DEBUG_NAMESPACE}:data`)(
+          `Generated ${filteredData.length} ${
+            timeframe === Timeframe.tick ? 'ticks' : 'OHLC candles'
+          }`
+        );
+
         const formatted = formatOutput({ processedData: filteredData, timeframe, format });
 
         savePayload = format === 'csv' ? formatted : JSON.stringify(formatted, null, 2);
@@ -101,7 +143,9 @@ const filePath = resolve(folderPath, fileName);
 
       await outputFile(filePath, savePayload);
 
-      progressBar.stop();
+      if (!isDebugActive) {
+        progressBar.stop();
+      }
 
       if (isEmpty) {
         printWarning(