feat(tracing): allow opting out from tracing (#503)

* feat(tracing): allow opting out from tracing (disable exporters + block instrumentations)
* fix(tests): fix flaky tests

README.md

@@ -100,6 +100,7 @@ Specifically supported are:
 
 `@lumigo/opentelemetry` additionally supports the following configuration options as environment variables:
 
+* `LUMIGO_ENABLE_TRACES` - Default: `true`. When set to `false`, turns off all of the *tracing* instrumentations. Note that this does not turn off the *logging* instrumentations, which are controlled by the `LUMIGO_ENABLE_LOGS` environment variable.
 * `LUMIGO_TRACER_TOKEN=<token>`: Configure the Lumigo token to enable to upload of telemetry to Lumigo; without this environment variable, your Node.js process will not send telemetry to Lumigo.
 * `LUMIGO_DEBUG=TRUE`: Enables debug logging
 * `LUMIGO_DEBUG_SPANDUMP=<path|console:log|console:error>`: Log all spans collected to the `<path>` file or, the value is `console:log` or `console:error`, to `console.log` or `console.error`; this is an option intended only for debugging purposes and should *not* be used in production.

src/bootstrap.ts

@@ -17,6 +17,8 @@ import {
   DEFAULT_DEPENDENCIES_ENDPOINT,
   DEFAULT_LUMIGO_TRACES_ENDPOINT,
   DEFAULT_LUMIGO_LOGS_ENDPOINT,
+  TRACING_ENABLED,
+  LOGGING_ENABLED,
 } from './constants';
 import { report } from './dependencies';
 import { FileLogExporter, FileSpanExporter } from './exporters';
@@ -241,14 +243,20 @@ export const init = async (): Promise<LumigoSdkInitialization> => {
         },
       });
 
-      tracerProvider.addSpanProcessor(
-        new LumigoSpanProcessor(otlpTraceExporter, {
-          // The maximum queue size. After the size is reached spans are dropped.
-          maxQueueSize: 1000,
-          // The maximum batch size of every export. It must be smaller or equal to maxQueueSize.
-          maxExportBatchSize: 100,
-        })
-      );
+      if (TRACING_ENABLED) {
+        tracerProvider.addSpanProcessor(
+          new LumigoSpanProcessor(otlpTraceExporter, {
+            // The maximum queue size. After the size is reached spans are dropped.
+            maxQueueSize: 1000,
+            // The maximum batch size of every export. It must be smaller or equal to maxQueueSize.
+            maxExportBatchSize: 100,
+          })
+        );
+      } else {
+        logger.info(
+          'Tracing is disabled (the "LUMIGO_ENABLE_TRACES" environment variable is not set to "true"): no traces will be sent to Lumigo.'
+        );
+      }
 
       const otlpLogExporter = new OTLPLogExporter({
         url: lumigoLogEndpoint,
@@ -257,14 +265,20 @@ export const init = async (): Promise<LumigoSdkInitialization> => {
         },
       });
 
-      loggerProvider.addLogRecordProcessor(
-        new BatchLogRecordProcessor(otlpLogExporter, {
-          // The maximum queue size. After the size is reached logs are dropped.
-          maxQueueSize: 1000,
-          // The maximum batch size of every export. It must be smaller or equal to maxQueueSize.
-          maxExportBatchSize: 100,
-        })
-      );
+      if (LOGGING_ENABLED) {
+        loggerProvider.addLogRecordProcessor(
+          new BatchLogRecordProcessor(otlpLogExporter, {
+            // The maximum queue size. After the size is reached logs are dropped.
+            maxQueueSize: 1000,
+            // The maximum batch size of every export. It must be smaller or equal to maxQueueSize.
+            maxExportBatchSize: 100,
+          })
+        );
+      } else {
+        logger.info(
+          'Logging is disabled (the "LUMIGO_ENABLE_LOGS" environment variable is not set to "true"): no logs will be sent to Lumigo.'
+        );
+      }
 
       /*
        * We do not wait for this promise, we do not want to delay the application.

src/constants.ts

@@ -1,7 +1,17 @@
 export const LUMIGO_LOGGING_NAMESPACE = '@lumigo/opentelemetry';
+
 export const DEFAULT_LUMIGO_TRACES_ENDPOINT =
   'https://ga-otlp.lumigo-tracer-edge.golumigo.com/v1/traces';
+
 export const DEFAULT_LUMIGO_LOGS_ENDPOINT =
   'https://ga-otlp.lumigo-tracer-edge.golumigo.com/v1/logs';
+
 export const DEFAULT_DEPENDENCIES_ENDPOINT =
   'https://ga-otlp.lumigo-tracer-edge.golumigo.com/v1/dependencies';
+
+// Since tracing is on by default, we allow omitting it and consider it enabled
+export const TRACING_ENABLED =
+  process.env.LUMIGO_ENABLE_TRACES === undefined ||
+  process.env.LUMIGO_ENABLE_TRACES?.toLowerCase() === 'true';
+
+export const LOGGING_ENABLED = process.env.LUMIGO_ENABLE_LOGS?.toLowerCase() === 'true';

src/instrumentations/@grpc/grpc-js/GrpcInstrumentation.ts

@@ -1,9 +1,9 @@
 import { GrpcInstrumentation } from '@opentelemetry/instrumentation-grpc';
-import { Instrumentor } from '../../instrumentor';
+import { TracingInstrumentor } from '../../instrumentor';
 import { wrapServer } from './wrapGrpcServer';
 import { wrapClient } from './wrapGrpcClient';
 
-export default class LumigoGrpcInstrumentation extends Instrumentor<GrpcInstrumentation> {
+export default class LumigoGrpcInstrumentation extends TracingInstrumentor<GrpcInstrumentation> {
   getInstrumentedModule(): string {
     return '@grpc/grpc-js';
   }

src/instrumentations/@nestjs/core/NestInstrumentation.ts

@@ -1,7 +1,7 @@
 import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core';
-import { Instrumentor } from '../../instrumentor';
+import { TracingInstrumentor } from '../../instrumentor';
 
-export default class LumigoNestInstrumentation extends Instrumentor<NestInstrumentation> {
+export default class LumigoNestInstrumentation extends TracingInstrumentor<NestInstrumentation> {
   override isApplicable(): boolean {
     return (
       super.isApplicable() &&

src/instrumentations/amqplib/AmqplibInstrumentation.ts

@@ -6,9 +6,9 @@ import {
   PublishInfo,
 } from '@opentelemetry/instrumentation-amqplib';
 import { getSpanAttributeMaxLength } from '../../utils';
-import { Instrumentor } from '../instrumentor';
+import { TracingInstrumentor } from '../instrumentor';
 
-export default class LumigoAmqplibInstrumentation extends Instrumentor<AmqplibInstrumentation> {
+export default class LumigoAmqplibInstrumentation extends TracingInstrumentor<AmqplibInstrumentation> {
   getInstrumentedModule(): string {
     return 'amqplib';
   }

src/instrumentations/aws-sdk/LumigoAwsSdkLibInstrumentation.ts

@@ -1,8 +1,8 @@
 import { AwsInstrumentation } from '@opentelemetry/instrumentation-aws-sdk';
-import { Instrumentor } from '../instrumentor';
+import { TracingInstrumentor } from '../instrumentor';
 import { preRequestHook, responseHook, sqsProcessHook } from './hooks';
 
-export abstract class LumigoAwsSdkLibInstrumentation extends Instrumentor<AwsInstrumentation> {
+export abstract class LumigoAwsSdkLibInstrumentation extends TracingInstrumentor<AwsInstrumentation> {
   override isApplicable(): boolean {
     return (
       super.isApplicable() &&

src/instrumentations/bunyan/BunyanInstrumentation.ts

@@ -1,16 +1,12 @@
 import { BunyanInstrumentation } from '@opentelemetry/instrumentation-bunyan';
-import { Instrumentor } from '../instrumentor';
+import { LoggingInstrumentor } from '../instrumentor';
 
-export default class LumigoBunyanInstrumentation extends Instrumentor<BunyanInstrumentation> {
+export default class LumigoBunyanInstrumentation extends LoggingInstrumentor<BunyanInstrumentation> {
   getInstrumentedModule(): string {
     return 'bunyan';
   }
 
   getInstrumentation(): BunyanInstrumentation {
     return new BunyanInstrumentation();
   }
-
-  override isApplicable(): boolean {
-    return super.isApplicable() && process.env.LUMIGO_ENABLE_LOGS?.toLowerCase() === 'true';
-  }
 }

src/instrumentations/express/ExpressInstrumentation.ts

@@ -1,8 +1,8 @@
 import { ExpressHooks } from './express';
 import { ExpressInstrumentation } from 'opentelemetry-instrumentation-express';
-import { Instrumentor } from '../instrumentor';
+import { TracingInstrumentor } from '../instrumentor';
 
-export default class LumigoExpressInstrumentation extends Instrumentor<ExpressInstrumentation> {
+export default class LumigoExpressInstrumentation extends TracingInstrumentor<ExpressInstrumentation> {
   getInstrumentedModule(): string {
     return 'express';
   }

src/instrumentations/fastify/FastifyInstrumentation.ts

@@ -3,9 +3,9 @@ import type { Span } from '@opentelemetry/api';
 import { FastifyInstrumentation, FastifyRequestInfo } from '@opentelemetry/instrumentation-fastify';
 import { contentType, scrubHttpPayload } from '../../tools/payloads';
 import { getSpanAttributeMaxLength } from '../../utils';
-import { Instrumentor } from '../instrumentor';
+import { TracingInstrumentor } from '../instrumentor';
 
-export default class LumigoFastifyInstrumentation extends Instrumentor<FastifyInstrumentation> {
+export default class LumigoFastifyInstrumentation extends TracingInstrumentor<FastifyInstrumentation> {
   getInstrumentedModule(): string {
     return 'fastify';
   }

src/instrumentations/https/HttpInstrumentation.ts

@@ -2,9 +2,9 @@ import type { RequestOptions } from 'https';
 import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
 
 import { HttpHooks } from './http';
-import { Instrumentor } from '../instrumentor';
+import { TracingInstrumentor } from '../instrumentor';
 
-export default class LumigoHttpInstrumentation extends Instrumentor<HttpInstrumentation> {
+export default class LumigoHttpInstrumentation extends TracingInstrumentor<HttpInstrumentation> {
   private readonly ignoredHostnames: string[];
 
   constructor(...ignoredHostnames: string[]) {

src/instrumentations/instrumentor.ts

@@ -1,7 +1,8 @@
 import type { Instrumentation } from '@opentelemetry/instrumentation';
 import { canRequireModule } from '../utils';
+import { LOGGING_ENABLED, TRACING_ENABLED } from '../constants';
 
-export abstract class Instrumentor<T extends Instrumentation> {
+abstract class Instrumentor<T extends Instrumentation> {
   abstract getInstrumentedModule(): string;
 
   abstract getInstrumentation(options?): T;
@@ -10,3 +11,15 @@ export abstract class Instrumentor<T extends Instrumentation> {
     return canRequireModule(this.getInstrumentedModule());
   }
 }
+
+export abstract class LoggingInstrumentor<T extends Instrumentation> extends Instrumentor<T> {
+  override isApplicable() {
+    return LOGGING_ENABLED && super.isApplicable();
+  }
+}
+
+export abstract class TracingInstrumentor<T extends Instrumentation> extends Instrumentor<T> {
+  override isApplicable() {
+    return TRACING_ENABLED && super.isApplicable();
+  }
+}

src/instrumentations/ioredis/IORedisInstrumentation.ts

@@ -2,9 +2,9 @@ import { CommonUtils, ScrubContext } from '@lumigo/node-core';
 import type { Span } from '@opentelemetry/api';
 import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';
 import { getSpanAttributeMaxLength } from '../../utils';
-import { Instrumentor } from '../instrumentor';
+import { TracingInstrumentor } from '../instrumentor';
 
-export default class LumigoIORedisInstrumentation extends Instrumentor<IORedisInstrumentation> {
+export default class LumigoIORedisInstrumentation extends TracingInstrumentor<IORedisInstrumentation> {
   getInstrumentedModule(): string {
     return 'ioredis';
   }

src/instrumentations/kafkajs/KafkaJsInstrumentation.ts

@@ -3,9 +3,9 @@ import type { Span } from '@opentelemetry/api';
 import type { Message } from 'kafkajs';
 import { KafkaJsInstrumentation } from 'opentelemetry-instrumentation-kafkajs';
 import { getSpanAttributeMaxLength } from '../../utils';
-import { Instrumentor } from '../instrumentor';
+import { TracingInstrumentor } from '../instrumentor';
 
-export default class LumigoKafkaJsInstrumentation extends Instrumentor<KafkaJsInstrumentation> {
+export default class LumigoKafkaJsInstrumentation extends TracingInstrumentor<KafkaJsInstrumentation> {
   getInstrumentedModule(): string {
     return 'kafkajs';
   }

src/instrumentations/mongodb/MongoDBInstrumentation.ts

@@ -1,7 +1,7 @@
 import { MongoDBInstrumentation } from '@opentelemetry/instrumentation-mongodb';
-import { Instrumentor } from '../instrumentor';
+import { TracingInstrumentor } from '../instrumentor';
 
-export default class LumigoMongoDBInstrumentation extends Instrumentor<MongoDBInstrumentation> {
+export default class LumigoMongoDBInstrumentation extends TracingInstrumentor<MongoDBInstrumentation> {
   getInstrumentedModule(): string {
     return 'mongodb';
   }

src/instrumentations/pg/PgInstrumentation.ts

@@ -1,7 +1,7 @@
 import { PgInstrumentation } from '@opentelemetry/instrumentation-pg';
-import { Instrumentor } from '../instrumentor';
+import { TracingInstrumentor } from '../instrumentor';
 
-export default class LumigoPgInstrumentation extends Instrumentor<PgInstrumentation> {
+export default class LumigoPgInstrumentation extends TracingInstrumentor<PgInstrumentation> {
   override isApplicable(): boolean {
     return (
       super.isApplicable() &&

src/instrumentations/prisma/PrismaInstrumentation.ts

@@ -1,7 +1,7 @@
 import { PrismaInstrumentation } from '@prisma/instrumentation';
-import { Instrumentor } from '../instrumentor';
+import { TracingInstrumentor } from '../instrumentor';
 
-export default class LumigoPrismaInstrumentation extends Instrumentor<PrismaInstrumentation> {
+export default class LumigoPrismaInstrumentation extends TracingInstrumentor<PrismaInstrumentation> {
   getInstrumentedModule(): string {
     return 'prisma';
   }

src/instrumentations/redis/RedisInstrumentation.ts

@@ -2,9 +2,9 @@ import { CommonUtils, ScrubContext } from '@lumigo/node-core';
 import type { Span } from '@opentelemetry/api';
 import { RedisInstrumentation } from '@opentelemetry/instrumentation-redis-4';
 import { getSpanAttributeMaxLength } from '../../utils';
-import { Instrumentor } from '../instrumentor';
+import { TracingInstrumentor } from '../instrumentor';
 
-export default class LumigoRedisInstrumentation extends Instrumentor<RedisInstrumentation> {
+export default class LumigoRedisInstrumentation extends TracingInstrumentor<RedisInstrumentation> {
   getInstrumentedModule(): string {
     return 'redis';
   }

src/instrumentations/winston/WinstonInstrumentation.ts

@@ -1,16 +1,12 @@
 import { WinstonInstrumentation } from '@opentelemetry/instrumentation-winston';
-import { Instrumentor } from '../instrumentor';
+import { LoggingInstrumentor } from '../instrumentor';
 
-export default class LumigoWinstonInstrumentation extends Instrumentor<WinstonInstrumentation> {
+export default class LumigoWinstonInstrumentation extends LoggingInstrumentor<WinstonInstrumentation> {
   getInstrumentedModule(): string {
     return 'winston';
   }
 
   getInstrumentation(): WinstonInstrumentation {
     return new WinstonInstrumentation();
   }
-
-  override isApplicable(): boolean {
-    return super.isApplicable() && process.env.LUMIGO_ENABLE_LOGS?.toLowerCase() === 'true';
-  }
 }

test/instrumentations/bunyan/bunyan.test.ts

@@ -70,11 +70,8 @@ describe.each(versionsToTest(INSTRUMENTATION_NAME, INSTRUMENTATION_NAME))(
         await writeLogLine('Hello Bunyan!');
         await writeLogLine({ a: 1, sekret: 'this is secret!' });
 
-        await fakeEdge.waitFor(
-          () => fakeEdge.resources.length == 1,
-          'waiting for resources to be processed'
-        );
-        await fakeEdge.waitFor(() => fakeEdge.logs.length == 2, 'waiting for logs to be processed');
+        await fakeEdge.waitFor(({ resources }) => resources.length > 1, 'waiting for resources to be processed');
+        await fakeEdge.waitFor(({ logs } ) => logs.length == 2, 'waiting for logs to be processed');
 
         expect(fakeEdge.resources[0].attributes).toIncludeAllMembers([
           {
@@ -103,28 +100,6 @@ describe.each(versionsToTest(INSTRUMENTATION_NAME, INSTRUMENTATION_NAME))(
       }
     );
 
-    itTest(
-      {
-        testName: `${INSTRUMENTATION_NAME} logger: ${versionToTest} - logging off`,
-        packageName: INSTRUMENTATION_NAME,
-        version: versionToTest,
-        timeout: 20_000,
-      },
-      async function () {
-        testApp = new TestApp(TEST_APP_DIR, INSTRUMENTATION_NAME, {
-          logDumpPath: `${LOGS_DIR}/${INSTRUMENTATION_NAME}.${INSTRUMENTATION_NAME}-logs@${versionToTest}.json`,
-          env: {
-            LUMIGO_ENABLE_LOGS: 'false',
-          },
-        });
-
-        await writeLogLine('Hello Bunyan!');
-
-        // We expect no logs to be sent, therefore waiting for 1 log should fail
-        await expect(testApp.getFinalLogs(1)).rejects.toThrow();
-      }
-    );
-
     const writeLogLine = async (logLine: any) =>
       testApp.invokeGetPath(
         `/write-log-line?logLine=${encodeURIComponent(JSON.stringify(logLine))}`

test/instrumentations/features/app/.gitignore

@@ -0,0 +1 @@
+package-lock.json

test/instrumentations/features/app/app.js

@@ -0,0 +1,31 @@
+const lumigo = require('@lumigo/opentelemetry');
+const fastify = require('fastify')({ logger: true });
+const bunyan = require('bunyan');
+const bunyanLogger = bunyan.createLogger({ name: __filename })
+
+let tracerProvider;
+let loggerProvider;
+
+fastify.get('/', async (request, reply) => reply.send('server is ready'));
+
+fastify.get('/quit', async (request, reply) => {
+  bunyanLogger.info('this should not be exported to Lumigo');
+
+  console.log('Received quit command, flushing and exiting');
+  await tracerProvider.forceFlush();
+  await loggerProvider.forceFlush();
+
+  // we could have used fastify.close(), but it just takes too long
+  reply.send({}).then(() => process.exit(0))
+});
+
+fastify.listen({ port: 0 }, async (err, address) => {
+  if (err) {
+    throw err;
+  }
+  const lumigoSdk = await lumigo.init
+  tracerProvider = lumigoSdk.tracerProvider;
+  loggerProvider = lumigoSdk.loggerProvider;
+  const port = fastify.server.address().port;
+  console.error(`HTTP server listening on port ${port}`);
+});

test/instrumentations/features/app/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "lumigo-features-test",
+  "version": "1.0.0",
+  "description": "",
+  "scripts": {
+    "start": "node -r @lumigo/opentelemetry app.js"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@lumigo/opentelemetry": "file:../../../../distro.tgz",
+    "fastify": "4.0.1",
+    "bunyan": "1.8.15"
+  }
+}