[MLOB-1524] feat(llmobs): Introduce LLM Observability SDK (#4742)

* [MLOB-1540] add llmobs configuration to global tracer config (#4696)

add llmobs config

* [MLOB-1555] LLM Observability writers (#4699)

LLM Observability writers

* [MLOB-1556] LLM Observability tagger (#4718)

LLM Observability tagger

* [MLOB-1560] LLMObs Span Processor (#4738)

* span processor

* tests

* remove agent exporter log and do not stringify tags

* remove llmobs from exporter tests

* add in default unserializable value

* review comments

* warning log for metric

* todo-ify

* remove some duplicate logic

* decouple llmobs span processing with a channel

* use a static weakmap to store llmobs tags/annotations instead of span tags

* do not register span in map if it does not have an llmobs span kind

* span is passed on an object from sp publisher

* re-clarify TODOs

* only send span in publish

* log multiple warnings and return conditional undefined

* update error logic

* [MLOB-1561] LLM Observability SDK API (#4773)

* wip

* type definitions

* active + try/catch eval metric writer append

* test ts

* use tagger map and processor as a channel subscriber

* change decorate and add in dev changes

* try some api changes

* add decorate to noop

* fix breaking proxy tests

* experimental decorators for TS docs

* api changes, fix unit + e2e tests

* try removing global log mocks

* add some util tests

* remove logger mocks

* add module tests + do not enable when not specified

* fix eval metric integration test

* wip

* memoize getFunctionArguments

* move any subscriber and global writer to the module enablement level instead of sdk

* should fix TS tests

* add ts integration test and fix decorator

* devex for ts versions

* add noop typescript test

* remove startSpan

* remove unneeded change

* dedup decorator code

* Update index.d.ts

Co-authored-by: Yun Kim <35776586+Yun-Kim@users.noreply.github.com>

* map metrics names

* change validKind to validateKind and throw

* tagger for metrics follow-up

* review feedback

* add some tests for not auto-annotating in certain cases

---------

Co-authored-by: Yun Kim <35776586+Yun-Kim@users.noreply.github.com>

* hard fail instead of soft fail, except for `wrap` span name

* add ml-observability codeowners

* resolve ts test

* update auto-annotation check

* tagger can soft fail

* using custom ASL instance and scope activation

* fix test comments and remove

* address review comments

* remove llmobs.apiKey config, only rely on global

* fix evaulations test

* make llmobs storage accessible

---------

Co-authored-by: Yun Kim <35776586+Yun-Kim@users.noreply.github.com>

.github/workflows/llmobs.yml

@@ -0,0 +1,30 @@
+name: LLMObs
+
+on:
+  pull_request:
+  push:
+    branches: [master]
+  schedule:
+    - cron: '0 4 * * *'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  ubuntu:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/testagent/start
+      - uses: ./.github/actions/node/setup
+      - uses: ./.github/actions/install
+      - uses: ./.github/actions/node/18
+      - run: yarn test:llmobs:ci
+      - uses: ./.github/actions/node/20
+      - run: yarn test:llmobs:ci
+      - uses: ./.github/actions/node/latest
+      - run: yarn test:llmobs:ci
+      - if: always()
+        uses: ./.github/actions/testagent/logs
+      - uses: codecov/codecov-action@v3

CODEOWNERS

@@ -53,6 +53,9 @@
 /packages/dd-trace/src/service-naming/ @Datadog/apm-idm-js
 /packages/dd-trace/test/service-naming/ @Datadog/apm-idm-js
 
+/packages/dd-trace/src/llmobs/ @DataDog/ml-observability
+/packages/dd-trace/test/llmobs/ @DataDog/ml-observability
+
 # CI
 /.github/workflows/appsec.yml @DataDog/asm-js
 /.github/workflows/ci-visibility-performance.yml @DataDog/ci-app-libraries

docs/package.json

@@ -11,6 +11,6 @@
   "private": true,
   "devDependencies": {
     "typedoc": "^0.25.8",
-    "typescript": "^4.6"
+    "typescript": "^5.0"
   }
 }

docs/test.ts

@@ -536,3 +536,93 @@ const otelTraceId: string = spanContext.traceId
 const otelSpanId: string = spanContext.spanId
 const otelTraceFlags: number = spanContext.traceFlags
 const otelTraceState: opentelemetry.TraceState = spanContext.traceState!
+
+// -- LLM Observability --
+const llmobsEnableOptions = {
+  mlApp: 'mlApp',
+  agentlessEnabled: true
+}
+tracer.init({
+  llmobs: llmobsEnableOptions,
+})
+const llmobs = tracer.llmobs
+const enabled = llmobs.enabled
+
+// manually enable
+llmobs.enable({
+  mlApp: 'mlApp',
+  agentlessEnabled: true
+})
+
+// manually disable
+llmobs.disable()
+
+// trace block of code
+llmobs.trace({ name: 'name', kind: 'llm' }, () => {})
+llmobs.trace({ kind: 'llm', name: 'myLLM', modelName: 'myModel', modelProvider: 'myProvider' }, () => {})
+llmobs.trace({ name: 'name', kind: 'llm' }, (span, cb) => {
+  llmobs.annotate(span, {})
+  span.setTag('foo', 'bar')
+  cb(new Error('boom'))
+})
+
+// wrap a function
+llmobs.wrap({ kind: 'llm' }, function myLLM () {})()
+llmobs.wrap({ kind: 'llm', name: 'myLLM', modelName: 'myModel', modelProvider: 'myProvider' }, function myFunction () {})()
+
+// decorate a function
+class MyClass {
+  @llmobs.decorate({ kind: 'llm' })
+  myLLM () {}
+
+  @llmobs.decorate({ kind: 'llm', name: 'myOtherLLM', modelName: 'myModel', modelProvider: 'myProvider' })
+  myOtherLLM () {}
+}
+
+const cls = new MyClass()
+cls.myLLM()
+cls.myOtherLLM()
+
+// export a span
+llmobs.enable({ mlApp: 'myApp' })
+llmobs.trace({ kind: 'llm', name: 'myLLM' }, (span) => {
+  const llmobsSpanCtx = llmobs.exportSpan(span)
+  llmobsSpanCtx.traceId;
+  llmobsSpanCtx.spanId;
+
+  // submit evaluation
+  llmobs.disable()
+  llmobs.submitEvaluation(llmobsSpanCtx, {
+    label: 'my-eval-metric',
+    metricType: 'categorical',
+    value: 'good',
+    mlApp: 'myApp',
+    tags: {},
+    timestampMs: Date.now()
+  })
+})
+
+// annotate a span
+llmobs.annotate({
+  inputData: 'input',
+  outputData: 'output',
+  metadata: {},
+  metrics: {
+    inputTokens: 10,
+    outputTokens: 5,
+    totalTokens: 15
+  },
+  tags: {}
+})
+llmobs.annotate(span, {
+  inputData: 'input',
+  outputData: 'output',
+  metadata: {},
+  metrics: {},
+  tags: {}
+})
+
+
+
+// flush
+llmobs.flush()

docs/yarn.lock

@@ -61,10 +61,10 @@ typedoc@^0.25.8:
     minimatch "^9.0.3"
     shiki "^0.14.7"
 
-typescript@^4.6:
-  version "4.9.5"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.5.tgz#095979f9bcc0d09da324d58d03ce8f8374cbe65a"
-  integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==
+typescript@^5.0:
+  version "5.6.3"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.6.3.tgz#5f3449e31c9d94febb17de03cc081dd56d81db5b"
+  integrity sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==
 
 vscode-oniguruma@^1.7.0:
   version "1.7.0"