feat: recording verification rule

forge.config.ts

@@ -15,6 +15,7 @@ const config: ForgeConfig = {
     extraResource: [
       './resources/json_output.py',
       './resources/group_snippet.js',
+      './resources/checks_snippet.js',
       './resources/' + getPlatform() + '/' + getArch(),
     ],
     osxSign: {

resources/checks_snippet.js

@@ -0,0 +1,14 @@
+export function handleSummary(data) {
+  const checks = []
+  data.root_group.checks.forEach((check) => {
+    checks.push(check)
+  })
+  data.root_group.groups.forEach((group) => {
+    group.checks.forEach((check) => {
+      checks.push(check)
+    })
+  })
+  return {
+    stdout: JSON.stringify(checks),
+  }
+}

src/codegen/codegen.test.ts

@@ -9,6 +9,7 @@ import { CorrelationStateMap, TestRule } from '@/types/rules'
 import { generateSequentialInt } from '@/rules/utils'
 import { ProxyData } from '@/types'
 import { correlationRecording } from '@/test/fixtures/correlationRecording'
+import { checksRecording } from '@/test/fixtures/checksRecording'
 import { ThinkTime } from '@/types/testOptions'
 
 describe('Code generation', () => {
@@ -25,7 +26,7 @@ describe('Code generation', () => {
   describe('generateScript', () => {
     it('should generate script', () => {
       const expectedResult = `
-      import { group, sleep } from 'k6'
+      import { group, sleep, check } from 'k6'
       import http from 'k6/http'
 
       export const options = {}
@@ -198,6 +199,42 @@ describe('Code generation', () => {
         ).replace(/\s/g, '')
       ).toBe(expectedResult.replace(/\s/g, ''))
     })
+
+    it('should generate checks', () => {
+      const rules: TestRule[] = [
+        {
+          type: 'recording-verification',
+          id: '1',
+        },
+      ]
+      const correlationStateMap: CorrelationStateMap = {}
+      const sequentialIdGenerator = generateSequentialInt()
+      const thinkTime: ThinkTime = {
+        sleepType: 'iterations',
+        timing: {
+          type: 'fixed',
+          value: 1,
+        },
+      }
+
+      const expectedResult = `
+        params = { headers: {}, cookies: {} }
+        url = http.url\`http://test.k6.io/api/v1/foo\`
+        resp = http.request('POST', url, null, params)
+        check(resp,{'RecordingVerificationRule:statusmatchesrecording':(r)=>r.status===200,})
+
+      `
+
+      expect(
+        generateRequestSnippets(
+          checksRecording,
+          rules,
+          correlationStateMap,
+          sequentialIdGenerator,
+          thinkTime
+        ).replace(/\s/g, '')
+      ).toBe(expectedResult.replace(/\s/g, ''))
+    })
   })
 
   describe('generateGroupSnippet', () => {

src/constants/imports.ts

@@ -126,7 +126,10 @@ export const ALL_EXPORTS = {
 export const REQUIRED_IMPORTS: ImportModule[] = [
   {
     path: 'k6',
-    imports: { type: 'named', imports: [{ name: 'group' }, { name: 'sleep' }] },
+    imports: {
+      type: 'named',
+      imports: [{ name: 'group' }, { name: 'sleep' }, { name: 'check' }],
+    },
   },
   { path: 'k6/http', default: { name: 'http' } },
 ]

src/proxy.ts

@@ -59,9 +59,10 @@ export const launchProxy = (
   const stdoutReader = readline.createInterface(proxy.stdout)
 
   stdoutReader.on('line', (data) => {
-    console.log(`stdout: ${data}`)
+    // console.log(`stdout: ${data}`)
 
     if (data === 'Proxy Started~') {
+      console.log(data)
       onReady?.()
       return
     }

src/rules/rules.ts

@@ -4,6 +4,7 @@ import { CorrelationStateMap, TestRule } from '@/types/rules'
 import { exhaustive } from '../utils/typescript'
 import { applyCustomCodeRule } from './customCode'
 import { applyCorrelationRule } from './correlation'
+import { applyRecordingVerificationRule } from './verification'
 
 export function applyRule(
   requestSnippetSchema: RequestSnippetSchema,
@@ -24,6 +25,8 @@ export function applyRule(
     case 'parameterization':
     case 'verification':
       return requestSnippetSchema
+    case 'recording-verification':
+      return applyRecordingVerificationRule(requestSnippetSchema)
     default:
       return exhaustive(rule)
   }

src/rules/utils.ts

@@ -44,6 +44,9 @@ export function matchFilter(
         const { filter } = rule
         return new RegExp(filter.path).test(request.url)
       }
+      case 'recording-verification':
+        // NOTE: no filtering yet on recording verification
+        return true
       default:
         return exhaustive(rule)
     }

src/rules/verification.ts

@@ -0,0 +1,21 @@
+import { RequestSnippetSchema } from '@/types'
+
+export function applyRecordingVerificationRule(
+  requestSnippetSchema: RequestSnippetSchema
+): RequestSnippetSchema {
+  const response = requestSnippetSchema.data.response
+
+  if (!response) {
+    return requestSnippetSchema
+  }
+
+  const verificationSnippet = `
+check(resp, {
+    'Recording Verification Rule: status matches recording': (r) => r.status === ${response.statusCode},
+  })
+`
+  return {
+    ...requestSnippetSchema,
+    after: [...requestSnippetSchema['after'], verificationSnippet],
+  }
+}

src/schemas/rules.ts

@@ -49,12 +49,23 @@ export const CustomCodeSelectorSchema = z.object({
   snippet: z.string(),
 })
 
+export const StatusCodeSelectorSchema = z.object({
+  type: z.literal('status-code'),
+})
+
 export const SelectorSchema = z.discriminatedUnion('type', [
   BeginEndSelectorSchema,
   RegexSelectorSchema,
   JsonSelectorSchema,
 ])
 
+export const VerificationRuleSelectorSchema = z.discriminatedUnion('type', [
+  BeginEndSelectorSchema,
+  RegexSelectorSchema,
+  JsonSelectorSchema,
+  StatusCodeSelectorSchema,
+])
+
 export const CorrelationExtractorSchema = z.object({
   filter: FilterSchema,
   selector: SelectorSchema,
@@ -90,7 +101,7 @@ export const CorrelationRuleSchema = RuleBaseSchema.extend({
 export const VerificationRuleSchema = RuleBaseSchema.extend({
   type: z.literal('verification'),
   filter: FilterSchema,
-  selector: SelectorSchema,
+  selector: VerificationRuleSelectorSchema,
   value: z.discriminatedUnion('type', [
     VariableValueSchema,
     ArrayValueSchema,
@@ -99,6 +110,10 @@ export const VerificationRuleSchema = RuleBaseSchema.extend({
   ]),
 })
 
+export const RecordingVerificationRuleSchema = RuleBaseSchema.extend({
+  type: z.literal('recording-verification'),
+})
+
 export const CustomCodeRuleSchema = RuleBaseSchema.extend({
   type: z.literal('customCode'),
   filter: FilterSchema,
@@ -111,4 +126,5 @@ export const TestRuleSchema = z.discriminatedUnion('type', [
   CorrelationRuleSchema,
   VerificationRuleSchema,
   CustomCodeRuleSchema,
+  RecordingVerificationRuleSchema,
 ])

src/script.ts

@@ -3,7 +3,7 @@ import { spawn, ChildProcessWithoutNullStreams } from 'node:child_process'
 import { readFile, writeFile } from 'fs/promises'
 import path from 'path'
 import readline from 'readline/promises'
-import { K6Log } from './types'
+import { K6Check, K6Log } from './types'
 import { getArch, getPlatform } from './utils/electron'
 
 export type K6Process = ChildProcessWithoutNullStreams
@@ -65,7 +65,6 @@ export const runScript = async (
       '--iterations=1',
       '--insecure-skip-tls-verify',
       '--log-format=json',
-      '--no-summary',
       '--quiet',
     ],
     {
@@ -75,14 +74,16 @@ export const runScript = async (
 
   // we use a reader to read entire lines from stderr instead of buffered data
   const stderrReader = readline.createInterface(k6.stderr)
+  const stdoutReader = readline.createInterface(k6.stdout)
 
-  k6.stdout.on('data', (data) => {
-    console.error(`stdout: ${data}`)
+  stdoutReader.on('line', (data) => {
+    console.log(`stdout: ${data}`)
+
+    const checkData: K6Check[] = JSON.parse(data)
+    browserWindow.webContents.send('script:check', checkData)
   })
 
   stderrReader.on('line', (data) => {
-    console.log(`stderr: ${data}`)
-
     const logData: K6Log = JSON.parse(data)
     browserWindow.webContents.send('script:log', logData)
   })
@@ -96,12 +97,25 @@ export const runScript = async (
 }
 
 const enhanceScript = async (scriptPath: string) => {
-  const groupSnippet = await getGroupSnippet()
+  const groupSnippet = await getJsSnippet('group_snippet.js')
+  const checksSnippet = await getJsSnippet('checks_snippet.js')
   const scriptContent = await readFile(scriptPath, { encoding: 'utf-8' })
   const scriptLines = scriptContent.split('\n')
   const httpImportIndex = scriptLines.findIndex((line) =>
     line.includes('k6/http')
   )
+  const handleSummaryIndex = scriptLines.findIndex(
+    (line) =>
+      // NOTE: if the custom handle summary is commented out we can still insert our snippet
+      // this check should be improved
+      line.includes('export function handleSummary(') && !line.includes('//')
+  )
+
+  // NOTE: checks works only if the user doesn't define a custom summary handler
+  // if no custom handleSummary is defined we add our version to retrieve checks
+  if (handleSummaryIndex === -1) {
+    scriptLines.push(checksSnippet)
+  }
 
   if (httpImportIndex !== -1) {
     scriptLines.splice(httpImportIndex + 1, 0, groupSnippet)
@@ -112,19 +126,15 @@ const enhanceScript = async (scriptPath: string) => {
   }
 }
 
-const getGroupSnippet = async () => {
-  let groupSnippetPath: string
+const getJsSnippet = async (snippetName: string) => {
+  let jsSnippetPath: string
 
   // if we are in dev server we take resources directly, otherwise look in the app resources folder.
   if (MAIN_WINDOW_VITE_DEV_SERVER_URL) {
-    groupSnippetPath = path.join(
-      app.getAppPath(),
-      'resources',
-      'group_snippet.js'
-    )
+    jsSnippetPath = path.join(app.getAppPath(), 'resources', snippetName)
   } else {
-    groupSnippetPath = path.join(process.resourcesPath, 'group_snippet.js')
+    jsSnippetPath = path.join(process.resourcesPath, snippetName)
   }
 
-  return readFile(groupSnippetPath, { encoding: 'utf-8' })
+  return readFile(jsSnippetPath, { encoding: 'utf-8' })
 }

src/test/fixtures/checksRecording.ts

@@ -0,0 +1,33 @@
+import { ProxyData } from '@/types'
+
+export const checksRecording: ProxyData[] = [
+  {
+    id: '1',
+    request: {
+      method: 'POST',
+      url: 'http://test.k6.io/api/v1/foo',
+      headers: [],
+      cookies: [],
+      query: [],
+      scheme: 'http',
+      host: 'localhost:3000',
+      content: '',
+      path: '/api/v1/foo',
+      timestampStart: 0,
+      timestampEnd: 0,
+      contentLength: 0,
+      httpVersion: '1.1',
+    },
+    response: {
+      statusCode: 200,
+      path: '/api/v1/foo',
+      reason: 'OK',
+      httpVersion: '1.1',
+      headers: [['Content-Type', 'application/json']],
+      cookies: [],
+      content: JSON.stringify({ user_id: '444' }),
+      contentLength: 0,
+      timestampStart: 0,
+    },
+  },
+]

src/types/index.ts

@@ -64,6 +64,14 @@ export interface K6Log {
   error?: string
 }
 
+export interface K6Check {
+  id: string
+  name: string
+  path: string
+  passes: number
+  fails: number
+}
+
 export type GroupedProxyData = Record<string, ProxyData[]>
 
 export interface RequestSnippetSchema {

src/types/rules.ts

@@ -13,12 +13,15 @@ import {
   JsonSelectorSchema,
   ParameterizationRuleSchema,
   RecordedValueSchema,
+  RecordingVerificationRuleSchema,
   RegexSelectorSchema,
   RuleBaseSchema,
   SelectorSchema,
+  StatusCodeSelectorSchema,
   TestRuleSchema,
   VariableValueSchema,
   VerificationRuleSchema,
+  VerificationRuleSelectorSchema,
 } from '@/schemas/rules'
 
 interface CorrelationState {
@@ -39,13 +42,20 @@ export type Filter = z.infer<typeof FilterSchema>
 export type BeginEndSelector = z.infer<typeof BeginEndSelectorSchema>
 export type RegexSelector = z.infer<typeof RegexSelectorSchema>
 export type JsonSelector = z.infer<typeof JsonSelectorSchema>
+export type StatusCodeSelector = z.infer<typeof StatusCodeSelectorSchema>
 export type CustomCodeSelector = z.infer<typeof CustomCodeSelectorSchema>
 export type Selector = z.infer<typeof SelectorSchema>
+export type VerificationRuleSelector = z.infer<
+  typeof VerificationRuleSelectorSchema
+>
 export type CorrelationExtractor = z.infer<typeof CorrelationExtractorSchema>
 export type CorrelationReplacer = z.infer<typeof CorrelationReplacerSchema>
 export type RuleBase = z.infer<typeof RuleBaseSchema>
 export type ParameterizationRule = z.infer<typeof ParameterizationRuleSchema>
 export type CorrelationRule = z.infer<typeof CorrelationRuleSchema>
 export type VerificationRule = z.infer<typeof VerificationRuleSchema>
 export type CustomCodeRule = z.infer<typeof CustomCodeRuleSchema>
+export type RecordingVerificationRule = z.infer<
+  typeof RecordingVerificationRuleSchema
+>
 export type TestRule = z.infer<typeof TestRuleSchema>

src/utils/generator.ts

@@ -1,5 +1,6 @@
 import { GeneratorFileData } from '@/types/generator'
 import { RampingStage } from '@/types/testOptions'
+import { createEmptyRule } from './rules'
 
 export function createNewGeneratorFile(recordingPath = ''): GeneratorFileData {
   return {
@@ -21,7 +22,7 @@ export function createNewGeneratorFile(recordingPath = ''): GeneratorFileData {
     testData: {
       variables: [],
     },
-    rules: [],
+    rules: [createEmptyRule('recording-verification')],
     allowlist: [],
     includeStaticAssets: false,
   }