fix: report studio usage to Dashboard (#16318)

packages/server/__snapshots__/3_studio_spec.ts.js

@@ -246,7 +246,7 @@ describe('creates new test', () => {
       saveStudio('My New Test')
     })
 
-    /* === Test Created with Cypress Studio === */
+    /* ==== Test Created with Cypress Studio ==== */
     it('My New Test', function() {
       /* ==== Generated with Cypress Studio ==== */
       cy.visit('new.html');

packages/server/__snapshots__/7_record_spec.js

@@ -1624,6 +1624,10 @@ exports['e2e record passing passes 2'] = [
       "start": "2018-02-01T20:14:19.323Z",
       "end": "2018-02-01T20:14:19.323Z",
       "duration": 1234
+    },
+    "metadata": {
+      "studioCreated": 0,
+      "studioExtended": 0
     }
   },
   {
@@ -1700,6 +1704,10 @@ exports['e2e record passing passes 2'] = [
       "start": "2018-02-01T20:14:19.323Z",
       "end": "2018-02-01T20:14:19.323Z",
       "duration": 1234
+    },
+    "metadata": {
+      "studioCreated": 0,
+      "studioExtended": 0
     }
   },
   {
@@ -1773,6 +1781,10 @@ exports['e2e record passing passes 2'] = [
       "start": "2018-02-01T20:14:19.323Z",
       "end": "2018-02-01T20:14:19.323Z",
       "duration": 1234
+    },
+    "metadata": {
+      "studioCreated": 0,
+      "studioExtended": 0
     }
   }
 ]
@@ -2647,6 +2659,78 @@ Available browsers found on your system are:
 - browser3
 `
 
+exports['e2e record metadata sends Studio usage metadata 1'] = `
+
+====================================================================================================
+
+  (Run Starting)
+
+  ┌────────────────────────────────────────────────────────────────────────────────────────────────┐
+  │ Cypress:    1.2.3                                                                              │
+  │ Browser:    FooBrowser 88                                                                      │
+  │ Specs:      1 found (studio_written.spec.js)                                                   │
+  │ Searched:   cypress/integration/studio_written.spec.js                                         │
+  │ Params:     Tag: false, Group: false, Parallel: false                                          │
+  │ Run URL:    https://dashboard.cypress.io/projects/cjvoj7/runs/12                               │
+  └────────────────────────────────────────────────────────────────────────────────────────────────┘
+
+
+────────────────────────────────────────────────────────────────────────────────────────────────────
+                                                                                                    
+  Running:  studio_written.spec.js                                                          (1 of 1)
+  Estimated: 8 seconds
+
+
+  suite
+    ✓ extended test
+    ✓ created test
+    nested suite
+      ✓ extended test
+      ✓ created test
+
+
+  4 passing
+
+
+  (Results)
+
+  ┌────────────────────────────────────────────────────────────────────────────────────────────────┐
+  │ Tests:        4                                                                                │
+  │ Passing:      4                                                                                │
+  │ Failing:      0                                                                                │
+  │ Pending:      0                                                                                │
+  │ Skipped:      0                                                                                │
+  │ Screenshots:  0                                                                                │
+  │ Video:        true                                                                             │
+  │ Duration:     X seconds                                                                        │
+  │ Estimated:    8 seconds                                                                        │
+  │ Spec Ran:     studio_written.spec.js                                                           │
+  └────────────────────────────────────────────────────────────────────────────────────────────────┘
+
+
+  (Uploading Results)
+
+  - Nothing to Upload
+
+====================================================================================================
+
+  (Run Finished)
+
+
+       Spec                                              Tests  Passing  Failing  Pending  Skipped  
+  ┌────────────────────────────────────────────────────────────────────────────────────────────────┐
+  │ ✔  studio_written.spec.js                   XX:XX        4        4        -        -        - │
+  └────────────────────────────────────────────────────────────────────────────────────────────────┘
+    ✔  All specs passed!                        XX:XX        4        4        -        -        -  
+
+
+───────────────────────────────────────────────────────────────────────────────────────────────────────
+                                                                                                       
+  Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12                                   
+
+
+`
+
 exports['e2e record quiet mode respects quiet mode 1'] = `
 
 

packages/server/__snapshots__/spec_writer_spec.ts.js

@@ -22,7 +22,7 @@ cy.get(".btn").click();
 `
 
 exports['lib/util/spec_writer #generateTest creates a new test with body 1'] = `
-/* === Test Created with Cypress Studio === */
+/* ==== Test Created with Cypress Studio ==== */
 it("my new test", function() {
     /* ==== Generated with Cypress Studio ==== */
     cy.get(".input").type("typed text");
@@ -133,7 +133,7 @@ describe('top level suite', () => {
       cy.get('.btn').click()
     })
 
-    /* === Test Created with Cypress Studio === */
+    /* ==== Test Created with Cypress Studio ==== */
     it('test added to describe', function() {
       /* ==== Generated with Cypress Studio ==== */
       cy.get('.input').type('typed text');
@@ -180,7 +180,7 @@ describe('top level suite', () => {
   })
 
   context('inner suite with context', () => {
-    /* === Test Created with Cypress Studio === */
+    /* ==== Test Created with Cypress Studio ==== */
     it('test added to context', function() {
       /* ==== Generated with Cypress Studio ==== */
       cy.get('.input').type('typed text');
@@ -269,7 +269,7 @@ describe('top level suite', () => {
 
   // eslint-disable-next-line mocha/no-exclusive-tests
   describe.only('inner suite with describe.only', () => {
-    /* === Test Created with Cypress Studio === */
+    /* ==== Test Created with Cypress Studio ==== */
     it('test added to describe only', function() {
       /* ==== Generated with Cypress Studio ==== */
       cy.get('.input').type('typed text');
@@ -357,7 +357,7 @@ describe('top level suite', () => {
   })
 
   describe('suite with config', { responseTimeout: 60000 }, () => {
-    /* === Test Created with Cypress Studio === */
+    /* ==== Test Created with Cypress Studio ==== */
     it('test added to describe with config', function() {
       /* ==== Generated with Cypress Studio ==== */
       cy.get('.input').type('typed text');
@@ -404,7 +404,7 @@ describe('top level suite', () => {
   })
 })
 
-/* === Test Created with Cypress Studio === */
+/* ==== Test Created with Cypress Studio ==== */
 it('test added to file', function() {
   /* ==== Generated with Cypress Studio ==== */
   cy.get('.input').type('typed text');
@@ -421,7 +421,7 @@ exports['lib/util/spec_writer #createNewTestInFile preserves comments in a compl
 that should be accurately
 preserved in the output
  */
-/* === Test Created with Cypress Studio === */
+/* ==== Test Created with Cypress Studio ==== */
 it('test added to empty file', function() {
  /* ==== Generated with Cypress Studio ==== */
  cy.get('.input').type('typed text');

packages/server/lib/api.js

@@ -366,6 +366,7 @@ module.exports = {
           'video',
           'screenshots',
           'reporterStats',
+          'metadata',
         ]),
       })
       .catch(RequestErrors.StatusCodeError, formatResponseBody)

packages/server/lib/modes/record.js

@@ -19,6 +19,7 @@ const terminal = require('../util/terminal')
 const ciProvider = require('../util/ci_provider')
 const settings = require('../util/settings')
 const testsUtils = require('../util/tests_utils')
+const specWriter = require('../util/spec_writer')
 
 const logException = (err) => {
   // give us up to 1 second to
@@ -200,7 +201,7 @@ const updateInstanceStdout = (options = {}) => {
 }
 
 const postInstanceResults = (options = {}) => {
-  const { runId, instanceId, results, group, parallel, ciBuildId } = options
+  const { runId, instanceId, results, group, parallel, ciBuildId, metadata } = options
   let { stats, tests, video, screenshots, reporterStats, error } = results
 
   video = Boolean(video)
@@ -226,6 +227,7 @@ const postInstanceResults = (options = {}) => {
     video,
     reporterStats,
     screenshots,
+    metadata,
   })
   .catch((err) => {
     debug('failed updating instance %o', {
@@ -684,13 +686,17 @@ const createRunAndRecordSpecs = (options = {}) => {
           console.log('')
         }
 
-        return postInstanceResults({
-          group,
-          config,
-          results,
-          parallel,
-          ciBuildId,
-          instanceId,
+        return specWriter.countStudioUsage(spec.absolute)
+        .then((metadata) => {
+          return postInstanceResults({
+            group,
+            config,
+            results,
+            parallel,
+            ciBuildId,
+            instanceId,
+            metadata,
+          })
         })
         .then((resp) => {
           if (!resp) {

packages/server/lib/util/spec_writer.ts

@@ -26,6 +26,12 @@ export interface FileDetails {
   line: number
 }
 
+const generateCommentText = (comment) => ` ==== ${comment} ==== `
+
+const createdComment = generateCommentText('Test Created with Cypress Studio')
+const extendedStartComment = generateCommentText('Generated with Cypress Studio')
+const extendedEndComment = generateCommentText('End Cypress Studio')
+
 export const generateCypressCommand = (cmd: Command) => {
   const { selector, name, message } = cmd
 
@@ -92,7 +98,7 @@ export const generateTest = (name: string, body: n.BlockStatement) => {
   )
 
   // adding the comment like this also adds a newline before the comment
-  stmt.comments = [b.block(' === Test Created with Cypress Studio === ', true, false)]
+  stmt.comments = [b.block(createdComment, true, false)]
 
   return stmt
 }
@@ -109,13 +115,13 @@ export const addCommentToBody = (body: Array<{}>, comment: string) => {
 }
 
 export const addCommandsToBody = (body: Array<{}>, commands: Command[]) => {
-  addCommentToBody(body, ' ==== Generated with Cypress Studio ==== ')
+  addCommentToBody(body, extendedStartComment)
 
   commands.forEach((command) => {
     body.push(generateCypressCommand(command))
   })
 
-  addCommentToBody(body, ' ==== End Cypress Studio ==== ')
+  addCommentToBody(body, extendedEndComment)
 
   return body
 }
@@ -258,3 +264,23 @@ export const rewriteSpec = (path: string, astRules: Visitor<{}>) => {
 export const createFile = (path: string) => {
   return fs.writeFile(path, newFileTemplate(path))
 }
+
+export const countStudioUsage = (path: string) => {
+  return fs.readFile(path)
+  .then((specBuffer) => {
+    const specContents = specBuffer.toString()
+    const createdRegex = new RegExp(createdComment, 'g')
+    const extendedRegex = new RegExp(extendedStartComment, 'g')
+
+    // TODO: remove when Studio goes GA
+    // earlier versions of studio used this comment to mark a created test
+    // which was later changed to be consistent with other Studio comments
+    const oldCreatedRegex = / === Test Created with Cypress Studio === /g
+    const oldStudioCreated = (specContents.match(oldCreatedRegex) || []).length
+
+    return {
+      studioCreated: (specContents.match(createdRegex) || []).length + oldStudioCreated,
+      studioExtended: (specContents.match(extendedRegex) || []).length,
+    }
+  })
+}

packages/server/package.json

@@ -126,7 +126,7 @@
     "@babel/core": "7.9.0",
     "@babel/preset-env": "7.9.0",
     "@cypress/debugging-proxy": "2.0.1",
-    "@cypress/json-schemas": "5.38.0",
+    "@cypress/json-schemas": "5.39.0",
     "@cypress/sinon-chai": "2.9.1",
     "@ffprobe-installer/ffprobe": "1.1.0",
     "@packages/desktop-gui": "0.0.0-development",

packages/server/test/e2e/7_record_spec.js

@@ -380,6 +380,27 @@ describe('e2e record', () => {
     })
   })
 
+  context('metadata', () => {
+    setupStubbedServer(createRoutes())
+
+    it('sends Studio usage metadata', function () {
+      return e2e.exec(this, {
+        key: 'f858a2bc-b469-4e48-be67-0876339ee7e1',
+        spec: 'studio_written.spec.js',
+        record: true,
+        snapshot: true,
+      })
+      .then(() => {
+        const postResults = requests[3]
+
+        expect(postResults.url).to.eq(`POST /instances/${instanceId}/results`)
+
+        expect(postResults.body.metadata.studioCreated).to.eq(2)
+        expect(postResults.body.metadata.studioExtended).to.eq(4)
+      })
+    })
+  })
+
   context('misconfiguration', () => {
     setupStubbedServer([])
 

packages/server/test/support/fixtures/projects/e2e/cypress/integration/studio_written.spec.js

@@ -0,0 +1,29 @@
+describe('suite', () => {
+  it('extended test', () => {
+    /* ==== Generated with Cypress Studio ==== */
+    cy.log('command')
+    /* ==== End Cypress Studio ==== */
+  })
+
+  /* ==== Test Created with Cypress Studio ==== */
+  it('created test', () => {
+    /* ==== Generated with Cypress Studio ==== */
+    cy.log('command')
+    /* ==== End Cypress Studio ==== */
+  })
+
+  context('nested suite', () => {
+    it('extended test', () => {
+      /* ==== Generated with Cypress Studio ==== */
+      cy.log('command')
+      /* ==== End Cypress Studio ==== */
+    })
+
+    /* ==== Test Created with Cypress Studio ==== */
+    it('created test', () => {
+      /* ==== Generated with Cypress Studio ==== */
+      cy.log('command')
+      /* ==== End Cypress Studio ==== */
+    })
+  })
+})