From 889a049f80a59042c9490e373ae9e5b748797dbd Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 6 Jan 2021 11:21:59 -0500 Subject: [PATCH 01/25] wip storybooks --- .ci/.storybook/main.js | 27 +++++++++++++++++++++++++++ src/dev/storybook/aliases.ts | 1 + test/scripts/jenkins_storybook.sh | 15 +++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 .ci/.storybook/main.js create mode 100644 test/scripts/jenkins_storybook.sh diff --git a/.ci/.storybook/main.js b/.ci/.storybook/main.js new file mode 100644 index 0000000000000..033ee290af2b2 --- /dev/null +++ b/.ci/.storybook/main.js @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +const config = require('@kbn/storybook').defaultConfig; +const aliases = require('../../src/dev/storybook/aliases.ts').storybookAliases; + +config.refs = {}; + +for(const alias of Object.keys(aliases).filter(a => a !== 'ci_composite')) { + const title = alias + .replace(/_/g, ' ') + .split(' ') + .map(n => n[0].toUpperCase()+n.slice(1) ) + .join(' '); + + config.refs[alias] = { + title: title, + url: 'TODO', + } +} + +console.log(config); + +module.exports = config; diff --git a/src/dev/storybook/aliases.ts b/src/dev/storybook/aliases.ts index a66661e7ae2ea..92e8877346aa9 100644 --- a/src/dev/storybook/aliases.ts +++ b/src/dev/storybook/aliases.ts @@ -21,6 +21,7 @@ export const storybookAliases = { apm: 'x-pack/plugins/apm/.storybook', canvas: 'x-pack/plugins/canvas/storybook', codeeditor: 'src/plugins/kibana_react/public/code_editor/.storybook', + ci_composite: '.ci/.storybook', dashboard: 'src/plugins/dashboard/.storybook', dashboard_enhanced: 'x-pack/plugins/dashboard_enhanced/.storybook', data_enhanced: 'x-pack/plugins/data_enhanced/.storybook', diff --git a/test/scripts/jenkins_storybook.sh b/test/scripts/jenkins_storybook.sh new file mode 100644 index 0000000000000..59e8bf927ae45 --- /dev/null +++ b/test/scripts/jenkins_storybook.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# source src/dev/ci_setup/setup_env.sh + +yarn storybook --site apm +yarn storybook --site canvas +yarn storybook --site codeeditor +yarn storybook --site dashboard +yarn storybook --site dashboard_enhanced +yarn storybook --site data_enhanced +yarn storybook --site embeddable +yarn storybook --site infra +yarn storybook --site security_solution +yarn storybook --site ui_actions_enhanced +yarn storybook --site observability From 688ef0f369aca7fe004d5d2374734462acbe6319 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 7 Jan 2021 13:38:30 -0500 Subject: [PATCH 02/25] WIP storybook building and uploading --- .ci/.storybook/main.js | 5 +-- test/scripts/jenkins_storybook.sh | 25 ++++++----- vars/kibanaPipeline.groovy | 37 +++++++++------- vars/storybooks.groovy | 74 +++++++++++++++++++++++++++++++ vars/tasks.groovy | 7 +++ 5 files changed, 117 insertions(+), 31 deletions(-) mode change 100644 => 100755 test/scripts/jenkins_storybook.sh create mode 100644 vars/storybooks.groovy diff --git a/.ci/.storybook/main.js b/.ci/.storybook/main.js index 033ee290af2b2..8813562e45ffb 100644 --- a/.ci/.storybook/main.js +++ b/.ci/.storybook/main.js @@ -10,6 +10,7 @@ const aliases = require('../../src/dev/storybook/aliases.ts').storybookAliases; config.refs = {}; for(const alias of Object.keys(aliases).filter(a => a !== 'ci_composite')) { + // snake_case -> Title Case const title = alias .replace(/_/g, ' ') .split(' ') @@ -18,10 +19,8 @@ for(const alias of Object.keys(aliases).filter(a => a !== 'ci_composite')) { config.refs[alias] = { title: title, - url: 'TODO', + url: `${process.env.STORYBOOK_BASE_URL}/${alias}`, } } -console.log(config); - module.exports = config; diff --git a/test/scripts/jenkins_storybook.sh b/test/scripts/jenkins_storybook.sh old mode 100644 new mode 100755 index 59e8bf927ae45..9e9d4e7a48db6 --- a/test/scripts/jenkins_storybook.sh +++ b/test/scripts/jenkins_storybook.sh @@ -1,15 +1,18 @@ #!/usr/bin/env bash -# source src/dev/ci_setup/setup_env.sh +source src/dev/ci_setup/setup_env.sh -yarn storybook --site apm -yarn storybook --site canvas -yarn storybook --site codeeditor +# TODO un-comment these after iterating + +# yarn storybook --site apm +# yarn storybook --site canvas +yarn storybook --site ci_composite +# yarn storybook --site codeeditor yarn storybook --site dashboard -yarn storybook --site dashboard_enhanced -yarn storybook --site data_enhanced -yarn storybook --site embeddable -yarn storybook --site infra -yarn storybook --site security_solution -yarn storybook --site ui_actions_enhanced -yarn storybook --site observability +# yarn storybook --site dashboard_enhanced +# yarn storybook --site data_enhanced +# yarn storybook --site embeddable +# yarn storybook --site infra +# yarn storybook --site security_solution +# yarn storybook --site ui_actions_enhanced +# yarn storybook --site observability diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy index 7991dd3252153..4d628298bada1 100644 --- a/vars/kibanaPipeline.groovy +++ b/vars/kibanaPipeline.groovy @@ -338,7 +338,7 @@ def call(Map params = [:], Closure closure) { buildState.set('shouldSetCommitStatus', true) } if (config.checkPrChanges && githubPr.isPr()) { - pipelineLibraryTests() + // pipelineLibraryTests() // TODO re-enable print "Checking PR for changes to determine if CI needs to be run..." @@ -421,17 +421,18 @@ def withTasks(Map params = [worker: [:]], Closure closure) { workers.ci(config) { withCiTaskQueue(parallel: 24) { - parallel([ - docker: { - retry(2) { - buildDocker() - } - }, - - // There are integration tests etc that require the plugins to be built first, so let's go ahead and build them before set up the parallel workspaces - ossPlugins: { buildOssPlugins() }, - xpackPlugins: { buildXpackPlugins() }, - ]) + // TODO re-enable + // parallel([ + // docker: { + // retry(2) { + // buildDocker() + // } + // }, + + // // There are integration tests etc that require the plugins to be built first, so let's go ahead and build them before set up the parallel workspaces + // ossPlugins: { buildOssPlugins() }, + // xpackPlugins: { buildXpackPlugins() }, + // ]) catchErrors { closure() @@ -443,11 +444,13 @@ def withTasks(Map params = [worker: [:]], Closure closure) { def allCiTasks() { withTasks { - tasks.check() - tasks.lint() - tasks.test() - tasks.functionalOss() - tasks.functionalXpack() + // TODO re-enable + // tasks.check() + // tasks.lint() + // tasks.test() + // tasks.functionalOss() + // tasks.functionalXpack() + tasks.storybooksCi() } } diff --git a/vars/storybooks.groovy b/vars/storybooks.groovy new file mode 100644 index 0000000000000..47ec7b606792b --- /dev/null +++ b/vars/storybooks.groovy @@ -0,0 +1,74 @@ +def STORYBOOKS_BUCKET = "ci-artifacts.kibana.dev/storybooks" + +def getDestinationDir() { + return env.ghprbPullId ? "pr-${env.ghprbPullId}" : buildState.get('checkoutInfo').branch.replace("/", "__") +} + +def getUrl() { + return "https://${STORYBOOKS_BUCKET}/${getDestinationDir()}" +} + +def getUrlLatest() { + return "${getUrl()}/latest" +} + +def getUrlForCommit() { + return "${getUrl()}/${buildState.get('checkoutInfo').commit}" +} + +def upload() { + dir("built_assets/storybook") { + sh "mv ci_composite composite" + + def storybooks = sh( + script: 'ls -1d */', + returnStdout: true + ).trim() + .split('\n') + .collect { it.replace('/', '') } + .findAll { it != 'composite' } + + def listHtml = storybooks.collect { """
  • ${it}
    • """ } + + def html = """ + + +

    Storybooks

    +

    Composite Storybook

    +

    All

    + + + + """ + + writeFile(file: 'index.html', text: html) + + googleStorageUpload( + credentialsId: 'kibana-ci-gcs-plugin', + bucket: "gs://${STORYBOOKS_BUCKET}/${getDestinationDir()}/${buildState.get('checkoutInfo').commit}", + pattern: "**/*", + sharedPublicly: true, + // showInline: false, + ) + + buildState.set('storybooksUrl', getUrlForCommit()) + + googleStorageUpload( + credentialsId: 'kibana-ci-gcs-plugin', + bucket: "gs://${STORYBOOKS_BUCKET}/${getDestinationDir()}/latest", + pattern: "index.html", + sharedPublicly: true, + // showInline: false, + ) + } +} + +def build() { + withEnv(["STORYBOOK_BASE_URL=${getUrlForCommit()}"]) { + kibanaPipeline.bash('test/scripts/jenkins_storybook.sh', 'Build Storybooks') + } +} + +return this diff --git a/vars/tasks.groovy b/vars/tasks.groovy index 18be6e69a2637..65549db0c4fdb 100644 --- a/vars/tasks.groovy +++ b/vars/tasks.groovy @@ -128,4 +128,11 @@ def functionalXpack(Map params = [:]) { } } +def storybooksCi() { + task { + storybooks.build() + storybooks.upload() + } +} + return this From a853aaabb1eb6d03bf62c2408b03b5bea7c0f334 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 7 Jan 2021 13:50:32 -0500 Subject: [PATCH 03/25] Switch variable to function --- vars/kibanaPipeline.groovy | 3 ++- vars/storybooks.groovy | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy index 4d628298bada1..ea888584edc14 100644 --- a/vars/kibanaPipeline.groovy +++ b/vars/kibanaPipeline.groovy @@ -420,7 +420,8 @@ def withTasks(Map params = [worker: [:]], Closure closure) { def config = [name: 'ci-worker', size: 'xxl', ramDisk: true] + (params.worker ?: [:]) workers.ci(config) { - withCiTaskQueue(parallel: 24) { + withCiTaskQueue(parallel: 1) { // TODO change back + // withCiTaskQueue(parallel: 24) { // TODO re-enable // parallel([ // docker: { diff --git a/vars/storybooks.groovy b/vars/storybooks.groovy index 47ec7b606792b..fbe3507054455 100644 --- a/vars/storybooks.groovy +++ b/vars/storybooks.groovy @@ -1,11 +1,13 @@ -def STORYBOOKS_BUCKET = "ci-artifacts.kibana.dev/storybooks" +def getStorybooksBucket() { + return "ci-artifacts.kibana.dev/storybooks" +} def getDestinationDir() { return env.ghprbPullId ? "pr-${env.ghprbPullId}" : buildState.get('checkoutInfo').branch.replace("/", "__") } def getUrl() { - return "https://${STORYBOOKS_BUCKET}/${getDestinationDir()}" + return "https://${getStorybooksBucket()}/${getDestinationDir()}" } def getUrlLatest() { @@ -47,7 +49,7 @@ def upload() { googleStorageUpload( credentialsId: 'kibana-ci-gcs-plugin', - bucket: "gs://${STORYBOOKS_BUCKET}/${getDestinationDir()}/${buildState.get('checkoutInfo').commit}", + bucket: "gs://${getStorybooksBucket()}/${getDestinationDir()}/${buildState.get('checkoutInfo').commit}", pattern: "**/*", sharedPublicly: true, // showInline: false, @@ -57,7 +59,7 @@ def upload() { googleStorageUpload( credentialsId: 'kibana-ci-gcs-plugin', - bucket: "gs://${STORYBOOKS_BUCKET}/${getDestinationDir()}/latest", + bucket: "gs://${getStorybooksBucket()}/${getDestinationDir()}/latest", pattern: "index.html", sharedPublicly: true, // showInline: false, From de28159bb1a87cecf3532659efd5c1b6bf210b17 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 7 Jan 2021 14:15:12 -0500 Subject: [PATCH 04/25] Fix path --- vars/storybooks.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/storybooks.groovy b/vars/storybooks.groovy index fbe3507054455..3cae0c8954c7b 100644 --- a/vars/storybooks.groovy +++ b/vars/storybooks.groovy @@ -36,7 +36,7 @@ def upload() {

    Storybooks

    -

    Composite Storybook

    +

    Composite Storybook

    All

      ${listHtml} From 12409da515e981078d0c893a990e3c4a5285a03e Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 7 Jan 2021 14:31:52 -0500 Subject: [PATCH 05/25] Don't try to set individual object permissions --- vars/storybooks.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/storybooks.groovy b/vars/storybooks.groovy index 3cae0c8954c7b..a85ccc9dff685 100644 --- a/vars/storybooks.groovy +++ b/vars/storybooks.groovy @@ -51,7 +51,7 @@ def upload() { credentialsId: 'kibana-ci-gcs-plugin', bucket: "gs://${getStorybooksBucket()}/${getDestinationDir()}/${buildState.get('checkoutInfo').commit}", pattern: "**/*", - sharedPublicly: true, + sharedPublicly: false, // showInline: false, ) @@ -61,7 +61,7 @@ def upload() { credentialsId: 'kibana-ci-gcs-plugin', bucket: "gs://${getStorybooksBucket()}/${getDestinationDir()}/latest", pattern: "index.html", - sharedPublicly: true, + sharedPublicly: false, // showInline: false, ) } From c3dc05ae5c622219bc86353dcf68d0d34917e335 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 7 Jan 2021 14:47:56 -0500 Subject: [PATCH 06/25] Add showInline --- vars/storybooks.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/storybooks.groovy b/vars/storybooks.groovy index a85ccc9dff685..859345655af44 100644 --- a/vars/storybooks.groovy +++ b/vars/storybooks.groovy @@ -52,7 +52,7 @@ def upload() { bucket: "gs://${getStorybooksBucket()}/${getDestinationDir()}/${buildState.get('checkoutInfo').commit}", pattern: "**/*", sharedPublicly: false, - // showInline: false, + showInline: true, ) buildState.set('storybooksUrl', getUrlForCommit()) @@ -62,7 +62,7 @@ def upload() { bucket: "gs://${getStorybooksBucket()}/${getDestinationDir()}/latest", pattern: "index.html", sharedPublicly: false, - // showInline: false, + showInline: true, ) } } From 2fccd8a2b8c429bb5fd4e6a7c93d9b0e467ca1e0 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 7 Jan 2021 16:30:44 -0500 Subject: [PATCH 07/25] Make storybook assets relative paths --- packages/kbn-storybook/lib/templates/index.ejs | 10 +++++----- test/scripts/jenkins_storybook.sh | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/kbn-storybook/lib/templates/index.ejs b/packages/kbn-storybook/lib/templates/index.ejs index a4f8204c95d7a..b193c87824d40 100644 --- a/packages/kbn-storybook/lib/templates/index.ejs +++ b/packages/kbn-storybook/lib/templates/index.ejs @@ -16,12 +16,12 @@ - - - - + + + + <% if (typeof headHtmlSnippet !== 'undefined') { %> <%= headHtmlSnippet %> <% } %> <% diff --git a/test/scripts/jenkins_storybook.sh b/test/scripts/jenkins_storybook.sh index 9e9d4e7a48db6..9b54cc77703fd 100755 --- a/test/scripts/jenkins_storybook.sh +++ b/test/scripts/jenkins_storybook.sh @@ -5,9 +5,9 @@ source src/dev/ci_setup/setup_env.sh # TODO un-comment these after iterating # yarn storybook --site apm -# yarn storybook --site canvas +yarn storybook --site canvas yarn storybook --site ci_composite -# yarn storybook --site codeeditor +yarn storybook --site codeeditor yarn storybook --site dashboard # yarn storybook --site dashboard_enhanced # yarn storybook --site data_enhanced From c447a9df0a8b41170bd4bf58ca1ae86354ea2910 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 7 Jan 2021 16:43:43 -0500 Subject: [PATCH 08/25] Build canvas dll --- test/scripts/jenkins_storybook.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/scripts/jenkins_storybook.sh b/test/scripts/jenkins_storybook.sh index 9b54cc77703fd..76f761fe2972c 100755 --- a/test/scripts/jenkins_storybook.sh +++ b/test/scripts/jenkins_storybook.sh @@ -2,6 +2,11 @@ source src/dev/ci_setup/setup_env.sh +cd "$XPACK_DIR/plugins/canvas" +node scripts/storybook --dll + +cd "$KIBANA_DIR" + # TODO un-comment these after iterating # yarn storybook --site apm From 9da83af942cecf5b1053a76effa872c4021aa5ef Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 7 Jan 2021 17:10:56 -0500 Subject: [PATCH 09/25] Change canvas asset paths also --- x-pack/plugins/canvas/storybook/preview-head.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/canvas/storybook/preview-head.html b/x-pack/plugins/canvas/storybook/preview-head.html index bef08a5120a36..f8a7de6ddbaf1 100644 --- a/x-pack/plugins/canvas/storybook/preview-head.html +++ b/x-pack/plugins/canvas/storybook/preview-head.html @@ -2,5 +2,5 @@ This file is looked for by Storybook and included in the HEAD element if it exists. This is how we load the DLL content into the Storybook UI. --> - - + + From d7d00ad7f7139c1f48b4ce35e2adf33d327308b8 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 7 Jan 2021 17:38:21 -0500 Subject: [PATCH 10/25] Build all storybooks --- test/scripts/jenkins_storybook.sh | 16 ++++++++-------- vars/storybooks.groovy | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/scripts/jenkins_storybook.sh b/test/scripts/jenkins_storybook.sh index 76f761fe2972c..4368ddad920b1 100755 --- a/test/scripts/jenkins_storybook.sh +++ b/test/scripts/jenkins_storybook.sh @@ -9,15 +9,15 @@ cd "$KIBANA_DIR" # TODO un-comment these after iterating -# yarn storybook --site apm +yarn storybook --site apm yarn storybook --site canvas yarn storybook --site ci_composite yarn storybook --site codeeditor yarn storybook --site dashboard -# yarn storybook --site dashboard_enhanced -# yarn storybook --site data_enhanced -# yarn storybook --site embeddable -# yarn storybook --site infra -# yarn storybook --site security_solution -# yarn storybook --site ui_actions_enhanced -# yarn storybook --site observability +yarn storybook --site dashboard_enhanced +yarn storybook --site data_enhanced +yarn storybook --site embeddable +yarn storybook --site infra +yarn storybook --site security_solution +yarn storybook --site ui_actions_enhanced +yarn storybook --site observability diff --git a/vars/storybooks.groovy b/vars/storybooks.groovy index 859345655af44..5b73fbf18e2d3 100644 --- a/vars/storybooks.groovy +++ b/vars/storybooks.groovy @@ -30,7 +30,7 @@ def upload() { .collect { it.replace('/', '') } .findAll { it != 'composite' } - def listHtml = storybooks.collect { """
    • ${it}
      • """ } + def listHtml = storybooks.collect { """
    • ${it}
      • """ }.join("\n") def html = """ From 9cc3944e23fe71f5bf54335a2c97bb62db3226ec Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 7 Jan 2021 17:44:54 -0500 Subject: [PATCH 11/25] Add storybooks pr comment link --- vars/githubPr.groovy | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/vars/githubPr.groovy b/vars/githubPr.groovy index 546a6785ac2f4..c635225eccb4e 100644 --- a/vars/githubPr.groovy +++ b/vars/githubPr.groovy @@ -169,12 +169,18 @@ def getNextCommentMessage(previousCommentInfo = [:], isFinal = false) { ? getBuildStatusIncludingMetrics() : buildUtils.getBuildStatus() + def storybooksUrl = buildState.get('storybooksUrl') + def storybooksMessage = storybooksUrl ? "* [Storybooks Preview](${storybooksUrl})" : "* Storybooks not built" + if (!isFinal) { + storybooksMessage = storybooksUrl ? storybooksMessage : "* Storybooks not built yet" + def failuresPart = status != 'SUCCESS' ? ', with failures' : '' messages << """ ## :hourglass_flowing_sand: Build in-progress${failuresPart} * [continuous-integration/kibana-ci/pull-request](${env.BUILD_URL}) * Commit: ${getCommitHash()} + ${storybooksMessage} * This comment will update when the build is complete """ } else if (status == 'SUCCESS') { @@ -182,12 +188,14 @@ def getNextCommentMessage(previousCommentInfo = [:], isFinal = false) { ## :green_heart: Build Succeeded * [continuous-integration/kibana-ci/pull-request](${env.BUILD_URL}) * Commit: ${getCommitHash()} + ${storybooksMessage} """ } else if(status == 'UNSTABLE') { def message = """ ## :yellow_heart: Build succeeded, but was flaky * [continuous-integration/kibana-ci/pull-request](${env.BUILD_URL}) * Commit: ${getCommitHash()} + ${storybooksMessage} """.stripIndent() def failures = retryable.getFlakyFailures() @@ -202,6 +210,7 @@ def getNextCommentMessage(previousCommentInfo = [:], isFinal = false) { ## :broken_heart: Build Failed * [continuous-integration/kibana-ci/pull-request](${env.BUILD_URL}) * Commit: ${getCommitHash()} + ${storybooksMessage} * [Pipeline Steps](${env.BUILD_URL}flowGraphTable) (look for red circles / failed steps) * [Interpreting CI Failures](https://www.elastic.co/guide/en/kibana/current/interpreting-ci-failures.html) """ From 922ba2cab3ea1164d6b2e4c36d86d0b8ab93268c Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 11 Jan 2021 14:47:40 -0500 Subject: [PATCH 12/25] Use gsutil for upload storybooks assets --- vars/storybooks.groovy | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/vars/storybooks.groovy b/vars/storybooks.groovy index 5b73fbf18e2d3..f09018bca1c40 100644 --- a/vars/storybooks.groovy +++ b/vars/storybooks.groovy @@ -47,23 +47,14 @@ def upload() { writeFile(file: 'index.html', text: html) - googleStorageUpload( - credentialsId: 'kibana-ci-gcs-plugin', - bucket: "gs://${getStorybooksBucket()}/${getDestinationDir()}/${buildState.get('checkoutInfo').commit}", - pattern: "**/*", - sharedPublicly: false, - showInline: true, - ) + withGcpServiceAccount.fromVaultSecret('secret/kibana-issues/dev/ci-artifacts-key', 'value') { + kibanaPipeline.bash(""" + gsutil -m cp -r -z js,css,html '*' 'gs://${getStorybooksBucket()}/${getDestinationDir()}/${buildState.get('checkoutInfo').commit}/' + gsutil -h "Cache-Control:no-cache, max-age=0, no-transform" cp -z html 'index.html' 'gs://${getStorybooksBucket()}/${getDestinationDir()}/latest/' + """, "Upload Storybooks to GCS") + } buildState.set('storybooksUrl', getUrlForCommit()) - - googleStorageUpload( - credentialsId: 'kibana-ci-gcs-plugin', - bucket: "gs://${getStorybooksBucket()}/${getDestinationDir()}/latest", - pattern: "index.html", - sharedPublicly: false, - showInline: true, - ) } } From 6c5dca53b671f41da26d6d73ea536e4f8f2de511 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 11 Jan 2021 15:01:39 -0500 Subject: [PATCH 13/25] Quiet gsutil and add more compressable file types --- vars/storybooks.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/storybooks.groovy b/vars/storybooks.groovy index f09018bca1c40..3ae2a074efa29 100644 --- a/vars/storybooks.groovy +++ b/vars/storybooks.groovy @@ -49,7 +49,7 @@ def upload() { withGcpServiceAccount.fromVaultSecret('secret/kibana-issues/dev/ci-artifacts-key', 'value') { kibanaPipeline.bash(""" - gsutil -m cp -r -z js,css,html '*' 'gs://${getStorybooksBucket()}/${getDestinationDir()}/${buildState.get('checkoutInfo').commit}/' + gsutil -q -m cp -r -z js,css,html,json,map,txt,svg '*' 'gs://${getStorybooksBucket()}/${getDestinationDir()}/${buildState.get('checkoutInfo').commit}/' gsutil -h "Cache-Control:no-cache, max-age=0, no-transform" cp -z html 'index.html' 'gs://${getStorybooksBucket()}/${getDestinationDir()}/latest/' """, "Upload Storybooks to GCS") } From b54b34c91b028276a734889c13801179d6ae6878 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 11 Jan 2021 16:31:09 -0500 Subject: [PATCH 14/25] Add github commit status for storybooks --- test/scripts/jenkins_storybook.sh | 20 ++++++++++---------- vars/githubCommitStatus.groovy | 6 ++++-- vars/storybooks.groovy | 16 ++++++++++++++++ vars/tasks.groovy | 3 +-- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/test/scripts/jenkins_storybook.sh b/test/scripts/jenkins_storybook.sh index 4368ddad920b1..fa9bc867a27e2 100755 --- a/test/scripts/jenkins_storybook.sh +++ b/test/scripts/jenkins_storybook.sh @@ -9,15 +9,15 @@ cd "$KIBANA_DIR" # TODO un-comment these after iterating -yarn storybook --site apm -yarn storybook --site canvas +# yarn storybook --site apm +# yarn storybook --site canvas yarn storybook --site ci_composite -yarn storybook --site codeeditor +# yarn storybook --site codeeditor yarn storybook --site dashboard -yarn storybook --site dashboard_enhanced -yarn storybook --site data_enhanced -yarn storybook --site embeddable -yarn storybook --site infra -yarn storybook --site security_solution -yarn storybook --site ui_actions_enhanced -yarn storybook --site observability +# yarn storybook --site dashboard_enhanced +# yarn storybook --site data_enhanced +# yarn storybook --site embeddable +# yarn storybook --site infra +# yarn storybook --site security_solution +# yarn storybook --site ui_actions_enhanced +# yarn storybook --site observability diff --git a/vars/githubCommitStatus.groovy b/vars/githubCommitStatus.groovy index 248d226169a61..0603a07c11634 100644 --- a/vars/githubCommitStatus.groovy +++ b/vars/githubCommitStatus.groovy @@ -41,13 +41,15 @@ def trackBuild(commit, context, Closure closure) { } // state: error|failure|pending|success -def create(sha, state, description, context) { +def create(sha, state, description, context, targetUrl = null) { + targetUrl = targetUrl ?: env.BUILD_URL + withGithubCredentials { return githubApi.post("repos/elastic/kibana/statuses/${sha}", [ state: state, description: description, context: context, - target_url: env.BUILD_URL + target_url: targetUrl ]) } } diff --git a/vars/storybooks.groovy b/vars/storybooks.groovy index 3ae2a074efa29..6580151ee3790 100644 --- a/vars/storybooks.groovy +++ b/vars/storybooks.groovy @@ -64,4 +64,20 @@ def build() { } } +def buildAndUpload() { + def sha = buildState.get('checkoutInfo').commit + def context = 'kibana-ci-storybooks' + + githubCommitStatus.create(sha, 'pending', 'Building Storybooks', context) + + try { + build() + upload() + githubCommitStatus.create(sha, 'success', 'Storybooks built', context, buildState.get('storybooksUrl')) + } catch(ex) { + githubCommitStatus.create(sha, 'error', 'Building Storybooks failed', context) + throw ex + } +} + return this diff --git a/vars/tasks.groovy b/vars/tasks.groovy index 65549db0c4fdb..2cfe420923743 100644 --- a/vars/tasks.groovy +++ b/vars/tasks.groovy @@ -130,8 +130,7 @@ def functionalXpack(Map params = [:]) { def storybooksCi() { task { - storybooks.build() - storybooks.upload() + storybooks.buildAndUpload() } } From f44215993ab4598629dbe623ec4e4c4abfe4d5be Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 11 Jan 2021 16:48:24 -0500 Subject: [PATCH 15/25] Change storybooks context and debug commit status --- vars/githubCommitStatus.groovy | 5 +++++ vars/storybooks.groovy | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/vars/githubCommitStatus.groovy b/vars/githubCommitStatus.groovy index 0603a07c11634..36614f8f8ec9c 100644 --- a/vars/githubCommitStatus.groovy +++ b/vars/githubCommitStatus.groovy @@ -42,8 +42,13 @@ def trackBuild(commit, context, Closure closure) { // state: error|failure|pending|success def create(sha, state, description, context, targetUrl = null) { + print "Target URL:" + print targetUrl + targetUrl = targetUrl ?: env.BUILD_URL + print targetUrl + withGithubCredentials { return githubApi.post("repos/elastic/kibana/statuses/${sha}", [ state: state, diff --git a/vars/storybooks.groovy b/vars/storybooks.groovy index 6580151ee3790..76b18b541ec7a 100644 --- a/vars/storybooks.groovy +++ b/vars/storybooks.groovy @@ -66,14 +66,14 @@ def build() { def buildAndUpload() { def sha = buildState.get('checkoutInfo').commit - def context = 'kibana-ci-storybooks' + def context = 'ci/storybooks' githubCommitStatus.create(sha, 'pending', 'Building Storybooks', context) try { build() upload() - githubCommitStatus.create(sha, 'success', 'Storybooks built', context, buildState.get('storybooksUrl')) + githubCommitStatus.create(sha, 'success', 'Storybooks built', context, getUrlForCommit()) } catch(ex) { githubCommitStatus.create(sha, 'error', 'Building Storybooks failed', context) throw ex From 705049a0abba00108ea6092d3866596ed32f6699 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 11 Jan 2021 17:03:50 -0500 Subject: [PATCH 16/25] Update context again and add toString() to URL --- vars/githubCommitStatus.groovy | 7 +------ vars/storybooks.groovy | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/vars/githubCommitStatus.groovy b/vars/githubCommitStatus.groovy index 36614f8f8ec9c..175dbe0c90542 100644 --- a/vars/githubCommitStatus.groovy +++ b/vars/githubCommitStatus.groovy @@ -42,19 +42,14 @@ def trackBuild(commit, context, Closure closure) { // state: error|failure|pending|success def create(sha, state, description, context, targetUrl = null) { - print "Target URL:" - print targetUrl - targetUrl = targetUrl ?: env.BUILD_URL - print targetUrl - withGithubCredentials { return githubApi.post("repos/elastic/kibana/statuses/${sha}", [ state: state, description: description, context: context, - target_url: targetUrl + target_url: targetUrl.toString() ]) } } diff --git a/vars/storybooks.groovy b/vars/storybooks.groovy index 76b18b541ec7a..f3c4a97a7d436 100644 --- a/vars/storybooks.groovy +++ b/vars/storybooks.groovy @@ -66,7 +66,7 @@ def build() { def buildAndUpload() { def sha = buildState.get('checkoutInfo').commit - def context = 'ci/storybooks' + def context = 'Build and Publish Storybooks' githubCommitStatus.create(sha, 'pending', 'Building Storybooks', context) From b891071cf90dd77c0bcd98eb588b5f61a716c795 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 11 Jan 2021 17:23:04 -0500 Subject: [PATCH 17/25] Build all storybooks again --- test/scripts/jenkins_storybook.sh | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/test/scripts/jenkins_storybook.sh b/test/scripts/jenkins_storybook.sh index fa9bc867a27e2..333956cd969ec 100755 --- a/test/scripts/jenkins_storybook.sh +++ b/test/scripts/jenkins_storybook.sh @@ -7,17 +7,15 @@ node scripts/storybook --dll cd "$KIBANA_DIR" -# TODO un-comment these after iterating - -# yarn storybook --site apm -# yarn storybook --site canvas +yarn storybook --site apm +yarn storybook --site canvas yarn storybook --site ci_composite -# yarn storybook --site codeeditor +yarn storybook --site codeeditor yarn storybook --site dashboard -# yarn storybook --site dashboard_enhanced -# yarn storybook --site data_enhanced -# yarn storybook --site embeddable -# yarn storybook --site infra -# yarn storybook --site security_solution -# yarn storybook --site ui_actions_enhanced -# yarn storybook --site observability +yarn storybook --site dashboard_enhanced +yarn storybook --site data_enhanced +yarn storybook --site embeddable +yarn storybook --site infra +yarn storybook --site security_solution +yarn storybook --site ui_actions_enhanced +yarn storybook --site observability From 5b1ed9654696e349f24875ac695464220ab29eb6 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 13 Jan 2021 11:57:10 -0500 Subject: [PATCH 18/25] Re-enable full pipeline --- vars/kibanaPipeline.groovy | 39 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy index ea888584edc14..8f6ef8cc40a90 100644 --- a/vars/kibanaPipeline.groovy +++ b/vars/kibanaPipeline.groovy @@ -338,7 +338,7 @@ def call(Map params = [:], Closure closure) { buildState.set('shouldSetCommitStatus', true) } if (config.checkPrChanges && githubPr.isPr()) { - // pipelineLibraryTests() // TODO re-enable + pipelineLibraryTests() print "Checking PR for changes to determine if CI needs to be run..." @@ -420,20 +420,18 @@ def withTasks(Map params = [worker: [:]], Closure closure) { def config = [name: 'ci-worker', size: 'xxl', ramDisk: true] + (params.worker ?: [:]) workers.ci(config) { - withCiTaskQueue(parallel: 1) { // TODO change back - // withCiTaskQueue(parallel: 24) { - // TODO re-enable - // parallel([ - // docker: { - // retry(2) { - // buildDocker() - // } - // }, - - // // There are integration tests etc that require the plugins to be built first, so let's go ahead and build them before set up the parallel workspaces - // ossPlugins: { buildOssPlugins() }, - // xpackPlugins: { buildXpackPlugins() }, - // ]) + withCiTaskQueue(parallel: 24) { + parallel([ + docker: { + retry(2) { + buildDocker() + } + }, + + // There are integration tests etc that require the plugins to be built first, so let's go ahead and build them before set up the parallel workspaces + ossPlugins: { buildOssPlugins() }, + xpackPlugins: { buildXpackPlugins() }, + ]) catchErrors { closure() @@ -445,12 +443,11 @@ def withTasks(Map params = [worker: [:]], Closure closure) { def allCiTasks() { withTasks { - // TODO re-enable - // tasks.check() - // tasks.lint() - // tasks.test() - // tasks.functionalOss() - // tasks.functionalXpack() + tasks.check() + tasks.lint() + tasks.test() + tasks.functionalOss() + tasks.functionalXpack() tasks.storybooksCi() } } From dbd057408039e4fbe27bc3489d9baa094d176e08 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 13 Jan 2021 16:58:53 -0500 Subject: [PATCH 19/25] Limit storybook webpack parallelism in CI, and enable caching --- packages/kbn-storybook/lib/default_config.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/kbn-storybook/lib/default_config.ts b/packages/kbn-storybook/lib/default_config.ts index c3bc65059d4a6..a3f16418620e8 100644 --- a/packages/kbn-storybook/lib/default_config.ts +++ b/packages/kbn-storybook/lib/default_config.ts @@ -25,4 +25,11 @@ export const defaultConfig: StorybookConfig = { typescript: { reactDocgen: false, }, + webpackFinal: (config, options) => { + if (process.env.CI) { + config.parallelism = 4; + config.cache = true; + } + return config; + }, }; From 7cdeeea9b2f1fc2a67f26afc8a5361b127111798 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 29 Jan 2021 12:07:23 -0500 Subject: [PATCH 20/25] Add comment --- src/dev/storybook/aliases.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dev/storybook/aliases.ts b/src/dev/storybook/aliases.ts index 8a73145ac07e3..0a6c9351c9850 100644 --- a/src/dev/storybook/aliases.ts +++ b/src/dev/storybook/aliases.ts @@ -6,6 +6,7 @@ * Public License, v 1. */ +// Please also add new aliases to test/scripts/jenkins_storybook.sh export const storybookAliases = { apm: 'x-pack/plugins/apm/.storybook', canvas: 'x-pack/plugins/canvas/storybook', From 1ca927ac0a9bb647bd48ef84f60264cb81b7ea52 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 29 Jan 2021 12:08:43 -0500 Subject: [PATCH 21/25] Add missing storybook --- test/scripts/jenkins_storybook.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/scripts/jenkins_storybook.sh b/test/scripts/jenkins_storybook.sh index 333956cd969ec..6bd4288f97d15 100755 --- a/test/scripts/jenkins_storybook.sh +++ b/test/scripts/jenkins_storybook.sh @@ -19,3 +19,4 @@ yarn storybook --site infra yarn storybook --site security_solution yarn storybook --site ui_actions_enhanced yarn storybook --site observability +yarn storybook --site presentation From d186f39437c7e47afbb5c2a34f65ceb729865830 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 1 Feb 2021 14:13:53 -0500 Subject: [PATCH 22/25] Disable APM storybook in CI for now --- test/scripts/jenkins_storybook.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/scripts/jenkins_storybook.sh b/test/scripts/jenkins_storybook.sh index 6bd4288f97d15..744b8cc76c717 100755 --- a/test/scripts/jenkins_storybook.sh +++ b/test/scripts/jenkins_storybook.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -euo pipefail + source src/dev/ci_setup/setup_env.sh cd "$XPACK_DIR/plugins/canvas" @@ -7,7 +9,7 @@ node scripts/storybook --dll cd "$KIBANA_DIR" -yarn storybook --site apm +# yarn storybook --site apm # TODO re-enable after being fixed yarn storybook --site canvas yarn storybook --site ci_composite yarn storybook --site codeeditor From ccdc9dd084128fe388adfc34cfb144ed9a62d8af Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 1 Feb 2021 16:32:51 -0500 Subject: [PATCH 23/25] Remove set --- test/scripts/jenkins_storybook.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/scripts/jenkins_storybook.sh b/test/scripts/jenkins_storybook.sh index 744b8cc76c717..4407cc3d1bfb6 100755 --- a/test/scripts/jenkins_storybook.sh +++ b/test/scripts/jenkins_storybook.sh @@ -1,7 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail - source src/dev/ci_setup/setup_env.sh cd "$XPACK_DIR/plugins/canvas" From 3c651d0f71d36e27332ebefc05ec41781c8c1552 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 5 Feb 2021 13:10:02 -0500 Subject: [PATCH 24/25] Fix license --- .ci/.storybook/main.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.ci/.storybook/main.js b/.ci/.storybook/main.js index 8813562e45ffb..663f743c6e2c6 100644 --- a/.ci/.storybook/main.js +++ b/.ci/.storybook/main.js @@ -1,7 +1,9 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ const config = require('@kbn/storybook').defaultConfig; From ce87f6a60042809a1a375f3d533ece053513548c Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 5 Feb 2021 13:17:00 -0500 Subject: [PATCH 25/25] Make sure eslint lints the .ci directory --- .ci/.storybook/main.js | 6 +++--- .eslintignore | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.ci/.storybook/main.js b/.ci/.storybook/main.js index 663f743c6e2c6..e399ec087e168 100644 --- a/.ci/.storybook/main.js +++ b/.ci/.storybook/main.js @@ -11,18 +11,18 @@ const aliases = require('../../src/dev/storybook/aliases.ts').storybookAliases; config.refs = {}; -for(const alias of Object.keys(aliases).filter(a => a !== 'ci_composite')) { +for (const alias of Object.keys(aliases).filter((a) => a !== 'ci_composite')) { // snake_case -> Title Case const title = alias .replace(/_/g, ' ') .split(' ') - .map(n => n[0].toUpperCase()+n.slice(1) ) + .map((n) => n[0].toUpperCase() + n.slice(1)) .join(' '); config.refs[alias] = { title: title, url: `${process.env.STORYBOOK_BASE_URL}/${alias}`, - } + }; } module.exports = config; diff --git a/.eslintignore b/.eslintignore index 5513ad1320232..83927ea09852a 100644 --- a/.eslintignore +++ b/.eslintignore @@ -15,6 +15,7 @@ node_modules target snapshots.js +!/.ci !/.eslintrc.js !.storybook