From 941434a7466925e3c7db3c2eb7fb07edf83d1be6 Mon Sep 17 00:00:00 2001 From: Ablai Akhazhanov Date: Mon, 30 Jan 2023 23:37:53 +0000 Subject: [PATCH 1/4] Fix bug in the front-end test --- .../helloworld.spec.js | 43 ++++++++++++------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/test/frontend-integration-test/helloworld.spec.js b/test/frontend-integration-test/helloworld.spec.js index 581ac42a9da..d75d5524ddf 100644 --- a/test/frontend-integration-test/helloworld.spec.js +++ b/test/frontend-integration-test/helloworld.spec.js @@ -72,6 +72,10 @@ describe('deploy helloworld sample run', () => { $('#experimentDescription').setValue(experimentDescription); $('#createExperimentBtn').click(); + + browser.waitUntil(() => { + return new URL(browser.getUrl()).hash.startsWith('#/runs/new'); + }, waitTimeout); }); it('creates a new run in the experiment', () => { @@ -94,6 +98,7 @@ describe('deploy helloworld sample run', () => { $('#usePipelineVersionBtn').click(); $('#pipelineVersionSelectorDialog').waitForVisible(waitTimeout, true); + browser.pause(1000); browser.keys(runName); @@ -135,31 +140,35 @@ describe('deploy helloworld sample run', () => { assert(attempts, 'waited for 30 seconds but run did not start.'); assert.equal($$('.tableRow').length, 1, 'should only show one run'); - - // Navigate to details of the deployed run by clicking its anchor element - browser.execute('document.querySelector(".tableRow a").click()'); - }); - - it('switches to config tab', () => { - $('button=Config').waitForVisible(waitTimeout); - $('button=Config').click(); }); + // Wait for a reasonable amount of time until the run is done + // and navigate to the run details page it('waits for run to finish', () => { - let status = getValueFromDetailsTable('Status'); - let attempts = 0; - const maxAttempts = 60; + const maxAttempts = 90; - // Wait for a reasonable amount of time until the run is done - while (attempts < maxAttempts && status.trim() !== 'Succeeded') { + while (attempts < maxAttempts) { browser.pause(1000); - status = getValueFromDetailsTable('Status'); attempts++; } - assert(attempts < maxAttempts, `waited for ${maxAttempts} seconds but run did not succeed. ` + - 'Current status is: ' + status); + browser.execute('document.querySelector(".tableRow a").click()'); + browser.waitUntil(() => { + return new URL(browser.getUrl()).hash.startsWith('#/runs/details'); + }, waitTimeout); + }); + + it('switches to config tab', () => { + $('button=Config').waitForVisible(); + $('button=Config').click(); + browser.pause(waitTimeout); + }); + + it('verifies run status', () => { + const status = getValueFromDetailsTable('Status'); + assert.equal(status.trim(), 'Succeeded', + 'run has not finished on time. Current status is: ' + status); }); it('displays run created at date correctly', () => { @@ -179,6 +188,7 @@ describe('deploy helloworld sample run', () => { it('has a 4-node graph', () => { const nodeSelector = '.graphNode'; + $(nodeSelector).waitForVisible(); const nodes = $$(nodeSelector).length; assert(nodes === 4, 'should have a 4-node graph, instead has: ' + nodes); }); @@ -218,6 +228,7 @@ describe('deploy helloworld sample run', () => { $('#usePipelineBtn').click(); $('#pipelineSelectorDialog').waitForVisible(waitTimeout, true); + browser.pause(1000); browser.keys('Tab'); browser.keys(runWithoutExperimentName); From 56a83ac0f4721934ad0395b00fac25abb49d3595 Mon Sep 17 00:00:00 2001 From: Ablai Akhazhanov Date: Mon, 30 Jan 2023 23:38:34 +0000 Subject: [PATCH 2/4] Add local debugging to the FE test --- test/frontend-integration-test/README.md | 24 +++++++++ test/frontend-integration-test/run_test.sh | 61 +++++++++++++++------- 2 files changed, 65 insertions(+), 20 deletions(-) create mode 100644 test/frontend-integration-test/README.md diff --git a/test/frontend-integration-test/README.md b/test/frontend-integration-test/README.md new file mode 100644 index 00000000000..b2090b051b2 --- /dev/null +++ b/test/frontend-integration-test/README.md @@ -0,0 +1,24 @@ +# Frontend integration test + +This test gets triggered by the end-to-end testing workflows. + +## Local run + +1. Deploy KFP on a k8s cluster and enable port forwarding. Replace the default namespace `kubeflow` below if needed. + + ```bash + POD=`kubectl get pods -n kubeflow -l app=ml-pipeline-ui -o jsonpath='{.items[0].metadata.name}'` + kubectl port-forward -n kubeflow ${POD} 3000:3000 & + ``` + +1. Build the container with the tests: + + ```bash + docker build . -t kfp-frontend-integration-test:local + ``` + +1. Run the test with enabled networking (**this exposes your local networking stack to the testing container**): + + ```bash + docker run --net=host kfp-frontend-integration-test:local --remote-run true + ``` diff --git a/test/frontend-integration-test/run_test.sh b/test/frontend-integration-test/run_test.sh index eaa4965afeb..362ef51af4c 100755 --- a/test/frontend-integration-test/run_test.sh +++ b/test/frontend-integration-test/run_test.sh @@ -18,16 +18,29 @@ set -xe # K8s Namespace that all resources deployed to NAMESPACE=kubeflow +REMOTE_RUN=false usage() { echo "usage: run_test.sh --results-gcs-dir GCS directory for the test results. Usually gs:////e2e_test [--namespace k8s namespace where ml-pipelines is deployed. The tests run against the instance in this namespace] + [--remote-run host address of a remote KFP UI. Used for local tests only.] [-h help]" } -while [ "$1" != "" ]; do +function parse_bool { + local str="${1:-false}" + local pat='^(true|1|yes)$' + if [[ "$str" =~ $pat ]] + then + echo 'true' + else + echo 'false' + fi +} + +while [[ "$1" != "" ]]; do case $1 in --results-gcs-dir )shift RESULTS_GCS_DIR=$1 @@ -35,6 +48,9 @@ while [ "$1" != "" ]; do --namespace ) shift NAMESPACE=$1 ;; + --remote-run ) shift + REMOTE_RUN=$(parse_bool "${1:-}") + ;; -h | --help ) usage exit ;; @@ -44,29 +60,32 @@ while [ "$1" != "" ]; do shift done -if [ -z "$RESULTS_GCS_DIR" ]; then - usage - exit 1 -fi - -if [[ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then - gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}" +if [[ "$REMOTE_RUN" != "true" ]]; then + if [[ -z "$RESULTS_GCS_DIR" ]]; then + usage + exit 1 + fi + if [[ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then + gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}" + fi fi npm install -function clean_up() { - set +e +if [[ "$REMOTE_RUN" != "true" ]]; then + function clean_up() { + set +e - echo "Stopping background jobs..." - kill -15 %1 - kill -15 %2 -} -trap clean_up EXIT SIGINT SIGTERM + echo "Stopping background jobs..." + kill -15 %1 + kill -15 %2 + } + trap clean_up EXIT SIGINT SIGTERM -# Port forward the UI so tests can work against localhost -POD=`kubectl get pods -n ${NAMESPACE} -l app=ml-pipeline-ui -o jsonpath='{.items[0].metadata.name}'` -kubectl port-forward -n ${NAMESPACE} ${POD} 3000:3000 & + # Port forward the UI so tests can work against localhost + POD=`kubectl get pods -n ${NAMESPACE} -l app=ml-pipeline-ui -o jsonpath='{.items[0].metadata.name}'` + kubectl port-forward -n ${NAMESPACE} ${POD} 3000:3000 & +fi # Run Selenium server /opt/bin/entry_point.sh & @@ -82,7 +101,9 @@ set -e JUNIT_TEST_RESULT=junit_FrontendIntegrationTestOutput.xml -echo "Copy test result to GCS ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT}" -gsutil cp ${JUNIT_TEST_RESULT} ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT} +if [[ "$REMOTE_RUN" != "true" ]]; then + echo "Copy test result to GCS ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT}" + gsutil cp ${JUNIT_TEST_RESULT} ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT} +fi exit $TEST_EXIT_CODE From 6b3fdeab8c9adabe75f19a9b1909dbc64f81c3a7 Mon Sep 17 00:00:00 2001 From: Ablai Akhazhanov Date: Mon, 30 Jan 2023 23:39:04 +0000 Subject: [PATCH 3/4] Save testing artifacts to GCS --- test/deploy-cluster.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/deploy-cluster.sh b/test/deploy-cluster.sh index a9daf87c436..07d427c31a4 100755 --- a/test/deploy-cluster.sh +++ b/test/deploy-cluster.sh @@ -41,6 +41,7 @@ function clean_up { ALL_PODS=($(kubectl get pods -o=custom-columns=:metadata.name -n $NAMESPACE)) for POD_NAME in "${ALL_PODS[@]}"; do pod_info_file="$POD_INFO_DIR/$POD_NAME.txt" + echo "Saving log of $POD_NAME to $pod_info_file" echo "Pod name: $POD_NAME" >> "$pod_info_file" echo "Detailed logs:" >> "$pod_info_file" echo "https://console.cloud.google.com/logs/viewer?project=$PROJECT&advancedFilter=resource.type%3D%22k8s_container%22%0Aresource.labels.project_id%3D%22$PROJECT%22%0Aresource.labels.location%3D%22us-east1-b%22%0Aresource.labels.cluster_name%3D%22${TEST_CLUSTER}%22%0Aresource.labels.namespace_name%3D%22$NAMESPACE%22%0Aresource.labels.pod_name%3D%22$POD_NAME%22" \ @@ -50,6 +51,11 @@ function clean_up { echo "--------" >> "$pod_info_file" kubectl get pod $POD_NAME -n $NAMESPACE -o yaml >> "$pod_info_file" done + + echo "Archiving ${ARTIFACTS} into ./${COMMIT_SHA}_logs.tar.gz" + tar -czf ./${COMMIT_SHA}_logs.tar.gz ${ARTIFACTS} + echo "Uploading ./${COMMIT_SHA}_logs.tar.gz to ${TEST_RESULTS_GCS_DIR}/logs" + gsutil cp ${COMMIT_SHA}_logs.tar.gz "${TEST_RESULTS_GCS_DIR}/logs" echo "Clean up cluster..." if [ $SHOULD_CLEANUP_CLUSTER == true ]; then From 58d114ee2db8458e0880b4c38117ec4583ac784f Mon Sep 17 00:00:00 2001 From: Ablai Akhazhanov Date: Tue, 31 Jan 2023 02:34:28 +0000 Subject: [PATCH 4/4] Address comments --- .../helloworld.spec.js | 43 +++++++------------ 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/test/frontend-integration-test/helloworld.spec.js b/test/frontend-integration-test/helloworld.spec.js index d75d5524ddf..581ac42a9da 100644 --- a/test/frontend-integration-test/helloworld.spec.js +++ b/test/frontend-integration-test/helloworld.spec.js @@ -72,10 +72,6 @@ describe('deploy helloworld sample run', () => { $('#experimentDescription').setValue(experimentDescription); $('#createExperimentBtn').click(); - - browser.waitUntil(() => { - return new URL(browser.getUrl()).hash.startsWith('#/runs/new'); - }, waitTimeout); }); it('creates a new run in the experiment', () => { @@ -98,7 +94,6 @@ describe('deploy helloworld sample run', () => { $('#usePipelineVersionBtn').click(); $('#pipelineVersionSelectorDialog').waitForVisible(waitTimeout, true); - browser.pause(1000); browser.keys(runName); @@ -140,35 +135,31 @@ describe('deploy helloworld sample run', () => { assert(attempts, 'waited for 30 seconds but run did not start.'); assert.equal($$('.tableRow').length, 1, 'should only show one run'); + + // Navigate to details of the deployed run by clicking its anchor element + browser.execute('document.querySelector(".tableRow a").click()'); + }); + + it('switches to config tab', () => { + $('button=Config').waitForVisible(waitTimeout); + $('button=Config').click(); }); - // Wait for a reasonable amount of time until the run is done - // and navigate to the run details page it('waits for run to finish', () => { + let status = getValueFromDetailsTable('Status'); + let attempts = 0; - const maxAttempts = 90; + const maxAttempts = 60; - while (attempts < maxAttempts) { + // Wait for a reasonable amount of time until the run is done + while (attempts < maxAttempts && status.trim() !== 'Succeeded') { browser.pause(1000); + status = getValueFromDetailsTable('Status'); attempts++; } - browser.execute('document.querySelector(".tableRow a").click()'); - browser.waitUntil(() => { - return new URL(browser.getUrl()).hash.startsWith('#/runs/details'); - }, waitTimeout); - }); - - it('switches to config tab', () => { - $('button=Config').waitForVisible(); - $('button=Config').click(); - browser.pause(waitTimeout); - }); - - it('verifies run status', () => { - const status = getValueFromDetailsTable('Status'); - assert.equal(status.trim(), 'Succeeded', - 'run has not finished on time. Current status is: ' + status); + assert(attempts < maxAttempts, `waited for ${maxAttempts} seconds but run did not succeed. ` + + 'Current status is: ' + status); }); it('displays run created at date correctly', () => { @@ -188,7 +179,6 @@ describe('deploy helloworld sample run', () => { it('has a 4-node graph', () => { const nodeSelector = '.graphNode'; - $(nodeSelector).waitForVisible(); const nodes = $$(nodeSelector).length; assert(nodes === 4, 'should have a 4-node graph, instead has: ' + nodes); }); @@ -228,7 +218,6 @@ describe('deploy helloworld sample run', () => { $('#usePipelineBtn').click(); $('#pipelineSelectorDialog').waitForVisible(waitTimeout, true); - browser.pause(1000); browser.keys('Tab'); browser.keys(runWithoutExperimentName);