From cf8ce96cf8194e276da895d35df39315a571e015 Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Tue, 10 Oct 2017 06:15:34 -0700 Subject: [PATCH 01/20] Prow presubmit job to build mlkube.io container. * This job is supposed to build a Docker container with the CRD. Once this works we will add another job to actually run the tests. * We create a new image to be used by this job. * The image needs docker so that we can do a Docker build. --- images/mlkube/Dockerfile | 79 ++++++++++++++++++++++++++++++++++++++++ images/mlkube/Makefile | 28 ++++++++++++++ images/mlkube/runner.py | 48 ++++++++++++++++++++++++ prow/config.yaml | 35 ++++++++++++++++++ 4 files changed, 190 insertions(+) create mode 100644 images/mlkube/Dockerfile create mode 100644 images/mlkube/Makefile create mode 100755 images/mlkube/runner.py diff --git a/images/mlkube/Dockerfile b/images/mlkube/Dockerfile new file mode 100644 index 000000000000..dc29dc2862e3 --- /dev/null +++ b/images/mlkube/Dockerfile @@ -0,0 +1,79 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This Dockerfile is used by ProwJobs that build docker images for the +# TF CRD in (https://github.com/jlewi/mlkube.io) + +FROM golang:1.8.2 +LABEL authors="Jeremy Lewi " + +WORKDIR /workspace +RUN mkdir -p /workspace +ENV WORKSPACE=/workspace \ + TERM=xterm + +# common util tools +# https://github.com/GoogleCloudPlatform/gsutil/issues/446 for python-openssl +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget \ + curl \ + file \ + rsync \ + ca-certificates \ + build-essential \ + openssh-client \ + git \ + pkg-config \ + zip \ + unzip \ + xz-utils \ + zlib1g-dev \ + python \ + python-openssl \ + python-pip \ + && apt-get clean + +# Install gcloud + +ENV PATH=/google-cloud-sdk/bin:/workspace:${PATH} \ + CLOUDSDK_CORE_DISABLE_PROMPTS=1 + +RUN wget -q https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz && \ + tar xzf google-cloud-sdk.tar.gz -C / && \ + rm google-cloud-sdk.tar.gz && \ + /google-cloud-sdk/install.sh \ + --disable-installation-options \ + --bash-completion=false \ + --path-update=false \ + --usage-reporting=false && \ + gcloud components install alpha beta kubectl && \ + gcloud info | tee /workspace/gcloud-info.txt + +# Install docker +# Note: 1.11+ changes the tarball format +RUN curl -L "https://get.docker.com/builds/Linux/x86_64/docker-1.9.1.tgz" \ + | tar -C /usr/bin -xvzf- --strip-components=3 usr/local/bin/docker + + +# Install glide +RUN cd /tmp && \ + wget -O glide-v0.13.0-linux-amd64.tar.gz \ + https://github.com/Masterminds/glide/releases/download/v0.13.0/glide-v0.13.0-linux-amd64.tar.gz && \ + tar -xvf glide-v0.13.0-linux-amd64.tar.gz && \ + mv ./linux-amd64/glide /usr/local/bin/ + +ADD ["runner.py", \ + "/workspace/"] + +ENTRYPOINT ["python", "/workspace/runner.py"] diff --git a/images/mlkube/Makefile b/images/mlkube/Makefile new file mode 100644 index 000000000000..694e903cf912 --- /dev/null +++ b/images/mlkube/Makefile @@ -0,0 +1,28 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +IMG = gcr.io/mlkube-testing/builder +TAG = $(shell date +v%Y%m%d)-$(shell git describe --tags --always --dirty) + +all: build + +build: + docker build -t $(IMG):$(TAG) . + docker tag $(IMG):$(TAG) $(IMG):latest + @echo Built $(IMG):$(TAG) and tagged with latest + +push: build + gcloud docker -- push $(IMG):$(TAG) + gcloud docker -- push $(IMG):latest + @echo Pushed $(IMG) with :latest and :$(TAG) tags diff --git a/images/mlkube/runner.py b/images/mlkube/runner.py new file mode 100755 index 000000000000..8c1f6e3f2b03 --- /dev/null +++ b/images/mlkube/runner.py @@ -0,0 +1,48 @@ +#!/usr/bin/python +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import subprocess +import os +import shutil + +if __name__ == "__main__": + logging.getLogger().setLevel(logging.INFO) + + go_path = os.getenv("GOPATH") + repo_owner = os.getenv("REPO_OWNER") + repo_name = os.getenv("REPO_NAME") + + # Clone mlkube + repo = "https://github.com/{0}/{1}.git".format(repo_owner, repo_name) + src_dir = os.path.join(go_path, "src/github.com/jlewi") + os.makedirs(src_dir) + dest = os.path.join(src_dir, "mlkube.io") + + # TODO(jlewi): How can we figure out what branch + subprocess.check_call(["git", "clone", repo, dest]) + + pull_sha = os.getenv('PULL_PULL_SHA') + if pull_sha: + subprocess.check_call(["git", "checkout", pull_sha]) + + # Install dependencies + subprocess.check_call(["glide", "install"], cwd=dest) + + # We need to remove the vendored dependencies of apiextensions otherwise we get type conflicts. + shutil.rmtree(os.path.join(dest, "vendor/k8s.io/apiextensions-apiserver/vendor")) + + # Build and push the image + subprocess.check_call(["./images/tf_operator/build_and_push.sh",], cwd=dest) \ No newline at end of file diff --git a/prow/config.yaml b/prow/config.yaml index e8ab81ba29f7..b686bb18613c 100644 --- a/prow/config.yaml +++ b/prow/config.yaml @@ -1821,6 +1821,41 @@ presubmits: - name: cache-ssd hostPath: path: /mnt/disks/ssd0 + jlewi/mlkube.io: + - name: mlkube-build + agent: kubernetes + context: mlkube-build + always_run: true + rerun_command: "/test mlkube-build" + trigger: "(?m)^/test( all| mlkube-build),?(\\s+|$)" + spec: + containers: + - image: gcr.io/mlkube-testing/builder:v20171010-352a232c-dirty + # TODO(jlewi): Add a service account to push to GCR? + # args: + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /etc/service-account/service-account.json + # TODO(jlewi): Do we need privileged mode in order to run Docker? + securityContext: + privileged: true + volumeMounts: + - name: service + mountPath: /etc/service-account + readOnly: true + - name: docker + mountPath: /var/run/docker.sock + ports: + # TODO(jlewi): Do we need this port? + - containerPort: 9999 + hostPort: 9999 + volumes: + - name: service + secret: + secretName: service-account + - name: docker + hostPath: + path: /var/run/docker.sock postsubmits: kubernetes/kubernetes: From 01d41a2615ca2bcb68d262c600db0437a15a9916 Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Tue, 10 Oct 2017 16:21:14 -0700 Subject: [PATCH 02/20] Fix starter.yaml * Need to specify namespace, resync_period and some other values or the config won't be parseable. --- prow/cluster/starter.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/prow/cluster/starter.yaml b/prow/cluster/starter.yaml index fce77e03cbfe..e4599011d4a8 100644 --- a/prow/cluster/starter.yaml +++ b/prow/cluster/starter.yaml @@ -15,6 +15,10 @@ metadata: name: config data: config: | + prowjob_namespace: default + pod_namespace: test-pods + log_level: info + periodics: - interval: 10m agent: kubernetes @@ -23,6 +27,10 @@ data: containers: - image: alpine command: ["/bin/date"] + sinker: + resync_period: 1h + max_prowjob_age: 48h + max_pod_age: 1h --- apiVersion: extensions/v1beta1 kind: ThirdPartyResource From 49da9b2b606f5b413f5ebb3fba74c1c433179f40 Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Thu, 12 Oct 2017 05:59:41 -0700 Subject: [PATCH 03/20] * mlkube image moved to mlkube.io repo. * Add pre and postsubmit jobs for mlkube. * Remove the periodic job; I don't think we want that. --- images/mlkube/Dockerfile | 79 ------------------------ images/mlkube/Makefile | 28 --------- images/mlkube/runner.py | 48 --------------- prow/cluster/starter.yaml | 8 --- prow/config.yaml | 123 ++++++++++++++++++-------------------- 5 files changed, 57 insertions(+), 229 deletions(-) delete mode 100644 images/mlkube/Dockerfile delete mode 100644 images/mlkube/Makefile delete mode 100755 images/mlkube/runner.py diff --git a/images/mlkube/Dockerfile b/images/mlkube/Dockerfile deleted file mode 100644 index dc29dc2862e3..000000000000 --- a/images/mlkube/Dockerfile +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This Dockerfile is used by ProwJobs that build docker images for the -# TF CRD in (https://github.com/jlewi/mlkube.io) - -FROM golang:1.8.2 -LABEL authors="Jeremy Lewi " - -WORKDIR /workspace -RUN mkdir -p /workspace -ENV WORKSPACE=/workspace \ - TERM=xterm - -# common util tools -# https://github.com/GoogleCloudPlatform/gsutil/issues/446 for python-openssl -RUN apt-get update && apt-get install -y --no-install-recommends \ - wget \ - curl \ - file \ - rsync \ - ca-certificates \ - build-essential \ - openssh-client \ - git \ - pkg-config \ - zip \ - unzip \ - xz-utils \ - zlib1g-dev \ - python \ - python-openssl \ - python-pip \ - && apt-get clean - -# Install gcloud - -ENV PATH=/google-cloud-sdk/bin:/workspace:${PATH} \ - CLOUDSDK_CORE_DISABLE_PROMPTS=1 - -RUN wget -q https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz && \ - tar xzf google-cloud-sdk.tar.gz -C / && \ - rm google-cloud-sdk.tar.gz && \ - /google-cloud-sdk/install.sh \ - --disable-installation-options \ - --bash-completion=false \ - --path-update=false \ - --usage-reporting=false && \ - gcloud components install alpha beta kubectl && \ - gcloud info | tee /workspace/gcloud-info.txt - -# Install docker -# Note: 1.11+ changes the tarball format -RUN curl -L "https://get.docker.com/builds/Linux/x86_64/docker-1.9.1.tgz" \ - | tar -C /usr/bin -xvzf- --strip-components=3 usr/local/bin/docker - - -# Install glide -RUN cd /tmp && \ - wget -O glide-v0.13.0-linux-amd64.tar.gz \ - https://github.com/Masterminds/glide/releases/download/v0.13.0/glide-v0.13.0-linux-amd64.tar.gz && \ - tar -xvf glide-v0.13.0-linux-amd64.tar.gz && \ - mv ./linux-amd64/glide /usr/local/bin/ - -ADD ["runner.py", \ - "/workspace/"] - -ENTRYPOINT ["python", "/workspace/runner.py"] diff --git a/images/mlkube/Makefile b/images/mlkube/Makefile deleted file mode 100644 index 694e903cf912..000000000000 --- a/images/mlkube/Makefile +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -IMG = gcr.io/mlkube-testing/builder -TAG = $(shell date +v%Y%m%d)-$(shell git describe --tags --always --dirty) - -all: build - -build: - docker build -t $(IMG):$(TAG) . - docker tag $(IMG):$(TAG) $(IMG):latest - @echo Built $(IMG):$(TAG) and tagged with latest - -push: build - gcloud docker -- push $(IMG):$(TAG) - gcloud docker -- push $(IMG):latest - @echo Pushed $(IMG) with :latest and :$(TAG) tags diff --git a/images/mlkube/runner.py b/images/mlkube/runner.py deleted file mode 100755 index 8c1f6e3f2b03..000000000000 --- a/images/mlkube/runner.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/python -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import subprocess -import os -import shutil - -if __name__ == "__main__": - logging.getLogger().setLevel(logging.INFO) - - go_path = os.getenv("GOPATH") - repo_owner = os.getenv("REPO_OWNER") - repo_name = os.getenv("REPO_NAME") - - # Clone mlkube - repo = "https://github.com/{0}/{1}.git".format(repo_owner, repo_name) - src_dir = os.path.join(go_path, "src/github.com/jlewi") - os.makedirs(src_dir) - dest = os.path.join(src_dir, "mlkube.io") - - # TODO(jlewi): How can we figure out what branch - subprocess.check_call(["git", "clone", repo, dest]) - - pull_sha = os.getenv('PULL_PULL_SHA') - if pull_sha: - subprocess.check_call(["git", "checkout", pull_sha]) - - # Install dependencies - subprocess.check_call(["glide", "install"], cwd=dest) - - # We need to remove the vendored dependencies of apiextensions otherwise we get type conflicts. - shutil.rmtree(os.path.join(dest, "vendor/k8s.io/apiextensions-apiserver/vendor")) - - # Build and push the image - subprocess.check_call(["./images/tf_operator/build_and_push.sh",], cwd=dest) \ No newline at end of file diff --git a/prow/cluster/starter.yaml b/prow/cluster/starter.yaml index e4599011d4a8..fce77e03cbfe 100644 --- a/prow/cluster/starter.yaml +++ b/prow/cluster/starter.yaml @@ -15,10 +15,6 @@ metadata: name: config data: config: | - prowjob_namespace: default - pod_namespace: test-pods - log_level: info - periodics: - interval: 10m agent: kubernetes @@ -27,10 +23,6 @@ data: containers: - image: alpine command: ["/bin/date"] - sinker: - resync_period: 1h - max_prowjob_age: 48h - max_pod_age: 1h --- apiVersion: extensions/v1beta1 kind: ThirdPartyResource diff --git a/prow/config.yaml b/prow/config.yaml index b686bb18613c..0e99e9b391dc 100644 --- a/prow/config.yaml +++ b/prow/config.yaml @@ -1821,41 +1821,38 @@ presubmits: - name: cache-ssd hostPath: path: /mnt/disks/ssd0 + jlewi/mlkube.io: - - name: mlkube-build - agent: kubernetes - context: mlkube-build - always_run: true - rerun_command: "/test mlkube-build" - trigger: "(?m)^/test( all| mlkube-build),?(\\s+|$)" - spec: - containers: - - image: gcr.io/mlkube-testing/builder:v20171010-352a232c-dirty - # TODO(jlewi): Add a service account to push to GCR? - # args: - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /etc/service-account/service-account.json - # TODO(jlewi): Do we need privileged mode in order to run Docker? - securityContext: - privileged: true - volumeMounts: - - name: service - mountPath: /etc/service-account - readOnly: true - - name: docker - mountPath: /var/run/docker.sock - ports: - # TODO(jlewi): Do we need this port? - - containerPort: 9999 - hostPort: 9999 - volumes: + - name: mlkube-build-presubmit + agent: kubernetes + always_run: true # Run for every PR, or only when requested. + rerun_command: "/test mlkube-build" + trigger: "(?m)^/test( all| mlkube-build),?(\\s+|$)" + branches: + - master + spec: + containers: + # TODO(jlewi): Replace latest with a specific tag once the images stabilize. + - image: gcr.io/mlkube-testing/builder:latest + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /etc/service-account/service-account.json + # We need privileged mode to run Docker. + securityContext: + privileged: true + volumeMounts: - name: service - secret: - secretName: service-account + mountPath: /etc/service-account + readOnly: true - name: docker - hostPath: - path: /var/run/docker.sock + mountPath: /var/run/docker.sock + volumes: + - name: service + secret: + secretName: service-account + - name: docker + hostPath: + path: /var/run/docker.sock postsubmits: kubernetes/kubernetes: @@ -2545,6 +2542,34 @@ postsubmits: secret: secretName: service-account + jlewi/mlkube.io: + - name: mlkube-build-postsubmit + agent: kubernetes + branches: + - master + spec: + containers: + # TODO(jlewi): Replace latest with a specific tag once the images stabilize. + - image: gcr.io/mlkube-testing/builder:latest + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /etc/service-account/service-account.json + # We need privileged mode to run Docker. + securityContext: + privileged: true + volumeMounts: + - name: service + mountPath: /etc/service-account + readOnly: true + - name: docker + mountPath: /var/run/docker.sock + volumes: + - name: service + secret: + secretName: service-account + - name: docker + hostPath: + path: /var/run/docker.sock periodics: - interval: 1h @@ -14982,40 +15007,6 @@ periodics: defaultMode: 256 secretName: aws-cred -- interval: 30m - agent: kubernetes - name: ci-kubernetes-e2e-mlkube-gke - spec: - containers: - - args: - - --timeout=70 - - --repo=github.com/foxish/mlkube.io - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /etc/service-account/service-account.json - - name: USER - value: prow - - name: JENKINS_GCE_SSH_PRIVATE_KEY_FILE - value: /etc/ssh-key-secret/ssh-private - - name: JENKINS_GCE_SSH_PUBLIC_KEY_FILE - value: /etc/ssh-key-secret/ssh-public - image: gcr.io/k8s-testimages/kubekins-e2e:v20171006-bfbd469b-master - volumeMounts: - - mountPath: /etc/service-account - name: service - readOnly: true - - mountPath: /etc/ssh-key-secret - name: ssh - readOnly: true - volumes: - - name: service - secret: - secretName: service-account - - name: ssh - secret: - defaultMode: 256 - secretName: ssh-key-secret - - name: ci-kubernetes-e2e-node-canary interval: 1h agent: kubernetes From 39f70087bb9cb6eb8bc4a43ca2a1e919ea33cdde Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Thu, 12 Oct 2017 11:11:18 -0700 Subject: [PATCH 04/20] Restore the mlkube periodic job. * We probably want this and the testgrid requires it. --- prow/config.yaml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/prow/config.yaml b/prow/config.yaml index a952e529f2c0..3af234127bb7 100644 --- a/prow/config.yaml +++ b/prow/config.yaml @@ -15776,6 +15776,40 @@ periodics: defaultMode: 256 secretName: aws-cred +- interval: 30m + agent: kubernetes + name: ci-kubernetes-e2e-mlkube-gke + spec: + containers: + - args: + - --timeout=70 + - --repo=github.com/foxish/mlkube.io + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /etc/service-account/service-account.json + - name: USER + value: prow + - name: JENKINS_GCE_SSH_PRIVATE_KEY_FILE + value: /etc/ssh-key-secret/ssh-private + - name: JENKINS_GCE_SSH_PUBLIC_KEY_FILE + value: /etc/ssh-key-secret/ssh-public + image: gcr.io/k8s-testimages/kubekins-e2e:v20171011-8265ed12-master + volumeMounts: + - mountPath: /etc/service-account + name: service + readOnly: true + - mountPath: /etc/ssh-key-secret + name: ssh + readOnly: true + volumes: + - name: service + secret: + secretName: service-account + - name: ssh + secret: + defaultMode: 256 + secretName: ssh-key-secret + - name: ci-kubernetes-e2e-node-canary interval: 1h agent: kubernetes From d212e7b8e3ca33700cf8664fbbf85a038d3cdf12 Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Thu, 12 Oct 2017 11:26:21 -0700 Subject: [PATCH 05/20] Sort the configs. --- prow/config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/prow/config.yaml b/prow/config.yaml index 3af234127bb7..90d8d0ce95d0 100644 --- a/prow/config.yaml +++ b/prow/config.yaml @@ -2518,8 +2518,8 @@ presubmits: # TODO(jlewi): Replace latest with a specific tag once the images stabilize. - image: gcr.io/mlkube-testing/builder:latest env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /etc/service-account/service-account.json + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /etc/service-account/service-account.json # We need privileged mode to run Docker. securityContext: privileged: true @@ -3321,8 +3321,8 @@ postsubmits: # TODO(jlewi): Replace latest with a specific tag once the images stabilize. - image: gcr.io/mlkube-testing/builder:latest env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /etc/service-account/service-account.json + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /etc/service-account/service-account.json # We need privileged mode to run Docker. securityContext: privileged: true From d8ba6c9c26221e5df0e9246f4a238b57ecdaa719 Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Thu, 12 Oct 2017 11:36:53 -0700 Subject: [PATCH 06/20] Add a context to the presubmit job. --- prow/config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/prow/config.yaml b/prow/config.yaml index 90d8d0ce95d0..509904fe26b2 100644 --- a/prow/config.yaml +++ b/prow/config.yaml @@ -2507,6 +2507,7 @@ presubmits: jlewi/mlkube.io: - name: mlkube-build-presubmit + context: mlkube-build-presubmit agent: kubernetes always_run: true # Run for every PR, or only when requested. rerun_command: "/test mlkube-build" From b0d10e5c59854c0fc668a50246897e021e78b73f Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Thu, 12 Oct 2017 11:53:18 -0700 Subject: [PATCH 07/20] Fix the //prow/config:go_default_test --- jobs/config.json | 21 +++++++++++++++++++++ prow/config.yaml | 4 ++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/jobs/config.json b/jobs/config.json index f6733e07091d..aa2d8c005e83 100644 --- a/jobs/config.json +++ b/jobs/config.json @@ -8089,6 +8089,27 @@ "sig-big-data" ] }, + "mlkube-build-presubmit": { + "args": [ + "--charts", + "--check-leaked-resources", + "--deployment=gke", + "--extract=gke", + "--gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging", + "--gcp-node-image=gci", + "--gcp-project=mlkube-testing", + "--gcp-zone=us-central1-b", + "--gke-environment=prod", + "--mount-paths=$GOPATH/src/github.com/foxish/mlkube.io:/src/k8s.io/charts", + "--provider=gke", + "--test=false", + "--timeout=50m" + ], + "scenario": "kubernetes_e2e", + "sigOwners": [ + "sig-big-data" + ] + }, "ci-kubernetes-e2e-node-canary": { "args": [ "--deployment=node", diff --git a/prow/config.yaml b/prow/config.yaml index 509904fe26b2..2c613b3e06d1 100644 --- a/prow/config.yaml +++ b/prow/config.yaml @@ -2510,8 +2510,8 @@ presubmits: context: mlkube-build-presubmit agent: kubernetes always_run: true # Run for every PR, or only when requested. - rerun_command: "/test mlkube-build" - trigger: "(?m)^/test( all| mlkube-build),?(\\s+|$)" + rerun_command: "/test mlkube-build-presubmit" + trigger: "(?m)^/test( all| mlkube-build-presubmit),?(\\s+|$)" branches: - master spec: From 6a4bcec5ec1f1111ff2c6936f27983b7c24012f1 Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Thu, 12 Oct 2017 12:00:10 -0700 Subject: [PATCH 08/20] Sort the configs. --- jobs/config.json | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/jobs/config.json b/jobs/config.json index aa2d8c005e83..2ea28c808efe 100644 --- a/jobs/config.json +++ b/jobs/config.json @@ -8089,27 +8089,6 @@ "sig-big-data" ] }, - "mlkube-build-presubmit": { - "args": [ - "--charts", - "--check-leaked-resources", - "--deployment=gke", - "--extract=gke", - "--gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging", - "--gcp-node-image=gci", - "--gcp-project=mlkube-testing", - "--gcp-zone=us-central1-b", - "--gke-environment=prod", - "--mount-paths=$GOPATH/src/github.com/foxish/mlkube.io:/src/k8s.io/charts", - "--provider=gke", - "--test=false", - "--timeout=50m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-big-data" - ] - }, "ci-kubernetes-e2e-node-canary": { "args": [ "--deployment=node", @@ -9928,6 +9907,27 @@ "UNKNOWN" ] }, + "mlkube-build-presubmit": { + "args": [ + "--charts", + "--check-leaked-resources", + "--deployment=gke", + "--extract=gke", + "--gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging", + "--gcp-node-image=gci", + "--gcp-project=mlkube-testing", + "--gcp-zone=us-central1-b", + "--gke-environment=prod", + "--mount-paths=$GOPATH/src/github.com/foxish/mlkube.io:/src/k8s.io/charts", + "--provider=gke", + "--test=false", + "--timeout=50m" + ], + "scenario": "kubernetes_e2e", + "sigOwners": [ + "sig-big-data" + ] + }, "periodic-kubernetes-bazel-build-1-6": { "args": [ "--build=//cmd/... //pkg/... //federation/... //plugin/... //third_party/... //examples/... //test/...", From 789904796fdb9406570d50168d6b95db998f80a4 Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Thu, 12 Oct 2017 15:48:52 -0700 Subject: [PATCH 09/20] Fix the test. * Need to add to allowed list so that we can use the same project for multiple tests. --- jobs/config_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/jobs/config_test.py b/jobs/config_test.py index c98e58702615..96079943450b 100755 --- a/jobs/config_test.py +++ b/jobs/config_test.py @@ -789,6 +789,13 @@ def test_all_project_are_unique(self): 'ci-kubernetes-node-kubelet-conformance': 'ci-kubernetes-node-kubelet-*', 'ci-kubernetes-node-kubelet-benchmark': 'ci-kubernetes-node-kubelet-*', 'ci-kubernetes-node-kubelet': 'ci-kubernetes-node-kubelet-*', + + # The mlkube projects intentionally share projects, + # We map each test name to the same value so that below when we check for number of tests + # using this project there will just be one entry. + 'mlkube-build-presubmit': 'mlkube-*', + 'mlkube-build-postsubmit': 'mlkube-*', + 'ci-kubernetes-e2e-mlkube-gke': 'mlkube-*', } for soak_prefix in [ 'ci-kubernetes-soak-gce-1.5', From 98dfc265a17f8fb144571c9641a47dbc5df9a562 Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Thu, 12 Oct 2017 18:18:32 -0700 Subject: [PATCH 10/20] * We don't need jobs/config.json entries; this is legacy Jenkins stuff. * Update jobs_test.go; for ProwJobs there's no need for a .sh script since that functionality is specified via the PodSpec which specifies what container and command to run. --- jobs/config.json | 21 --------------------- prow/config/jobs_test.go | 2 -- 2 files changed, 23 deletions(-) diff --git a/jobs/config.json b/jobs/config.json index 2ea28c808efe..f6733e07091d 100644 --- a/jobs/config.json +++ b/jobs/config.json @@ -9907,27 +9907,6 @@ "UNKNOWN" ] }, - "mlkube-build-presubmit": { - "args": [ - "--charts", - "--check-leaked-resources", - "--deployment=gke", - "--extract=gke", - "--gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging", - "--gcp-node-image=gci", - "--gcp-project=mlkube-testing", - "--gcp-zone=us-central1-b", - "--gke-environment=prod", - "--mount-paths=$GOPATH/src/github.com/foxish/mlkube.io:/src/k8s.io/charts", - "--provider=gke", - "--test=false", - "--timeout=50m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-big-data" - ] - }, "periodic-kubernetes-bazel-build-1-6": { "args": [ "--build=//cmd/... //pkg/... //federation/... //plugin/... //third_party/... //examples/... //test/...", diff --git a/prow/config/jobs_test.go b/prow/config/jobs_test.go index 5443e8c1cdcf..4252e31dfe8e 100644 --- a/prow/config/jobs_test.go +++ b/prow/config/jobs_test.go @@ -143,8 +143,6 @@ func TestPresubmits(t *testing.T) { job.Name = strings.Replace(job.Name, "pull-security-kubernetes", "pull-kubernetes", 1) if j, present := bootstrapConfig[job.Name]; present { scenario = fmt.Sprintf("scenarios/%s.py", j.Scenario) - } else { - scenario = fmt.Sprintf("jobs/%s.sh", job.Name) } // Ensure that jobs have a shell script of the same name. From 8563b0c1fa76dfc890d05f38e6b9bb2faa5f76e6 Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Thu, 12 Oct 2017 20:02:33 -0700 Subject: [PATCH 11/20] Add a test-grid entry. --- testgrid/config/config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/testgrid/config/config.yaml b/testgrid/config/config.yaml index a90040f6951b..112781f8d0c0 100644 --- a/testgrid/config/config.yaml +++ b/testgrid/config/config.yaml @@ -1537,6 +1537,10 @@ test_groups: gcs_prefix: kubernetes-jenkins/logs/ci-kubernetes-e2e-gke-gci-new-gci-master-upgrade-cluster-new - name: ci-kubernetes-e2e-mlkube-gke gcs_prefix: kubernetes-jenkins/logs/ci-kubernetes-e2e-mlkube-gke +- name: mlkube-build-presubmit + gcs_prefix: kubernetes-jenkins/logs/mlkube-build-presubmit +- name: mlkube-build-postsubmit + gcs_prefix: kubernetes-jenkins/logs/mlkube-build-postsubmit # kube-proxy daemonset migration jobs - name: ci-kubernetes-e2e-gci-gce-latest-downgrade-kube-proxy-ds gcs_prefix: kubernetes-jenkins/logs/ci-kubernetes-e2e-gci-gce-latest-downgrade-kube-proxy-ds From 4309c6dfdf7f12b399e91ad9b62bf56816bc6ecf Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Thu, 12 Oct 2017 21:51:08 -0700 Subject: [PATCH 12/20] * Don't mount the docker socket or run in privileged mode. * We will rely on GCB to build containers. --- prow/config.yaml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/prow/config.yaml b/prow/config.yaml index 2c613b3e06d1..a727882a4140 100644 --- a/prow/config.yaml +++ b/prow/config.yaml @@ -2521,22 +2521,14 @@ presubmits: env: - name: GOOGLE_APPLICATION_CREDENTIALS value: /etc/service-account/service-account.json - # We need privileged mode to run Docker. - securityContext: - privileged: true volumeMounts: - name: service mountPath: /etc/service-account readOnly: true - - name: docker - mountPath: /var/run/docker.sock volumes: - name: service secret: secretName: service-account - - name: docker - hostPath: - path: /var/run/docker.sock postsubmits: kubernetes/kubernetes: @@ -3324,15 +3316,10 @@ postsubmits: env: - name: GOOGLE_APPLICATION_CREDENTIALS value: /etc/service-account/service-account.json - # We need privileged mode to run Docker. - securityContext: - privileged: true volumeMounts: - name: service mountPath: /etc/service-account readOnly: true - - name: docker - mountPath: /var/run/docker.sock volumes: - name: service secret: From 7507edb7dd83b96f6ee55f8f7e9a0641e8ceb9ce Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Fri, 13 Oct 2017 07:05:40 -0700 Subject: [PATCH 13/20] * Try to fix failure Testgrid group mlkube-build-presubmit does not have a matching jenkins or prow job by considering all presubmit jobs; not just presubmit jobs for repo: kubernetes/kubernetes. * The change to only consider kubernetes/kubernetes was added in https://github.com/kubernetes/test-infra/commit/108fad09f77c3f80c7a1a9357723b290f3b50bc0 * Not sure why we restrict to Kubernetes/Kubernetes for presubmits but not postsubmits or periodic jobs. --- testgrid/jenkins_verify/jenkins_validate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgrid/jenkins_verify/jenkins_validate.go b/testgrid/jenkins_verify/jenkins_validate.go index 72f3cccb524f..eb5223107de0 100644 --- a/testgrid/jenkins_verify/jenkins_validate.go +++ b/testgrid/jenkins_verify/jenkins_validate.go @@ -76,7 +76,7 @@ func main() { } // Also check k/k presubmit, prow postsubmit and periodic jobs - for _, job := range prowConfig.AllPresubmits([]string{"kubernetes/kubernetes"}) { + for _, job := range prowConfig.AllPresubmits([]string{}) { jobs[job.Name] = false } From 988f6572fd5f9b9a2e4e8c785b7a39afa4f3609e Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Fri, 13 Oct 2017 07:24:04 -0700 Subject: [PATCH 14/20] Revert "* Try to fix failure" This reverts commit 7507edb7dd83b96f6ee55f8f7e9a0641e8ceb9ce. If we load all presubmits we end up with failures due to other presubmits with no testgroup. --- testgrid/jenkins_verify/jenkins_validate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgrid/jenkins_verify/jenkins_validate.go b/testgrid/jenkins_verify/jenkins_validate.go index eb5223107de0..72f3cccb524f 100644 --- a/testgrid/jenkins_verify/jenkins_validate.go +++ b/testgrid/jenkins_verify/jenkins_validate.go @@ -76,7 +76,7 @@ func main() { } // Also check k/k presubmit, prow postsubmit and periodic jobs - for _, job := range prowConfig.AllPresubmits([]string{}) { + for _, job := range prowConfig.AllPresubmits([]string{"kubernetes/kubernetes"}) { jobs[job.Name] = false } From 5d6a304acfe69c3955b5982e0c31a094aa6f9866 Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Fri, 13 Oct 2017 07:25:19 -0700 Subject: [PATCH 15/20] Include presubmits from jlewi/mlkube.io --- testgrid/jenkins_verify/jenkins_validate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgrid/jenkins_verify/jenkins_validate.go b/testgrid/jenkins_verify/jenkins_validate.go index 72f3cccb524f..3c50d003a88b 100644 --- a/testgrid/jenkins_verify/jenkins_validate.go +++ b/testgrid/jenkins_verify/jenkins_validate.go @@ -76,7 +76,7 @@ func main() { } // Also check k/k presubmit, prow postsubmit and periodic jobs - for _, job := range prowConfig.AllPresubmits([]string{"kubernetes/kubernetes"}) { + for _, job := range prowConfig.AllPresubmits([]string{"jlewi/mlkube.io", "kubernetes/kubernetes",}) { jobs[job.Name] = false } From 00909832f0f27ce42e529784830bd4a81b536726 Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Fri, 13 Oct 2017 10:10:04 -0700 Subject: [PATCH 16/20] gofmt. --- testgrid/jenkins_verify/jenkins_validate.go | 246 ++++++++++---------- 1 file changed, 123 insertions(+), 123 deletions(-) diff --git a/testgrid/jenkins_verify/jenkins_validate.go b/testgrid/jenkins_verify/jenkins_validate.go index 3c50d003a88b..6650a7703f3b 100644 --- a/testgrid/jenkins_verify/jenkins_validate.go +++ b/testgrid/jenkins_verify/jenkins_validate.go @@ -17,130 +17,130 @@ limitations under the License. package main import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" - - prow_config "k8s.io/test-infra/prow/config" - "k8s.io/test-infra/testgrid/config/yaml2proto" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + prow_config "k8s.io/test-infra/prow/config" + "k8s.io/test-infra/testgrid/config/yaml2proto" ) func main() { - args := os.Args[1:] - - if len(args) != 3 { - fmt.Println("Missing args - usage: go run jenkins_validate.go ") - os.Exit(1) - } - - jobPath := args[0] - prowPath := args[1] - configPath := args[2] - - jobs := make(map[string]bool) - files, err := filepath.Glob(jobPath + "/*") - if err != nil { - fmt.Println("Failed to collect outputs.") - os.Exit(1) - } - - for _, file := range files { - file = strings.TrimPrefix(file, jobPath+"/") - jobs[file] = false - } - - data, err := ioutil.ReadFile(configPath) - if err != nil { - fmt.Printf("Failed reading %v\n", configPath) - os.Exit(1) - } - - c := yaml2proto.Config{} - if err := c.Update(data); err != nil { - fmt.Printf("Failed to convert yaml to protobuf: %v\n", err) - os.Exit(1) - } - - config, err := c.Raw() - if err != nil { - fmt.Printf("Error validating config: %v\n", err) - os.Exit(1) - } - - prowConfig, err := prow_config.Load(prowPath + "/config.yaml") - if err != nil { - fmt.Printf("Could not load prow configs: %v\n", err) - os.Exit(1) - } - - // Also check k/k presubmit, prow postsubmit and periodic jobs - for _, job := range prowConfig.AllPresubmits([]string{"jlewi/mlkube.io", "kubernetes/kubernetes",}) { - jobs[job.Name] = false - } - - for _, job := range prowConfig.AllPostsubmits([]string{}) { - if job.Agent != "jenkins" { - jobs[job.Name] = false - } - } - - for _, job := range prowConfig.AllPeriodics() { - if job.Agent != "jenkins" { - jobs[job.Name] = false - } - } - - // For now anything outsite k8s-jenkins/(pr-)logs are considered to be fine - testgroups := make(map[string]bool) - for _, testgroup := range config.TestGroups { - if strings.Contains(testgroup.GcsPrefix, "kubernetes-jenkins/logs/") { - job := strings.TrimPrefix(testgroup.GcsPrefix, "kubernetes-jenkins/logs/") - testgroups[job] = false - } - - if strings.Contains(testgroup.GcsPrefix, "kubernetes-jenkins/pr-logs/directory/") { - job := strings.TrimPrefix(testgroup.GcsPrefix, "kubernetes-jenkins/pr-logs/directory/") - testgroups[job] = false - } - } - - // Cross check - // -- Each job need to have a match testgrid group - for job := range jobs { - if _, ok := testgroups[job]; ok { - testgroups[job] = true - jobs[job] = true - } - } - - // Conclusion - badjobs := []string{} - for job, valid := range jobs { - if !valid { - badjobs = append(badjobs, job) - fmt.Printf("Job %v does not have a matching testgrid testgroup\n", job) - } - } - - badconfigs := []string{} - for testgroup, valid := range testgroups { - if !valid { - badconfigs = append(badconfigs, testgroup) - fmt.Printf("Testgrid group %v does not have a matching jenkins or prow job\n", testgroup) - } - } - - if len(badconfigs) > 0 { - fmt.Printf("Total bad config(s) - %v\n", len(badconfigs)) - } - - if len(badjobs) > 0 { - fmt.Printf("Total bad job(s) - %v\n", len(badjobs)) - } - - if len(badconfigs) > 0 || len(badjobs) > 0 { - os.Exit(1) - } + args := os.Args[1:] + + if len(args) != 3 { + fmt.Println("Missing args - usage: go run jenkins_validate.go ") + os.Exit(1) + } + + jobPath := args[0] + prowPath := args[1] + configPath := args[2] + + jobs := make(map[string]bool) + files, err := filepath.Glob(jobPath + "/*") + if err != nil { + fmt.Println("Failed to collect outputs.") + os.Exit(1) + } + + for _, file := range files { + file = strings.TrimPrefix(file, jobPath+"/") + jobs[file] = false + } + + data, err := ioutil.ReadFile(configPath) + if err != nil { + fmt.Printf("Failed reading %v\n", configPath) + os.Exit(1) + } + + c := yaml2proto.Config{} + if err := c.Update(data); err != nil { + fmt.Printf("Failed to convert yaml to protobuf: %v\n", err) + os.Exit(1) + } + + config, err := c.Raw() + if err != nil { + fmt.Printf("Error validating config: %v\n", err) + os.Exit(1) + } + + prowConfig, err := prow_config.Load(prowPath + "/config.yaml") + if err != nil { + fmt.Printf("Could not load prow configs: %v\n", err) + os.Exit(1) + } + + // Also check k/k presubmit, prow postsubmit and periodic jobs + for _, job := range prowConfig.AllPresubmits([]string{"jlewi/mlkube.io", "kubernetes/kubernetes",}) { + jobs[job.Name] = false + } + + for _, job := range prowConfig.AllPostsubmits([]string{}) { + if job.Agent != "jenkins" { + jobs[job.Name] = false + } + } + + for _, job := range prowConfig.AllPeriodics() { + if job.Agent != "jenkins" { + jobs[job.Name] = false + } + } + + // For now anything outsite k8s-jenkins/(pr-)logs are considered to be fine + testgroups := make(map[string]bool) + for _, testgroup := range config.TestGroups { + if strings.Contains(testgroup.GcsPrefix, "kubernetes-jenkins/logs/") { + job := strings.TrimPrefix(testgroup.GcsPrefix, "kubernetes-jenkins/logs/") + testgroups[job] = false + } + + if strings.Contains(testgroup.GcsPrefix, "kubernetes-jenkins/pr-logs/directory/") { + job := strings.TrimPrefix(testgroup.GcsPrefix, "kubernetes-jenkins/pr-logs/directory/") + testgroups[job] = false + } + } + + // Cross check + // -- Each job need to have a match testgrid group + for job := range jobs { + if _, ok := testgroups[job]; ok { + testgroups[job] = true + jobs[job] = true + } + } + + // Conclusion + badjobs := []string{} + for job, valid := range jobs { + if !valid { + badjobs = append(badjobs, job) + fmt.Printf("Job %v does not have a matching testgrid testgroup\n", job) + } + } + + badconfigs := []string{} + for testgroup, valid := range testgroups { + if !valid { + badconfigs = append(badconfigs, testgroup) + fmt.Printf("Testgrid group %v does not have a matching jenkins or prow job\n", testgroup) + } + } + + if len(badconfigs) > 0 { + fmt.Printf("Total bad config(s) - %v\n", len(badconfigs)) + } + + if len(badjobs) > 0 { + fmt.Printf("Total bad job(s) - %v\n", len(badjobs)) + } + + if len(badconfigs) > 0 || len(badjobs) > 0 { + os.Exit(1) + } } From 23987bf73bf9bc604d067e79c70034acfb127665 Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Fri, 13 Oct 2017 10:22:44 -0700 Subject: [PATCH 17/20] gofmt. --- testgrid/jenkins_verify/jenkins_validate.go | 246 ++++++++++---------- 1 file changed, 123 insertions(+), 123 deletions(-) diff --git a/testgrid/jenkins_verify/jenkins_validate.go b/testgrid/jenkins_verify/jenkins_validate.go index 6650a7703f3b..392a4ccb8365 100644 --- a/testgrid/jenkins_verify/jenkins_validate.go +++ b/testgrid/jenkins_verify/jenkins_validate.go @@ -17,130 +17,130 @@ limitations under the License. package main import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" - - prow_config "k8s.io/test-infra/prow/config" - "k8s.io/test-infra/testgrid/config/yaml2proto" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + prow_config "k8s.io/test-infra/prow/config" + "k8s.io/test-infra/testgrid/config/yaml2proto" ) func main() { - args := os.Args[1:] - - if len(args) != 3 { - fmt.Println("Missing args - usage: go run jenkins_validate.go ") - os.Exit(1) - } - - jobPath := args[0] - prowPath := args[1] - configPath := args[2] - - jobs := make(map[string]bool) - files, err := filepath.Glob(jobPath + "/*") - if err != nil { - fmt.Println("Failed to collect outputs.") - os.Exit(1) - } - - for _, file := range files { - file = strings.TrimPrefix(file, jobPath+"/") - jobs[file] = false - } - - data, err := ioutil.ReadFile(configPath) - if err != nil { - fmt.Printf("Failed reading %v\n", configPath) - os.Exit(1) - } - - c := yaml2proto.Config{} - if err := c.Update(data); err != nil { - fmt.Printf("Failed to convert yaml to protobuf: %v\n", err) - os.Exit(1) - } - - config, err := c.Raw() - if err != nil { - fmt.Printf("Error validating config: %v\n", err) - os.Exit(1) - } - - prowConfig, err := prow_config.Load(prowPath + "/config.yaml") - if err != nil { - fmt.Printf("Could not load prow configs: %v\n", err) - os.Exit(1) - } - - // Also check k/k presubmit, prow postsubmit and periodic jobs - for _, job := range prowConfig.AllPresubmits([]string{"jlewi/mlkube.io", "kubernetes/kubernetes",}) { - jobs[job.Name] = false - } - - for _, job := range prowConfig.AllPostsubmits([]string{}) { - if job.Agent != "jenkins" { - jobs[job.Name] = false - } - } - - for _, job := range prowConfig.AllPeriodics() { - if job.Agent != "jenkins" { - jobs[job.Name] = false - } - } - - // For now anything outsite k8s-jenkins/(pr-)logs are considered to be fine - testgroups := make(map[string]bool) - for _, testgroup := range config.TestGroups { - if strings.Contains(testgroup.GcsPrefix, "kubernetes-jenkins/logs/") { - job := strings.TrimPrefix(testgroup.GcsPrefix, "kubernetes-jenkins/logs/") - testgroups[job] = false - } - - if strings.Contains(testgroup.GcsPrefix, "kubernetes-jenkins/pr-logs/directory/") { - job := strings.TrimPrefix(testgroup.GcsPrefix, "kubernetes-jenkins/pr-logs/directory/") - testgroups[job] = false - } - } - - // Cross check - // -- Each job need to have a match testgrid group - for job := range jobs { - if _, ok := testgroups[job]; ok { - testgroups[job] = true - jobs[job] = true - } - } - - // Conclusion - badjobs := []string{} - for job, valid := range jobs { - if !valid { - badjobs = append(badjobs, job) - fmt.Printf("Job %v does not have a matching testgrid testgroup\n", job) - } - } - - badconfigs := []string{} - for testgroup, valid := range testgroups { - if !valid { - badconfigs = append(badconfigs, testgroup) - fmt.Printf("Testgrid group %v does not have a matching jenkins or prow job\n", testgroup) - } - } - - if len(badconfigs) > 0 { - fmt.Printf("Total bad config(s) - %v\n", len(badconfigs)) - } - - if len(badjobs) > 0 { - fmt.Printf("Total bad job(s) - %v\n", len(badjobs)) - } - - if len(badconfigs) > 0 || len(badjobs) > 0 { - os.Exit(1) - } + args := os.Args[1:] + + if len(args) != 3 { + fmt.Println("Missing args - usage: go run jenkins_validate.go ") + os.Exit(1) + } + + jobPath := args[0] + prowPath := args[1] + configPath := args[2] + + jobs := make(map[string]bool) + files, err := filepath.Glob(jobPath + "/*") + if err != nil { + fmt.Println("Failed to collect outputs.") + os.Exit(1) + } + + for _, file := range files { + file = strings.TrimPrefix(file, jobPath+"/") + jobs[file] = false + } + + data, err := ioutil.ReadFile(configPath) + if err != nil { + fmt.Printf("Failed reading %v\n", configPath) + os.Exit(1) + } + + c := yaml2proto.Config{} + if err := c.Update(data); err != nil { + fmt.Printf("Failed to convert yaml to protobuf: %v\n", err) + os.Exit(1) + } + + config, err := c.Raw() + if err != nil { + fmt.Printf("Error validating config: %v\n", err) + os.Exit(1) + } + + prowConfig, err := prow_config.Load(prowPath + "/config.yaml") + if err != nil { + fmt.Printf("Could not load prow configs: %v\n", err) + os.Exit(1) + } + + // Also check k/k presubmit, prow postsubmit and periodic jobs + for _, job := range prowConfig.AllPresubmits([]string{"jlewi/mlkube.io", "kubernetes/kubernetes"}) { + jobs[job.Name] = false + } + + for _, job := range prowConfig.AllPostsubmits([]string{}) { + if job.Agent != "jenkins" { + jobs[job.Name] = false + } + } + + for _, job := range prowConfig.AllPeriodics() { + if job.Agent != "jenkins" { + jobs[job.Name] = false + } + } + + // For now anything outsite k8s-jenkins/(pr-)logs are considered to be fine + testgroups := make(map[string]bool) + for _, testgroup := range config.TestGroups { + if strings.Contains(testgroup.GcsPrefix, "kubernetes-jenkins/logs/") { + job := strings.TrimPrefix(testgroup.GcsPrefix, "kubernetes-jenkins/logs/") + testgroups[job] = false + } + + if strings.Contains(testgroup.GcsPrefix, "kubernetes-jenkins/pr-logs/directory/") { + job := strings.TrimPrefix(testgroup.GcsPrefix, "kubernetes-jenkins/pr-logs/directory/") + testgroups[job] = false + } + } + + // Cross check + // -- Each job need to have a match testgrid group + for job := range jobs { + if _, ok := testgroups[job]; ok { + testgroups[job] = true + jobs[job] = true + } + } + + // Conclusion + badjobs := []string{} + for job, valid := range jobs { + if !valid { + badjobs = append(badjobs, job) + fmt.Printf("Job %v does not have a matching testgrid testgroup\n", job) + } + } + + badconfigs := []string{} + for testgroup, valid := range testgroups { + if !valid { + badconfigs = append(badconfigs, testgroup) + fmt.Printf("Testgrid group %v does not have a matching jenkins or prow job\n", testgroup) + } + } + + if len(badconfigs) > 0 { + fmt.Printf("Total bad config(s) - %v\n", len(badconfigs)) + } + + if len(badjobs) > 0 { + fmt.Printf("Total bad job(s) - %v\n", len(badjobs)) + } + + if len(badconfigs) > 0 || len(badjobs) > 0 { + os.Exit(1) + } } From 9a5f92c07dca09c86f6e9fc1ddc9bf96bd43872d Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Fri, 13 Oct 2017 10:44:47 -0700 Subject: [PATCH 18/20] Make the periodic job added by @foxish consistent with the pre/post submit tests. * All tests should be Prow jobs; delete jobs/config.json entry for the periodic job. * All jobs should use the same container image for the tests. * Create dashboard tabs on the bigdata dashboard. Remove tabs from the apps dashboard. --- jobs/config.json | 21 --------------------- prow/config.yaml | 27 ++++++--------------------- testgrid/config/config.yaml | 18 +++++++++++------- 3 files changed, 17 insertions(+), 49 deletions(-) diff --git a/jobs/config.json b/jobs/config.json index de4328485387..df0bfbe10c80 100644 --- a/jobs/config.json +++ b/jobs/config.json @@ -8068,27 +8068,6 @@ "sig-cluster-lifecycle" ] }, - "ci-kubernetes-e2e-mlkube-gke": { - "args": [ - "--charts", - "--check-leaked-resources", - "--deployment=gke", - "--extract=gke", - "--gcp-cloud-sdk=gs://cloud-sdk-testing/ci/staging", - "--gcp-node-image=gci", - "--gcp-project=mlkube-testing", - "--gcp-zone=us-central1-b", - "--gke-environment=prod", - "--mount-paths=$GOPATH/src/github.com/foxish/mlkube.io:/src/k8s.io/charts", - "--provider=gke", - "--test=false", - "--timeout=50m" - ], - "scenario": "kubernetes_e2e", - "sigOwners": [ - "sig-big-data" - ] - }, "ci-kubernetes-e2e-node-canary": { "args": [ "--deployment=node", diff --git a/prow/config.yaml b/prow/config.yaml index c47d381adc1d..ae80eb88ace7 100644 --- a/prow/config.yaml +++ b/prow/config.yaml @@ -15748,39 +15748,24 @@ periodics: defaultMode: 256 secretName: aws-cred -- interval: 30m +- interval: 8h agent: kubernetes - name: ci-kubernetes-e2e-mlkube-gke + name: mlkube-build-periodic spec: containers: - - args: - - --timeout=70 - - --repo=github.com/foxish/mlkube.io + # TODO(jlewi): Replace latest with a specific tag once the images stabilize. + - image: gcr.io/mlkube-testing/builder:latest env: - name: GOOGLE_APPLICATION_CREDENTIALS value: /etc/service-account/service-account.json - - name: USER - value: prow - - name: JENKINS_GCE_SSH_PRIVATE_KEY_FILE - value: /etc/ssh-key-secret/ssh-private - - name: JENKINS_GCE_SSH_PUBLIC_KEY_FILE - value: /etc/ssh-key-secret/ssh-public - image: gcr.io/k8s-testimages/kubekins-e2e:v20171011-8265ed12-master volumeMounts: - - mountPath: /etc/service-account - name: service - readOnly: true - - mountPath: /etc/ssh-key-secret - name: ssh + - name: service + mountPath: /etc/service-account readOnly: true volumes: - name: service secret: secretName: service-account - - name: ssh - secret: - defaultMode: 256 - secretName: ssh-key-secret - name: ci-kubernetes-e2e-node-canary interval: 1h diff --git a/testgrid/config/config.yaml b/testgrid/config/config.yaml index 64799f452e37..4f064b3f1d40 100644 --- a/testgrid/config/config.yaml +++ b/testgrid/config/config.yaml @@ -1535,8 +1535,8 @@ test_groups: gcs_prefix: kubernetes-jenkins/logs/ci-kubernetes-e2e-gke-gci-master-gci-new-downgrade-cluster-parallel - name: ci-kubernetes-e2e-gke-gci-new-gci-master-upgrade-cluster-new gcs_prefix: kubernetes-jenkins/logs/ci-kubernetes-e2e-gke-gci-new-gci-master-upgrade-cluster-new -- name: ci-kubernetes-e2e-mlkube-gke - gcs_prefix: kubernetes-jenkins/logs/ci-kubernetes-e2e-mlkube-gke +- name: mlkube-build-periodic + gcs_prefix: kubernetes-jenkins/logs/mlkube-build-periodic - name: mlkube-build-presubmit gcs_prefix: kubernetes-jenkins/logs/mlkube-build-presubmit - name: mlkube-build-postsubmit @@ -3085,8 +3085,6 @@ dashboards: dashboard_tab: - name: charts-gce test_group_name: ci-kubernetes-charts-gce - - name: mlkube-gke - test_group_name: ci-kubernetes-e2e-mlkube-gke - name: perf-tests dashboard_tab: @@ -3544,9 +3542,15 @@ dashboards: - name: sig-big-data dashboard_tab: - - name: mlkube-gke - description: what does this tab do exactly - test_group_name: ci-kubernetes-e2e-mlkube-gke + - name: mlkube-build-periodic + description: Periodic builds and testing of TfJob CRD at head running on stable GKE. + test_group_name: mlkube-build-periodic + - name: mlkube-build-postsubmit + description: Postsubmit tests of TfJob CRD running on stable GKE. + test_group_name: mlkube-build-postsubmit + - name: mlkube-build-presubmit + description: Presubmit tests of TfJob CRD running on stable GKE. + test_group_name: mlkube-build-presubmit notifications: - summary: Please configure this skeleton dashboard and remove this notification context_link: https://github.com/kubernetes/test-infra/tree/master/testgrid/config From 9e76d5f3f318ff3748b3ed96f7ac1528fd2311ea Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Fri, 13 Oct 2017 11:50:59 -0700 Subject: [PATCH 19/20] Address code review comments. --- jobs/config_test.py | 7 ------- prow/config.yaml | 38 ++++++++++++++++++------------------- testgrid/config/config.yaml | 3 --- 3 files changed, 19 insertions(+), 29 deletions(-) diff --git a/jobs/config_test.py b/jobs/config_test.py index 96079943450b..c98e58702615 100755 --- a/jobs/config_test.py +++ b/jobs/config_test.py @@ -789,13 +789,6 @@ def test_all_project_are_unique(self): 'ci-kubernetes-node-kubelet-conformance': 'ci-kubernetes-node-kubelet-*', 'ci-kubernetes-node-kubelet-benchmark': 'ci-kubernetes-node-kubelet-*', 'ci-kubernetes-node-kubelet': 'ci-kubernetes-node-kubelet-*', - - # The mlkube projects intentionally share projects, - # We map each test name to the same value so that below when we check for number of tests - # using this project there will just be one entry. - 'mlkube-build-presubmit': 'mlkube-*', - 'mlkube-build-postsubmit': 'mlkube-*', - 'ci-kubernetes-e2e-mlkube-gke': 'mlkube-*', } for soak_prefix in [ 'ci-kubernetes-soak-gce-1.5', diff --git a/prow/config.yaml b/prow/config.yaml index ae80eb88ace7..eff186e485a3 100644 --- a/prow/config.yaml +++ b/prow/config.yaml @@ -15748,25 +15748,6 @@ periodics: defaultMode: 256 secretName: aws-cred -- interval: 8h - agent: kubernetes - name: mlkube-build-periodic - spec: - containers: - # TODO(jlewi): Replace latest with a specific tag once the images stabilize. - - image: gcr.io/mlkube-testing/builder:latest - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /etc/service-account/service-account.json - volumeMounts: - - name: service - mountPath: /etc/service-account - readOnly: true - volumes: - - name: service - secret: - secretName: service-account - - name: ci-kubernetes-e2e-node-canary interval: 1h agent: kubernetes @@ -17273,6 +17254,25 @@ periodics: secret: secretName: velodrome-influxdb +- interval: 8h + agent: kubernetes + name: mlkube-build-periodic + spec: + containers: + # TODO(jlewi): Replace latest with a specific tag once the images stabilize. + - image: gcr.io/mlkube-testing/builder:latest + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /etc/service-account/service-account.json + volumeMounts: + - name: service + mountPath: /etc/service-account + readOnly: true + volumes: + - name: service + secret: + secretName: service-account + - name: periodic-kubernetes-bazel-build-1-6 interval: 2h agent: kubernetes diff --git a/testgrid/config/config.yaml b/testgrid/config/config.yaml index 4f064b3f1d40..d227be766c87 100644 --- a/testgrid/config/config.yaml +++ b/testgrid/config/config.yaml @@ -3551,9 +3551,6 @@ dashboards: - name: mlkube-build-presubmit description: Presubmit tests of TfJob CRD running on stable GKE. test_group_name: mlkube-build-presubmit - notifications: - - summary: Please configure this skeleton dashboard and remove this notification - context_link: https://github.com/kubernetes/test-infra/tree/master/testgrid/config - name: sig-autoscaling dashboard_tab: From 6603279da90b0ee7e0177863e5b5d28765810c90 Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Fri, 13 Oct 2017 12:08:27 -0700 Subject: [PATCH 20/20] * Remove reference to socket. * Configure trigger plugin for mlkube.io repo. --- prow/config.yaml | 3 --- prow/plugins.yaml | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/prow/config.yaml b/prow/config.yaml index eff186e485a3..d3662b58ca5c 100644 --- a/prow/config.yaml +++ b/prow/config.yaml @@ -3308,9 +3308,6 @@ postsubmits: - name: service secret: secretName: service-account - - name: docker - hostPath: - path: /var/run/docker.sock periodics: - interval: 1h diff --git a/prow/plugins.yaml b/prow/plugins.yaml index 931c9013a0f5..714ead64a28b 100644 --- a/prow/plugins.yaml +++ b/prow/plugins.yaml @@ -8,6 +8,7 @@ triggers: - kubernetes-incubator - kubernetes-security - google/cadvisor + - jlewi/mlkube.io trusted_org: kubernetes heart: @@ -88,6 +89,9 @@ plugins: kubernetes-security/kubernetes: - trigger + jlewi/mlkube.io: + - trigger + spxtr/envoy: - assign - close