From 73ee02a1aa7600a1a6e81c5018e751d3e63346e7 Mon Sep 17 00:00:00 2001 From: Alexander Apalikov Date: Tue, 11 Jun 2019 15:57:11 +0300 Subject: [PATCH] Add draft Golang Code generation PoC for SDK conformance testing. Code generator should use code templates from sdk_client use testReady scenarios from harness folder and produce go file which would run the scenario. --- build/build-sdk-conformance/Dockerfile | 36 ++++ build/build-sdk-conformance/jstest.sh | 25 +++ build/build-sdk-conformance/sidecar.sh | 26 +++ build/build-sdk-conformance/test.sh | 31 ++++ build/includes/sdk.mk | 12 ++ cmd/sdk-server/main.go | 22 ++- test/sdk/harness/testGetGameserver.yaml | 26 +++ test/sdk/harness/testHealth.yaml | 24 +++ test/sdk/harness/testReady.yaml | 26 +++ test/sdk/sdk_client/golang.yaml | 78 ++++++++ test/sdk/sdk_client/nodejs.yaml | 50 +++++ test/sdk/test_engine.go | 234 ++++++++++++++++++++++++ 12 files changed, 588 insertions(+), 2 deletions(-) create mode 100644 build/build-sdk-conformance/Dockerfile create mode 100644 build/build-sdk-conformance/jstest.sh create mode 100644 build/build-sdk-conformance/sidecar.sh create mode 100644 build/build-sdk-conformance/test.sh create mode 100644 test/sdk/harness/testGetGameserver.yaml create mode 100644 test/sdk/harness/testHealth.yaml create mode 100644 test/sdk/harness/testReady.yaml create mode 100644 test/sdk/sdk_client/golang.yaml create mode 100644 test/sdk/sdk_client/nodejs.yaml create mode 100644 test/sdk/test_engine.go diff --git a/build/build-sdk-conformance/Dockerfile b/build/build-sdk-conformance/Dockerfile new file mode 100644 index 0000000000..c3faebc951 --- /dev/null +++ b/build/build-sdk-conformance/Dockerfile @@ -0,0 +1,36 @@ +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +FROM golang:latest + +RUN apt-get update && \ + apt-get install -y wget jq && \ + apt-get clean + +# install go +WORKDIR /usr/local +ENV GO_VERSION=1.12 +ENV GO111MODULE=on +ENV GOPATH /go +RUN wget -q https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz && \ + tar -xzf go${GO_VERSION}.linux-amd64.tar.gz && rm go${GO_VERSION}.linux-amd64.tar.gz && mkdir -p ${GOPATH} + +WORKDIR /go/src/agones.dev/agones +ENV PATH /usr/local/go/bin:/go/bin:$PATH + +RUN curl -sL https://deb.nodesource.com/setup_11.x | bash - && \ + apt-get install -y nodejs + +# code generation scripts +COPY *.sh /root/ +RUN chmod +x /root/*.sh \ No newline at end of file diff --git a/build/build-sdk-conformance/jstest.sh b/build/build-sdk-conformance/jstest.sh new file mode 100644 index 0000000000..0bf6073fe0 --- /dev/null +++ b/build/build-sdk-conformance/jstest.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -ex + +cd /go/src/agones.dev/agones/test/sdk +./sdk --test="" --sdk="nodejs" +cd /go/src/agones.dev/agones/test/sdk/bin/nodejs +npm install --cache /tmp/empty-cache +npm rebuild + +cd /go/src/agones.dev/agones/test/sdk +./sdk --verify=true --sdk nodejs \ No newline at end of file diff --git a/build/build-sdk-conformance/sidecar.sh b/build/build-sdk-conformance/sidecar.sh new file mode 100644 index 0000000000..dcfbe31ee5 --- /dev/null +++ b/build/build-sdk-conformance/sidecar.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +GO111MODULE=off +cd /go/src/agones.dev/agones/cmd/sdk-server +go build +cd /go/src/agones.dev/agones/test/sdk +rm -rf ./bin +mkdir -p ./bin +mv /go/src/agones.dev/agones/cmd/sdk-server/sdk-server ./bin +go build diff --git a/build/build-sdk-conformance/test.sh b/build/build-sdk-conformance/test.sh new file mode 100644 index 0000000000..5f2c0bddb8 --- /dev/null +++ b/build/build-sdk-conformance/test.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex +GO111MODULE=off + +cd /go/src/agones.dev/agones/test/sdk +./sdk --test="" +cd ./bin/golang/ +dirs=($(find . -mindepth 1 -type d)) +for dir in "${dirs[@]}"; do + cd "$dir" + go build + echo $PWD + cd - +done +cd /go/src/agones.dev/agones/test/sdk +./sdk --verify=true diff --git a/build/includes/sdk.mk b/build/includes/sdk.mk index bbe2db5de9..f730ded24c 100644 --- a/build/includes/sdk.mk +++ b/build/includes/sdk.mk @@ -26,6 +26,7 @@ build_sdk_base_remote_tag = $(REGISTRY)/$(build_sdk_base_tag) build_sdk_prefix = agones-build-sdk- grpc_release_tag = v1.16.1 sdk_build_folder = build-sdk-images/ +sdk_build_conformance-folder = build-sdk-conformance/ SDK_FOLDER ?= go COMMAND ?= gen @@ -108,3 +109,14 @@ ensure-build-sdk-image-base: # create the build image sdk if it doesn't exist ensure-build-sdk-image: $(MAKE) build-build-sdk-image SDK_FOLDER=$(SDK_FOLDER) + +build-conformance-tests: + cd $(sdk_build_conformance-folder); \ + docker build --tag=conformance:$(build_version) ./ $(DOCKER_BUILD_ARGS) + docker run --rm $(common_mounts) -e "VERSION=$(VERSION)" $(DOCKER_RUN_ARGS) conformance:$(build_version) /root/sidecar.sh + +run-conformance-tests: + docker run --rm $(common_mounts) -e "VERSION=$(VERSION)" $(DOCKER_RUN_ARGS) conformance:$(build_version) /root/test.sh + +run-conformance-jstests: + docker run --rm $(common_mounts) -e "VERSION=$(VERSION)" $(DOCKER_RUN_ARGS) conformance:$(build_version) /root/jstest.sh \ No newline at end of file diff --git a/cmd/sdk-server/main.go b/cmd/sdk-server/main.go index 27eff9091c..a37db2d18d 100644 --- a/cmd/sdk-server/main.go +++ b/cmd/sdk-server/main.go @@ -22,6 +22,7 @@ import ( "os" "path/filepath" "strings" + "time" "agones.dev/agones/pkg" "agones.dev/agones/pkg/client/clientset/versioned" @@ -51,6 +52,7 @@ const ( localFlag = "local" fileFlag = "file" addressFlag = "address" + timeout = "timeout" ) var ( @@ -69,6 +71,7 @@ func main() { logger.WithField("grpcPort", grpcPort).WithField("Address", ctlConf.Address).Fatalf("Could not listen on grpcPort") } stop := signals.NewStopChannel() + timedStop := make(chan struct{}) grpcServer := grpc.NewServer() // don't graceful stop, because if we get a kill signal // then the gameserver is being shut down, and we no longer @@ -89,6 +92,12 @@ func main() { if err != nil { logger.WithError(err).Fatal("Could not start local sdk server") } + if ctlConf.Timeout != 0 { + go func() { + time.Sleep(time.Duration(ctlConf.Timeout) * time.Second) + close(timedStop) + }() + } } else { var config *rest.Config config, err = rest.InClusterConfig() @@ -127,7 +136,11 @@ func main() { go runGrpc(grpcServer, lis) go runGateway(ctx, grpcEndpoint, mux, httpServer) - <-stop + select { + case <-stop: + case <-timedStop: + } + logger.Info("shutting down sdk server") } @@ -190,22 +203,26 @@ func parseEnvFlags() config { viper.SetDefault(localFlag, false) viper.SetDefault(fileFlag, "") viper.SetDefault(addressFlag, "localhost") + viper.SetDefault(timeout, 0) pflag.Bool(localFlag, viper.GetBool(localFlag), "Set this, or LOCAL env, to 'true' to run this binary in local development mode. Defaults to 'false'") pflag.StringP(fileFlag, "f", viper.GetString(fileFlag), "Set this, or FILE env var to the path of a local yaml or json file that contains your GameServer resoure configuration") - pflag.String(addressFlag, viper.GetString(addressFlag), "The Address to bind the server grpcPort to. Defaults to 'localhost") + pflag.String(addressFlag, viper.GetString(addressFlag), "The Address to bind the server grpcPort to. Defaults to 'localhost'") + pflag.Int(timeout, viper.GetInt(timeout), "Time of execution before close. Useful for tests") pflag.Parse() viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) runtime.Must(viper.BindEnv(localFlag)) runtime.Must(viper.BindEnv(gameServerNameEnv)) runtime.Must(viper.BindEnv(podNamespaceEnv)) + runtime.Must(viper.BindEnv(timeout)) runtime.Must(viper.BindPFlags(pflag.CommandLine)) return config{ IsLocal: viper.GetBool(localFlag), Address: viper.GetString(addressFlag), LocalFile: viper.GetString(fileFlag), + Timeout: viper.GetInt(timeout), } } @@ -214,4 +231,5 @@ type config struct { Address string IsLocal bool LocalFile string + Timeout int } diff --git a/test/sdk/harness/testGetGameserver.yaml b/test/sdk/harness/testGetGameserver.yaml new file mode 100644 index 0000000000..3524c64f89 --- /dev/null +++ b/test/sdk/harness/testGetGameserver.yaml @@ -0,0 +1,26 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + - init + - ready + - getGameServer + - setlabel + - setannotation + - cleanup +expected: + - Ready + - getting GameServer details + - Setting label + - Setting annotation diff --git a/test/sdk/harness/testHealth.yaml b/test/sdk/harness/testHealth.yaml new file mode 100644 index 0000000000..9fe63b36d4 --- /dev/null +++ b/test/sdk/harness/testHealth.yaml @@ -0,0 +1,24 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + - init + - ready + - health + - shutdown + - cleanup +expected: + - Ready + - Health + - Shutdown diff --git a/test/sdk/harness/testReady.yaml b/test/sdk/harness/testReady.yaml new file mode 100644 index 0000000000..9ee6245870 --- /dev/null +++ b/test/sdk/harness/testReady.yaml @@ -0,0 +1,26 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + - init + - ready + - allocate + - setlabel + - setannotation + - cleanup +expected: + - Ready + - Allocate + - Setting label + - Setting annotation diff --git a/test/sdk/sdk_client/golang.yaml b/test/sdk/sdk_client/golang.yaml new file mode 100644 index 0000000000..ae15a656a8 --- /dev/null +++ b/test/sdk/sdk_client/golang.yaml @@ -0,0 +1,78 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +functions: + health: | + time.Sleep(1*time.Second) + for i := 0 ; i < 5; i ++ { + err = s.Health() + if err != nil { + log.Fatalf("Could not send Health check") + } else { + log.Println("Health message sent") + } + } + ready: | + err = s.Ready() + if err != nil { + log.Fatalf("Could not send ready message") + } + shutdown: | + err = s.Shutdown() + if err != nil { + log.Fatalf("Could not send shutdown request") + } + getGameServer: | + gs, err := s.GameServer() + if err != nil { + log.Fatalf("Could not get gameserver parameters") + } + log.Println(gs) + allocate: | + err = s.Allocate() + if err != nil { + log.Fatalf("Could not send allocate request") + } + setlabel: | + err = s.SetLabel("new", "label") + if err != nil { + log.Fatalf("Could not set label") + } + setannotation: | + err = s.SetAnnotation("new", "annotation") + if err != nil { + log.Fatalf("Could not set annotation") + } + init: | + package main + + import ( + "time" + "log" + + sdk "agones.dev/agones/sdks/go" + ) + + func main() { + log.Println("starting") + time.Sleep(100 * time.Millisecond) + s, err := sdk.NewSDK() + if err != nil { + log.Fatalf("Could not connect to sdk: %v", err) + } else { + log.Println("SDK initialised") + } + cleanup: | + time.Sleep(3*time.Second) + } diff --git a/test/sdk/sdk_client/nodejs.yaml b/test/sdk/sdk_client/nodejs.yaml new file mode 100644 index 0000000000..37fc1f4052 --- /dev/null +++ b/test/sdk/sdk_client/nodejs.yaml @@ -0,0 +1,50 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +functions: + health: | + let result = await agonesSDK.health(); + ready: | + await agonesSDK.ready(); + setlabel: | + await agonesSDK.setLabel("label", "labelValue"); + setannotation: | + await agonesSDK.setAnnotation("annotation", "annotationValue"); + init: | + const AgonesSDK = require('agones'); + const agonesSDK = new AgonesSDK(); + const connect = async function() { + agonesSDK.watchGameServer((result) => { + console.log('watch', result); + }); + try { + getGameServer: | + const result = await agonesSDK.getGameServer(); + console.log('gameServer', result); + shutdown: | + setTimeout(() => { + console.log('send shutdown request'); + agonesSDK.shutdown(); + }, 1000); + allocate: | + setTimeout(() => { + console.log('send shutdown request'); + agonesSDK.allocate(); + }, 1000); + cleanup: | + } catch (error) { + console.error(error); + } + }; + connect(); \ No newline at end of file diff --git a/test/sdk/test_engine.go b/test/sdk/test_engine.go new file mode 100644 index 0000000000..663f1ab323 --- /dev/null +++ b/test/sdk/test_engine.go @@ -0,0 +1,234 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "strings" + "time" + + "gopkg.in/yaml.v2" +) + +type testCase struct { + Steps []string `yaml:"steps,omitempty"` + Result []string `yaml:"expected,omitempty"` +} + +func runSidecar(language string) []bool { + files, err := ioutil.ReadDir("./bin/" + language) + if err != nil { + log.Fatal(err) + } + var tc testCase + testResults := make([]bool, 0) + for _, f := range files { + if f.Name() == "node_modules" || !f.IsDir() { + continue + } + sidecar := "bin/sdk-server" + cmdSdk := exec.Command(sidecar, "--local", "-f", "../../examples/gameserver.yaml", + "--timeout", "7") + sdkServer, err := cmdSdk.StderrPipe() + if err != nil { + log.Fatal(err) + } + if err := cmdSdk.Start(); err != nil { + log.Fatal(err) + } + //Wait for GRPC gateway to start + time.Sleep(2 * time.Second) + + //cmd = exec.Command("cd", "./bin", "&&", "go", "build") + + yamlFile, err := ioutil.ReadFile("harness/" + f.Name() + ".yaml") + if err != nil { + log.Printf("yamlFile.Get err #%v ", err) + } + + if err := yaml.Unmarshal(yamlFile, &tc); err != nil { + log.Fatal(err) + } + + var cmdTest *exec.Cmd + + if language == "nodejs" { + cmdTest = exec.Command("npm", "run", f.Name()) + cmdTest.Dir = "./bin/nodejs/" + } else { + cmdTest = exec.Command("./bin/golang/" + f.Name() + "/" + f.Name()) + } + stderr, err := cmdTest.StderrPipe() + if err != nil { + log.Fatal(err) + } + + if err := cmdTest.Start(); err != nil { + log.Fatal(err) + } + + sc := bufio.NewScanner(stderr) + for sc.Scan() { + log.Printf("Line: %s\n", sc.Text()) + } + + sc = bufio.NewScanner(sdkServer) + + result := make(map[string]bool) + for sc.Scan() { + log.Printf("SDK Line : %s\n", sc.Text()) + + for _, v := range tc.Result { + if strings.Contains(sc.Text(), v) { + result[v] = true + } + } + } + + if err := cmdTest.Wait(); err != nil { + log.Fatal(err) + } + + if err := cmdSdk.Wait(); err != nil { + log.Fatal(err) + } + all := true + for _, v := range tc.Result { + if _, ok := result[v]; !ok { + all = false + log.Printf("FAIL in %s file. Could not find expected: %s\n", f.Name(), v) + } + + } + testResults = append(testResults, all) + } + return testResults +} + +// Code Generation function which would +// create a client code with test steps +// described by test files +func main() { + lang := flag.String("sdk", "golang", "sdk language") + testname := flag.String("test", "testReady", "test name") + verify := flag.Bool("verify", false, "run sidecar and appropriate test") + flag.Parse() + log.Println(*lang) + log.Println(*testname) + // Enable line numbers in logging + log.SetFlags(log.LstdFlags | log.Lshortfile) + + if *verify { + testResults := runSidecar(*lang) + log.Println("test Results: ", testResults) + return + } + fileExtensions := map[string]string{"golang": "go", "nodejs": "js", "cpp": "cpp"} + // Functions which contains code excerpts like init, ready, cleanup + var functions struct { + Functions map[string]string `yaml:"functions,omitempty"` + } + yamlFile, err := ioutil.ReadFile("sdk_client/" + *lang + ".yaml") + if err != nil { + log.Printf("yamlFile.Get err: %v ", err) + } + if err := yaml.Unmarshal(yamlFile, &functions); err != nil { + log.Fatal(err) + } + + for i := range functions.Functions { + log.Printf("Function loaded '%s' \n", i) + } + + testList := make([]string, 0) + if *testname == "" { + files, err := ioutil.ReadDir("./harness") + if err != nil { + log.Fatal(err) + } + + for _, f := range files { + testList = append(testList, f.Name()) + } + } else { + testList = append(testList, *testname+".yaml") + } + + var tc testCase + for i, f := range testList { + log.Println("Creating test ", i, " ", f) + yamlFile, err = ioutil.ReadFile("harness/" + f) + if err != nil { + log.Printf("yamlFile.Get err: %v ", err) + } + + if err := yaml.Unmarshal(yamlFile, &tc); err != nil { + log.Fatal(err) + } + code := "" + for _, v := range tc.Steps { + if f, ok := functions.Functions[v]; ok { + code += f + } else { + log.Printf("Could not find function declaration %s \n", v) + } + } + test := f[:strings.IndexByte(f, '.')] + path := "./bin/" + *lang + "/" + test + err = os.MkdirAll(path, 0700) + if err != nil { + log.Printf("Create dir error: %v ", err) + } + err = ioutil.WriteFile(path+"/"+test+"."+fileExtensions[*lang], []byte(code), 0644) + if err != nil { + log.Printf("Write to file error: %v ", err) + } + } + + if *lang == "nodejs" { + packagesBegin := ` +{ +"dependencies": { +"agones": "../../../../sdks/nodejs" +}, +"scripts": { + + ` + packagesEnd := ` + +} +} +` + scripts := "" + for i, f := range testList { + test := f[:strings.IndexByte(f, '.')] + scripts += fmt.Sprintf("\"%s\": \"node ./%s/%s.js\"", test, test, test) + if i+1 < len(testList) { + scripts += ",\n" + } + } + packages := packagesBegin + scripts + packagesEnd + err = ioutil.WriteFile("./bin/nodejs/package.json", []byte(packages), 0644) + if err != nil { + log.Printf("Write to file error: %v", err) + } + } +}