diff --git a/helmtestserver/server.go b/helmtestserver/server.go
index d7f3e1f19..c4d53c42a 100644
--- a/helmtestserver/server.go
+++ b/helmtestserver/server.go
@@ -17,7 +17,8 @@ limitations under the License.
 package helmtestserver
 
 import (
-	"fmt"
+	"crypto/rand"
+	"encoding/hex"
 	"os"
 	"path/filepath"
 
@@ -30,7 +31,7 @@ import (
 )
 
 const (
-	KeyRingName = "TestUser"
+	keyRingName = "TestUser"
 )
 
 // NewTempHelmServer returns a HTTP HelmServer with a newly created
@@ -79,8 +80,9 @@ func (s *HelmServer) PackageChartWithVersion(path, version string) error {
 }
 
 // PackageSignedChartWithVersion attempts to package the chart at the given path
-// with the given version and sign it using a PGP keyring, to be served by the HelmServer.
-// It returns an error in case of a packaging failure.
+// with the given version and sign it using a internally generated PGP keyring, to be served
+// by the HelmServer. publicKeyPath is the path where the public key should be written to, which
+// can be used to verify this chart. It returns an error in case of a packaging failure.
 func (s *HelmServer) PackageSignedChartWithVersion(path, version, publicKeyPath string) error {
 	return s.packageChart(path, version, publicKeyPath)
 }
@@ -90,12 +92,15 @@ func (s *HelmServer) packageChart(path, version, publicKeyPath string) error {
 	pkg.Destination = s.Root()
 	pkg.Version = version
 	if publicKeyPath != "" {
-		secretKeyPath := fmt.Sprintf("%s/%s", s.Root(), "secret.pgp")
+		randBytes := make([]byte, 16)
+		rand.Read(randBytes)
+		secretKeyPath := filepath.Join(s.Root(), "secret-"+hex.EncodeToString(randBytes)+".pgp")
 		if err := generateKeyring(secretKeyPath, publicKeyPath); err != nil {
 			return err
 		}
+		defer os.Remove(secretKeyPath)
 		pkg.Keyring = secretKeyPath
-		pkg.Key = KeyRingName
+		pkg.Key = keyRingName
 		pkg.Sign = true
 	}
 	_, err := pkg.Run(path, nil)
@@ -103,22 +108,20 @@ func (s *HelmServer) packageChart(path, version, publicKeyPath string) error {
 }
 
 func generateKeyring(privateKeyPath, publicKeyPath string) error {
-	entity, err := openpgp.NewEntity(KeyRingName, "", "", nil)
+	entity, err := openpgp.NewEntity(keyRingName, "", "", nil)
 	if err != nil {
 		return err
 	}
 	priv, err := os.Create(privateKeyPath)
+	defer priv.Close()
 	if err != nil {
 		return err
 	}
 	pub, err := os.Create(publicKeyPath)
+	defer pub.Close()
 	if err != nil {
 		return err
 	}
-	defer func() {
-		priv.Close()
-		pub.Close()
-	}()
 	if err := entity.SerializePrivate(priv, nil); err != nil {
 		return err
 	}
diff --git a/helmtestserver/server_test.go b/helmtestserver/server_test.go
index 0f0d69816..0b5329c90 100644
--- a/helmtestserver/server_test.go
+++ b/helmtestserver/server_test.go
@@ -3,21 +3,22 @@ package helmtestserver
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"golang.org/x/crypto/openpgp"
 	"helm.sh/helm/v3/pkg/downloader"
 )
 
-func TestPackageChart(t *testing.T) {
+func TestPackageSignedChartWithVersion(t *testing.T) {
 	server, err := NewTempHelmServer()
+	defer os.RemoveAll(server.Root())
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer os.RemoveAll(server.Root())
-	publicKeyPath := fmt.Sprintf("%s/%s", server.Root(), "pub.pgp")
-	packagedChartPath := fmt.Sprintf("%s/%s", server.Root(), "helmchart-0.1.0.tgz")
-	if err := server.packageChart("./testdata/helmchart", "0.1.0", publicKeyPath); err != nil {
+	publicKeyPath := filepath.Join(server.Root(), "pub.pgp")
+	packagedChartPath := filepath.Join(server.Root(), "helmchart-0.1.0.tgz")
+	if err := server.PackageSignedChartWithVersion("./testdata/helmchart", "0.1.0", publicKeyPath); err != nil {
 		t.Fatal(err)
 	}
 
@@ -26,10 +27,10 @@ func TestPackageChart(t *testing.T) {
 	}
 
 	out, err := os.Open(publicKeyPath)
+	defer out.Close()
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer out.Close()
 
 	if _, err = openpgp.ReadKeyRing(out); err != nil {
 		t.Fatal(err)
@@ -46,10 +47,10 @@ func TestPackageChart(t *testing.T) {
 
 func TestGenerateIndex(t *testing.T) {
 	server, err := NewTempHelmServer()
+	defer os.RemoveAll(server.Root())
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer os.RemoveAll(server.Root())
 
 	if err := server.PackageChartWithVersion("./testdata/helmchart", "0.1.0"); err != nil {
 		t.Fatal(err)
diff --git a/helmtestserver/testdata/helmchart/templates/hpa.yaml b/helmtestserver/testdata/helmchart/templates/hpa.yaml
deleted file mode 100644
index acd8e4207..000000000
--- a/helmtestserver/testdata/helmchart/templates/hpa.yaml
+++ /dev/null
@@ -1,28 +0,0 @@
-{{- if .Values.autoscaling.enabled }}
-apiVersion: autoscaling/v2beta1
-kind: HorizontalPodAutoscaler
-metadata:
-  name: {{ include "helmchart.fullname" . }}
-  labels:
-    {{- include "helmchart.labels" . | nindent 4 }}
-spec:
-  scaleTargetRef:
-    apiVersion: apps/v1
-    kind: Deployment
-    name: {{ include "helmchart.fullname" . }}
-  minReplicas: {{ .Values.autoscaling.minReplicas }}
-  maxReplicas: {{ .Values.autoscaling.maxReplicas }}
-  metrics:
-    {{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
-    - type: Resource
-      resource:
-        name: cpu
-        targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
-    {{- end }}
-    {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
-    - type: Resource
-      resource:
-        name: memory
-        targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
-    {{- end }}
-{{- end }}
diff --git a/helmtestserver/testdata/helmchart/templates/ingress.yaml b/helmtestserver/testdata/helmchart/templates/ingress.yaml
deleted file mode 100644
index 9ec6a2f9e..000000000
--- a/helmtestserver/testdata/helmchart/templates/ingress.yaml
+++ /dev/null
@@ -1,61 +0,0 @@
-{{- if .Values.ingress.enabled -}}
-{{- $fullName := include "helmchart.fullname" . -}}
-{{- $svcPort := .Values.service.port -}}
-{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
-  {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
-  {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
-  {{- end }}
-{{- end }}
-{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
-apiVersion: networking.k8s.io/v1
-{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
-apiVersion: networking.k8s.io/v1beta1
-{{- else -}}
-apiVersion: extensions/v1beta1
-{{- end }}
-kind: Ingress
-metadata:
-  name: {{ $fullName }}
-  labels:
-    {{- include "helmchart.labels" . | nindent 4 }}
-  {{- with .Values.ingress.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-spec:
-  {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
-  ingressClassName: {{ .Values.ingress.className }}
-  {{- end }}
-  {{- if .Values.ingress.tls }}
-  tls:
-    {{- range .Values.ingress.tls }}
-    - hosts:
-        {{- range .hosts }}
-        - {{ . | quote }}
-        {{- end }}
-      secretName: {{ .secretName }}
-    {{- end }}
-  {{- end }}
-  rules:
-    {{- range .Values.ingress.hosts }}
-    - host: {{ .host | quote }}
-      http:
-        paths:
-          {{- range .paths }}
-          - path: {{ .path }}
-            {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
-            pathType: {{ .pathType }}
-            {{- end }}
-            backend:
-              {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
-              service:
-                name: {{ $fullName }}
-                port:
-                  number: {{ $svcPort }}
-              {{- else }}
-              serviceName: {{ $fullName }}
-              servicePort: {{ $svcPort }}
-              {{- end }}
-          {{- end }}
-    {{- end }}
-{{- end }}
diff --git a/helmtestserver/testdata/helmchart/templates/service.yaml b/helmtestserver/testdata/helmchart/templates/service.yaml
deleted file mode 100644
index 12e16ef71..000000000
--- a/helmtestserver/testdata/helmchart/templates/service.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ include "helmchart.fullname" . }}
-  labels:
-    {{- include "helmchart.labels" . | nindent 4 }}
-spec:
-  type: {{ .Values.service.type }}
-  ports:
-    - port: {{ .Values.service.port }}
-      targetPort: http
-      protocol: TCP
-      name: http
-  selector:
-    {{- include "helmchart.selectorLabels" . | nindent 4 }}
diff --git a/helmtestserver/testdata/helmchart/templates/serviceaccount.yaml b/helmtestserver/testdata/helmchart/templates/serviceaccount.yaml
deleted file mode 100644
index 0ff995909..000000000
--- a/helmtestserver/testdata/helmchart/templates/serviceaccount.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-{{- if .Values.serviceAccount.create -}}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: {{ include "helmchart.serviceAccountName" . }}
-  labels:
-    {{- include "helmchart.labels" . | nindent 4 }}
-  {{- with .Values.serviceAccount.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-{{- end }}
diff --git a/helmtestserver/testdata/helmchart/values.yaml b/helmtestserver/testdata/helmchart/values.yaml
index dd83a66d9..6285efd8b 100644
--- a/helmtestserver/testdata/helmchart/values.yaml
+++ b/helmtestserver/testdata/helmchart/values.yaml
@@ -36,26 +36,6 @@ securityContext: {}
   # runAsNonRoot: true
   # runAsUser: 1000
 
-service:
-  type: ClusterIP
-  port: 80
-
-ingress:
-  enabled: false
-  className: ""
-  annotations: {}
-    # kubernetes.io/ingress.class: nginx
-    # kubernetes.io/tls-acme: "true"
-  hosts:
-    - host: chart-example.local
-      paths:
-        - path: /
-          pathType: ImplementationSpecific
-  tls: []
-  #  - secretName: chart-example-tls
-  #    hosts:
-  #      - chart-example.local
-
 resources: {}
   # We usually recommend not to specify default resources and to leave this as a conscious
   # choice for the user. This also increases chances charts run on environments with little