Merge pull request #16439 from pravisankar/netdiags-image-check

Automatic merge from submit-queue

Bug 1481147 - Fix default pod image for network diagnostics

- This also ensures network diagnostics pod and test pod images uses
  deployed openshift version/tag (not the latest) so that it doesn't
  need to download another same image with latest tag.

- Print more details when network diagnostics test setup fails.
  Currently when network diags fails, it only informs how many test pods failed but doesn't provide why those pods failed. This change will fetch logs for the pods in case of setup failure.

pkg/diagnostics/network/objects.go

@@ -113,15 +113,7 @@ func GetTestPod(testPodImage, testPodProtocol, podName, nodeName string, testPod
 		},
 	}
 
-	var trimmedPodImage string
-	imageTokens := strings.Split(testPodImage, "/")
-	n := len(imageTokens)
-	if n < 2 {
-		trimmedPodImage = testPodImage
-	} else {
-		trimmedPodImage = imageTokens[n-2] + "/" + imageTokens[n-1]
-	}
-	if trimmedPodImage == util.NetworkDiagDefaultTestPodImage {
+	if getTrimmedImage(testPodImage) == getTrimmedImage(util.GetNetworkDiagDefaultTestPodImage()) {
 		pod.Spec.Containers[0].Command = []string{
 			"socat", "-T", "1", "-d",
 			fmt.Sprintf("%s-l:%d,reuseaddr,fork,crlf", testPodProtocol, testPodPort),
@@ -131,6 +123,14 @@ func GetTestPod(testPodImage, testPodProtocol, podName, nodeName string, testPod
 	return pod
 }
 
+func getTrimmedImage(image string) string {
+	// Image format could be: [<dns-name>/]openshift/origin-deployer[:<tag>]
+	tokens := strings.Split(image, "/")
+	trimImageWithTag := tokens[len(tokens)-1]
+
+	return strings.Split(trimImageWithTag, ":")[0]
+}
+
 func GetTestService(serviceName, podName, podProtocol, nodeName string, podPort int) *kapi.Service {
 	return &kapi.Service{
 		ObjectMeta: metav1.ObjectMeta{Name: serviceName},

pkg/diagnostics/network/setup.go

@@ -10,6 +10,7 @@ import (
 	kerrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/storage/names"
 	kclientcmd "k8s.io/client-go/tools/clientcmd"
@@ -66,7 +67,12 @@ func (d *NetworkDiagnostic) TestSetup() error {
 	}
 	// Wait for test pods and services to be up and running on all valid nodes
 	if err = d.waitForTestPodAndService(nsList); err != nil {
-		return fmt.Errorf("Failed to run network diags test pod and service: %v", err)
+		logData, er := d.getPodLogs(nsList)
+		if er != nil {
+			return fmt.Errorf("Failed to run network diags test pod and service: %v, fetching logs failed: %v", err, er)
+		} else {
+			return fmt.Errorf("Failed to run network diags test pod and service: %v, details: %s", err, logData)
+		}
 	}
 	return nil
 }
@@ -169,6 +175,41 @@ func (d *NetworkDiagnostic) waitForTestPodAndService(nsList []string) error {
 	return kerrors.NewAggregate(errList)
 }
 
+func (d *NetworkDiagnostic) getPodLogs(nsList []string) (string, error) {
+	logData := sets.String{}
+	errList := []error{}
+	limit := int64(1024)
+
+	for _, name := range nsList {
+		podList, err := d.getPodList(name, util.NetworkDiagTestPodNamePrefix)
+		if err != nil {
+			return "", err
+		}
+
+		for _, pod := range podList.Items {
+			opts := &kapi.PodLogOptions{
+				TypeMeta:   pod.TypeMeta,
+				Container:  pod.Name,
+				Follow:     true,
+				LimitBytes: &limit,
+			}
+
+			req, err := d.Factory.LogsForObject(&pod, opts, 10*time.Second)
+			if err != nil {
+				errList = append(errList, err)
+				continue
+			}
+			data, err := req.DoRaw()
+			if err != nil {
+				errList = append(errList, err)
+				continue
+			}
+			logData.Insert(string(data[:]))
+		}
+	}
+	return strings.Join(logData.List(), ", "), kerrors.NewAggregate(errList)
+}
+
 func (d *NetworkDiagnostic) getCountOfTestPods(nsList []string) (int, int, error) {
 	totalPodCount := 0
 	runningPodCount := 0

pkg/diagnostics/networkpod/util/util.go

@@ -37,10 +37,16 @@ const (
 	NetworkDiagDefaultTestPodPort     = 8080
 )
 
-var (
-	NetworkDiagDefaultPodImage     = variable.DefaultImagePrefix
-	NetworkDiagDefaultTestPodImage = variable.DefaultImagePrefix + "-deployer"
-)
+func GetNetworkDiagDefaultPodImage() string {
+	imageTemplate := variable.NewDefaultImageTemplate()
+	imageTemplate.Format = variable.DefaultImagePrefix + ":${version}"
+	return imageTemplate.ExpandOrDie("")
+}
+
+func GetNetworkDiagDefaultTestPodImage() string {
+	imageTemplate := variable.NewDefaultImageTemplate()
+	return imageTemplate.ExpandOrDie("deployer")
+}
 
 func GetOpenShiftNetworkPlugin(osClient *osclient.Client) (string, bool, error) {
 	cn, err := osClient.ClusterNetwork().Get(networkapi.ClusterNetworkDefault, metav1.GetOptions{})

pkg/oc/admin/diagnostics/diagnostics.go

@@ -151,8 +151,8 @@ func NewCmdDiagnostics(name string, fullName string, out io.Writer) *cobra.Comma
 	cmd.Flags().BoolVar(&o.ImageTemplate.Latest, options.FlagLatestImageName, false, "If true, when expanding the image template, use latest version, not release version")
 	cmd.Flags().BoolVar(&o.PreventModification, options.FlagPreventModificationName, false, "If true, may be set to prevent diagnostics making any changes via the API")
 	cmd.Flags().StringVar(&o.NetworkOptions.LogDir, options.FlagNetworkDiagLogDir, netutil.NetworkDiagDefaultLogDir, "Path to store network diagnostic results in case of errors")
-	cmd.Flags().StringVar(&o.NetworkOptions.PodImage, options.FlagNetworkDiagPodImage, netutil.NetworkDiagDefaultPodImage, "Image to use for network diagnostic pod")
-	cmd.Flags().StringVar(&o.NetworkOptions.TestPodImage, options.FlagNetworkDiagTestPodImage, netutil.NetworkDiagDefaultTestPodImage, "Image to use for network diagnostic test pod")
+	cmd.Flags().StringVar(&o.NetworkOptions.PodImage, options.FlagNetworkDiagPodImage, netutil.GetNetworkDiagDefaultPodImage(), "Image to use for network diagnostic pod")
+	cmd.Flags().StringVar(&o.NetworkOptions.TestPodImage, options.FlagNetworkDiagTestPodImage, netutil.GetNetworkDiagDefaultTestPodImage(), "Image to use for network diagnostic test pod")
 	cmd.Flags().StringVar(&o.NetworkOptions.TestPodProtocol, options.FlagNetworkDiagTestPodProtocol, netutil.NetworkDiagDefaultTestPodProtocol, "Protocol used to connect to network diagnostic test pod")
 	cmd.Flags().IntVar(&o.NetworkOptions.TestPodPort, options.FlagNetworkDiagTestPodPort, netutil.NetworkDiagDefaultTestPodPort, "Serving port on the network diagnostic test pod")
 	flagtypes.GLog(cmd.Flags())
@@ -195,10 +195,10 @@ func (o *DiagnosticsOptions) Complete(args []string) error {
 		o.NetworkOptions.LogDir = logdir
 	}
 	if len(o.NetworkOptions.PodImage) == 0 {
-		o.NetworkOptions.PodImage = netutil.NetworkDiagDefaultPodImage
+		o.NetworkOptions.PodImage = netutil.GetNetworkDiagDefaultPodImage()
 	}
 	if len(o.NetworkOptions.TestPodImage) == 0 {
-		o.NetworkOptions.TestPodImage = netutil.NetworkDiagDefaultTestPodImage
+		o.NetworkOptions.TestPodImage = netutil.GetNetworkDiagDefaultTestPodImage()
 	}
 
 	supportedProtocols := sets.NewString(string(kapi.ProtocolTCP), string(kapi.ProtocolUDP))