diff --git a/pkg/pillar/cmd/vcomlink/vsocksrv.go b/pkg/pillar/cmd/vcomlink/vsocksrv.go
index 0884308b2b..c82739b288 100644
--- a/pkg/pillar/cmd/vcomlink/vsocksrv.go
+++ b/pkg/pillar/cmd/vcomlink/vsocksrv.go
@@ -49,18 +49,6 @@ func startVcomServer(listener SocketListener) {
 	}
 	defer unix.Close(fd)
 
-	// Set read timeout and write timeout
-	err = unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &unix.Timeval{Sec: readTimeout})
-	if err != nil {
-		log.Errorf("Error setting read timeout: %v", err)
-		return
-	}
-	err = unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_SNDTIMEO, &unix.Timeval{Sec: writeTimeout})
-	if err != nil {
-		log.Errorf("Error setting write timeout: %v", err)
-		return
-	}
-
 	for {
 		conn, _, err := unix.Accept(fd)
 		if err != nil {
@@ -69,6 +57,18 @@ func startVcomServer(listener SocketListener) {
 			continue
 		}
 
+		// Set read timeout and write timeout
+		err = unix.SetsockoptTimeval(conn, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &unix.Timeval{Sec: readTimeout})
+		if err != nil {
+			log.Errorf("error setting read timeout: %v", err)
+			return
+		}
+		err = unix.SetsockoptTimeval(conn, unix.SOL_SOCKET, unix.SO_SNDTIMEO, &unix.Timeval{Sec: writeTimeout})
+		if err != nil {
+			log.Errorf("error setting write timeout: %v", err)
+			return
+		}
+
 		go handleConnection(conn)
 	}
 }