[WIP] Fix gateway datapath disrupt on update

Reimplementing the vswitchd reload using separate start and stop scripts
so that it can be executed partially between consecutives instances of
the pod.

Closes-Issue: OSPRH-6326

pkg/ovncontroller/daemonset.go

@@ -177,7 +177,7 @@ func CreateOVSDaemonSet(
 			Lifecycle: &corev1.Lifecycle{
 				PreStop: &corev1.LifecycleHandler{
 					Exec: &corev1.ExecAction{
-						Command: []string{"/usr/share/openvswitch/scripts/ovs-ctl", "stop", "--no-ovs-vswitchd"},
+						Command: []string{"/usr/local/bin/container-scripts/stop-ovsdb-server.sh"},
 					},
 				},
 			},
@@ -200,11 +200,11 @@ func CreateOVSDaemonSet(
 		{
 			Name:    "ovs-vswitchd",
 			Command: []string{"/bin/bash", "-c"},
-			Args:    []string{"/usr/local/bin/container-scripts/net_setup.sh && /usr/sbin/ovs-vswitchd --pidfile --mlockall"},
+			Args:    []string{"/usr/local/bin/container-scripts/net_setup.sh && /usr/local/bin/container-scripts/start-vswitchd.sh"},
 			Lifecycle: &corev1.Lifecycle{
 				PreStop: &corev1.LifecycleHandler{
 					Exec: &corev1.ExecAction{
-						Command: []string{"/usr/share/openvswitch/scripts/ovs-ctl", "stop", "--no-ovsdb-server"},
+						Command: []string{"/usr/local/bin/container-scripts/stop-vswitchd.sh"},
 					},
 				},
 			},

templates/ovncontroller/bin/start-ovsdb-server.sh

@@ -16,6 +16,11 @@
 set -ex
 
 CTL_ARGS="--system-id=random --no-ovs-vswitchd"
+ovs_folder=/var/lib/openvswitch
+
+# Remove the finish_ovsdb_server in case it exists since it is used as
+# semaphore for exiting the ovsdb-server container
+rm $ovs_folder/finish_ovsdb_server || true
 
 # Initialize or upgrade database if needed
 /usr/share/openvswitch/scripts/ovs-ctl start $CTL_ARGS

templates/ovncontroller/bin/start-vswitchd.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+#
+# Copyright 2024 Red Hat Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+set -x
+
+source $(dirname $0)/functions
+
+ovs_folder=/var/lib/openvswitch
+
+wait_for_ovsdb_server
+
+# Start vswitchd by asking it to wait till flow restore is finished.
+ovs-vsctl --no-wait set open_vswitch . other_config:flow-restore-wait=true
+/usr/sbin/ovs-vswitchd --pidfile --mlockall --detach
+
+# Restore saved flows and inform vswitchd that we are done.
+if [ -f ${ovs_folder}/flows ]; then
+    eval "$(cat ${ovs_folder}/flows)"
+fi
+
+ovs-vsctl --if-exists remove open_vswitch . other_config flow-restore-wait=true
+
+sleep infinity
+exit 0

templates/ovncontroller/bin/stop-ovsdb-server.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+#
+# Copyright 2024 Red Hat Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+ovs_folder=/var/lib/openvswitch
+
+# ovs_vswitchd container needs to terminate before ovsdb-server because it need
+# access to the db on the preStop script. This semaphore ensures it is not torn
+# down.
+while [ ! -f $ovs_folder/finish_ovsdb_server ]
+do
+  sleep 0.1
+done
+
+rm $ovs_folder/finish_ovsdb_server
+/usr/share/openvswitch/scripts/ovs-ctl stop --no-ovs-vswitchd
+exit 0

templates/ovncontroller/bin/stop-vswitchd.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+#
+# Copyright 2024 Red Hat Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+set -x
+
+ovs_folder=/var/lib/openvswitch
+
+if [ -f $ovs_folder/flows ]; then
+    rm -rf $ovs_folder/flows
+fi
+if [ -f $ovs_folder/bridges ]; then
+    rm -rf $ovs_folder/bridges
+fi
+if [ -d $ovs_folder/saved_flows ]; then
+    rm -rf $ovs_folder/saved_flows
+fi
+
+ovs-vsctl -- --real list-br > $ovs_folder/bridges
+/usr/share/openvswitch/scripts/ovs-save save-flows $(cat $ovs_folder/bridges) > $ovs_folder/flows
+
+tmp_folder=$(cat $ovs_folder/flows | tail -1 | awk '{split($0,a,"\""); split(a[2],b,"/"); print b[3]}')
+mkdir ${ovs_folder}/saved_flows
+cp /tmp/${tmp_folder}/* ${ovs_folder}/saved_flows/.
+sed -i "s|/tmp/ovs-save.*/|/var/lib/openvswitch/saved_flows/|g" ${ovs_folder}/flows
+
+/usr/share/openvswitch/scripts/ovs-ctl stop --no-ovsdb-server
+
+# Once save-flows logic is complete it no longer needs ovsdb-server, this file
+# unlocks the db preStop script, working as a semaphore
+touch $ovs_folder/finish_ovsdb_server
+exit 0