feat: api - add support to write custom script

api/v1alpha2/minicluster_types.go

@@ -586,6 +586,10 @@ type Commands struct {
 	// +optional
 	Prefix string `json:"prefix"`
 
+	// Custom script for submit (e.g., multiple lines)
+	// +optional
+	Script string `json:"script"`
+
 	// init command is run before anything
 	// +optional
 	Init string `json:"init"`
@@ -816,6 +820,12 @@ func (f *MiniCluster) Validate() bool {
 				return false
 			}
 		}
+
+		// If a custom script is provided AND a command, no go
+		if (container.Commands.Script != "" && container.Command != "") && container.RunFlux {
+			fmt.Printf("😥️ %s has both a script and command provided, choose one\n", name)
+			return false
+		}
 	}
 	if fluxRunners != 1 {
 		valid = false

api/v1alpha2/swagger.json

@@ -75,6 +75,11 @@
           "type": "string",
           "default": ""
         },
+        "script": {
+          "description": "Custom script for submit (e.g., multiple lines)",
+          "type": "string",
+          "default": ""
+        },
         "servicePre": {
           "description": "A command only for service start.sh tor run",
           "type": "string",

chart/templates/minicluster-crd.yaml

@@ -93,6 +93,9 @@ spec:
                             Prefix to flux start / submit / broker
                             Typically used for a wrapper command to mount, etc.
                           type: string
+                        script:
+                          description: Custom script for submit (e.g., multiple lines)
+                          type: string
                         servicePre:
                           description: A command only for service start.sh tor run
                           type: string
@@ -583,6 +586,9 @@ spec:
                             Prefix to flux start / submit / broker
                             Typically used for a wrapper command to mount, etc.
                           type: string
+                        script:
+                          description: Custom script for submit (e.g., multiple lines)
+                          type: string
                         servicePre:
                           description: A command only for service start.sh tor run
                           type: string

config/crd/bases/flux-framework.org_miniclusters.yaml

@@ -92,6 +92,9 @@ spec:
                             Prefix to flux start / submit / broker
                             Typically used for a wrapper command to mount, etc.
                           type: string
+                        script:
+                          description: Custom script for submit (e.g., multiple lines)
+                          type: string
                         servicePre:
                           description: A command only for service start.sh tor run
                           type: string
@@ -586,6 +589,9 @@ spec:
                             Prefix to flux start / submit / broker
                             Typically used for a wrapper command to mount, etc.
                           type: string
+                        script:
+                          description: Custom script for submit (e.g., multiple lines)
+                          type: string
                         servicePre:
                           description: A command only for service start.sh tor run
                           type: string

docs/getting_started/custom-resource-definition.md

@@ -924,6 +924,25 @@ containers:
         ls /workdir
 ```
 
+##### script
+
+If you want to write a custom entrypoint script (that will be chmod +x and provided to flux submit) you
+can do so with script. For example:
+
+```yaml
+containers:
+  - image: my-flux-image
+    ...
+    commands:
+      script: |
+        #!/bin/bash
+        echo "This is my custom script"
+        echo "This is rank \${FLUX_TASK_RANK}
+```
+
+Note that the environment variable `$` is escaped.
+For a container that is running flux you are not allowed to provide a script and command, and validation will check this.
+
 ##### workerPre and brokerPre
 
 This is akin to pre, but only for the broker OR the workers.

docs/tutorials/index.md

@@ -20,6 +20,11 @@ The following tutorials are provided from their respective directories (and are
 
 ### Experimental
 
+#### Distributed
+
+ - [Distributed-fractal](https://github.com/flux-framework/flux-operator/tree/main/examples/distributed/fractal/minicluster.yaml): Using [distributed-fractal](https://github.com/converged-computing/distributed-fractal) to generate a fractal across nodes with GRPC. You might want to use a volume or other means to keep the cluster running (sleep infinity post command) to retrieve the image.
+ - [dragonboat](https://github.com/flux-framework/flux-operator/tree/main/examples/distributed/dragonboat/minicluster.yaml): simple consensus algorithm-driven state machine demonstration using the flux operator. See [this repository](https://github.com/researchapps/dragonboat-example).
+
 #### Bursting
 
  - [Bursting to GKE](https://github.com/flux-framework/flux-operator/tree/main/examples/experimental/bursting/broker-gke) from a local broker to an external Google Kubernetes Engine cluster.

examples/dist/flux-operator-arm.yaml

@@ -98,6 +98,9 @@ spec:
                             Prefix to flux start / submit / broker
                             Typically used for a wrapper command to mount, etc.
                           type: string
+                        script:
+                          description: Custom script for submit (e.g., multiple lines)
+                          type: string
                         servicePre:
                           description: A command only for service start.sh tor run
                           type: string
@@ -592,6 +595,9 @@ spec:
                             Prefix to flux start / submit / broker
                             Typically used for a wrapper command to mount, etc.
                           type: string
+                        script:
+                          description: Custom script for submit (e.g., multiple lines)
+                          type: string
                         servicePre:
                           description: A command only for service start.sh tor run
                           type: string

examples/dist/flux-operator.yaml

@@ -98,6 +98,9 @@ spec:
                             Prefix to flux start / submit / broker
                             Typically used for a wrapper command to mount, etc.
                           type: string
+                        script:
+                          description: Custom script for submit (e.g., multiple lines)
+                          type: string
                         servicePre:
                           description: A command only for service start.sh tor run
                           type: string
@@ -592,6 +595,9 @@ spec:
                             Prefix to flux start / submit / broker
                             Typically used for a wrapper command to mount, etc.
                           type: string
+                        script:
+                          description: Custom script for submit (e.g., multiple lines)
+                          type: string
                         servicePre:
                           description: A command only for service start.sh tor run
                           type: string

examples/distributed/dragonboat/Dockerfile

@@ -0,0 +1,6 @@
+FROM golang:latest
+RUN git clone https://github.com/researchapps/dragonboat-example /opt/dragon
+WORKDIR /opt/dragon
+RUN  make 
+COPY ./entrypoint.sh /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

examples/distributed/dragonboat/entrypoint.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Can't be zero
+replica=$((FLUX_TASK_RANK+1))
+/opt/dragon/example-helloworld -replicaid $replica -addr flux-sample-$FLUX_TASK_RANK.flux-service.default.svc.cluster.local:6300${replica}

examples/distributed/dragonboat/minicluster.yaml

@@ -0,0 +1,13 @@
+apiVersion: flux-framework.org/v1alpha2
+kind: MiniCluster
+metadata:
+  name: flux-sample
+spec:
+  size: 4
+  tasks: 4
+  flux:
+    container: 
+      image: ghcr.io/converged-computing/flux-view-ubuntu:tag-focal
+  containers:
+  - image: vanessa/dragonboat-hello-workd
+    command: /entrypoint.sh

examples/distributed/fractal/minicluster.yaml

@@ -0,0 +1,29 @@
+apiVersion: flux-framework.org/v1alpha2
+kind: MiniCluster
+metadata:
+  name: flux-sample
+spec:
+  size: 4
+  tasks: 4
+  flux:
+    container: 
+      image: ghcr.io/converged-computing/flux-view-ubuntu:tag-jammy
+  containers:
+  - image: ghcr.io/converged-computing/distributed-fractal
+    commands:
+      script: |
+        #!/bin/bash
+        start_fractal() {
+           sleep 5
+           curl -X POST http://localhost:9092/start       
+        }
+        echo "This is task \${FLUX_TASK_RANK}"
+        if [[ "\${FLUX_TASK_RANK}" == "0" ]]; then
+           start_fractal &
+           fractal leader --host 0.0.0.0:50051 --force-exit
+           # How to wait for all other ranks to be ready?
+        else
+           fractal worker --host flux-sample-0.flux-service.default.svc.cluster.local:50051
+        fi
+        ls
+

pkg/flux/templates/components.sh

@@ -53,6 +53,16 @@ else
 fi
 {{end}}
 
+{{define "custom-script"}}
+{{ if .Container.Commands.Script }}
+cat <<EOF > ./custom-entrypoint.sh
+{{ .Container.Commands.Script }}
+EOF
+chmod +x ./custom-entrypoint.sh
+command="/bin/bash ./custom-entrypoint.sh"
+{{end}}
+{{end}}
+
 {{define "paths"}}
 foundroot=$(find $viewroot -maxdepth 2 -type d -path $viewroot/lib/python3\*) {{ if .Spec.Logging.Quiet }}> /dev/null 2>&1{{ end }}
 pythonversion=$(basename ${foundroot})

pkg/flux/templates/start.sh

@@ -26,6 +26,9 @@ fluxsocket=${viewroot}/run/flux/local
 # Ensure fluxsocket has local
 fluxsocket="local://$fluxsocket"
 
+# Is a custom script provided?
+{{template "custom-script" .}}
+
 {{ .Container.Command }}
 
 {{ .Container.Commands.Post}}

pkg/flux/templates/wait.sh

@@ -22,6 +22,9 @@ fluxuid=$(id -u $fluxuser)
 cfg="${viewroot}/etc/flux/config"
 command="{{ .Container.Command }}"
 
+# Is a custom script provided? This will override command
+{{template "custom-script" .}}
+
 {{ if not .Spec.Logging.Quiet }}
 echo 
 echo "Hello user ${fluxuser}"{{ end }}

sdk/python/v1alpha2/docs/Commands.md

@@ -9,6 +9,7 @@ Name | Type | Description | Notes
 **post** | **str** | post command is run in the entrypoint when the broker exits / finishes | [optional] [default to '']
 **pre** | **str** | pre command is run after global PreCommand, after asFlux is set (can override) | [optional] [default to '']
 **prefix** | **str** | Prefix to flux start / submit / broker Typically used for a wrapper command to mount, etc. | [optional] [default to '']
+**script** | **str** | Custom script for submit (e.g., multiple lines) | [optional] [default to '']
 **service_pre** | **str** | A command only for service start.sh tor run | [optional] [default to '']
 **worker_pre** | **str** | A command only for workers to run | [optional] [default to '']
 

sdk/python/v1alpha2/fluxoperator/models/commands.py

@@ -41,6 +41,7 @@ class Commands(object):
         'post': 'str',
         'pre': 'str',
         'prefix': 'str',
+        'script': 'str',
         'service_pre': 'str',
         'worker_pre': 'str'
     }
@@ -51,11 +52,12 @@ class Commands(object):
         'post': 'post',
         'pre': 'pre',
         'prefix': 'prefix',
+        'script': 'script',
         'service_pre': 'servicePre',
         'worker_pre': 'workerPre'
     }
 
-    def __init__(self, broker_pre='', init='', post='', pre='', prefix='', service_pre='', worker_pre='', local_vars_configuration=None):  # noqa: E501
+    def __init__(self, broker_pre='', init='', post='', pre='', prefix='', script='', service_pre='', worker_pre='', local_vars_configuration=None):  # noqa: E501
         """Commands - a model defined in OpenAPI"""  # noqa: E501
         if local_vars_configuration is None:
             local_vars_configuration = Configuration.get_default_copy()
@@ -66,6 +68,7 @@ def __init__(self, broker_pre='', init='', post='', pre='', prefix='', service_p
         self._post = None
         self._pre = None
         self._prefix = None
+        self._script = None
         self._service_pre = None
         self._worker_pre = None
         self.discriminator = None
@@ -80,6 +83,8 @@ def __init__(self, broker_pre='', init='', post='', pre='', prefix='', service_p
             self.pre = pre
         if prefix is not None:
             self.prefix = prefix
+        if script is not None:
+            self.script = script
         if service_pre is not None:
             self.service_pre = service_pre
         if worker_pre is not None:
@@ -200,6 +205,29 @@ def prefix(self, prefix):
 
         self._prefix = prefix
 
+    @property
+    def script(self):
+        """Gets the script of this Commands.  # noqa: E501
+
+        Custom script for submit (e.g., multiple lines)  # noqa: E501
+
+        :return: The script of this Commands.  # noqa: E501
+        :rtype: str
+        """
+        return self._script
+
+    @script.setter
+    def script(self, script):
+        """Sets the script of this Commands.
+
+        Custom script for submit (e.g., multiple lines)  # noqa: E501
+
+        :param script: The script of this Commands.  # noqa: E501
+        :type script: str
+        """
+
+        self._script = script
+
     @property
     def service_pre(self):
         """Gets the service_pre of this Commands.  # noqa: E501