From 4e7d840c2523b4fc21e5ce21b8db27a7ffc99715 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= <nat@nullable.se>
Date: Fri, 9 Feb 2024 20:29:58 +0100
Subject: [PATCH 01/18] Basic ability to create and attach a debug container

---
 Cargo.lock                                    |  84 +++++++++++-
 Cargo.toml                                    |   2 +-
 .../stackablectl/partials/commands/index.adoc |   1 +
 extra/completions/_stackablectl               |  41 ++++++
 extra/completions/stackablectl.bash           |  94 ++++++++++++-
 extra/completions/stackablectl.fish           |  31 +++--
 extra/man/stackablectl.1                      |   2 +
 rust/stackablectl/Cargo.toml                  |   3 +
 rust/stackablectl/README.md                   |   1 +
 rust/stackablectl/src/cli/mod.rs              |   8 +-
 rust/stackablectl/src/cmds/debug.rs           | 125 ++++++++++++++++++
 rust/stackablectl/src/cmds/mod.rs             |   1 +
 12 files changed, 380 insertions(+), 13 deletions(-)
 create mode 100644 rust/stackablectl/src/cmds/debug.rs

diff --git a/Cargo.lock b/Cargo.lock
index 2629edbb..d6a1a732 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -628,6 +628,12 @@ dependencies = [
  "syn 2.0.39",
 ]
 
+[[package]]
+name = "data-encoding"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5"
+
 [[package]]
 name = "delegate"
 version = "0.12.0"
@@ -1368,6 +1374,7 @@ dependencies = [
  "kube-core",
  "pem",
  "pin-project",
+ "rand",
  "rustls",
  "rustls-pemfile",
  "secrecy",
@@ -1376,6 +1383,7 @@ dependencies = [
  "serde_yaml",
  "thiserror",
  "tokio",
+ "tokio-tungstenite",
  "tokio-util",
  "tower",
  "tower-http",
@@ -1484,6 +1492,17 @@ dependencies = [
  "redox_syscall",
 ]
 
+[[package]]
+name = "libredox"
+version = "0.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3af92c55d7d839293953fcd0fda5ecfe93297cfde6ffbdec13b41d99c0ba6607"
+dependencies = [
+ "bitflags 2.4.1",
+ "libc",
+ "redox_syscall",
+]
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.4.11"
@@ -1608,6 +1627,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "numtoa"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef"
+
 [[package]]
 name = "object"
 version = "0.32.1"
@@ -2004,6 +2029,12 @@ dependencies = [
  "bitflags 1.3.2",
 ]
 
+[[package]]
+name = "redox_termios"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "20145670ba436b55d91fc92d25e71160fbfbdd57831631c8d7d36377a476f1cb"
+
 [[package]]
 name = "redox_users"
 version = "0.4.4"
@@ -2011,7 +2042,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4"
 dependencies = [
  "getrandom",
- "libredox",
+ "libredox 0.0.1",
  "thiserror",
 ]
 
@@ -2738,6 +2769,7 @@ dependencies = [
  "comfy-table",
  "directories",
  "dotenvy",
+ "futures",
  "indexmap 2.1.0",
  "lazy_static",
  "rand",
@@ -2750,6 +2782,7 @@ dependencies = [
  "stackable-cockpit",
  "stackable-operator",
  "tera",
+ "termion",
  "tokio",
  "tracing",
  "tracing-subscriber",
@@ -2860,6 +2893,18 @@ dependencies = [
  "unic-segment",
 ]
 
+[[package]]
+name = "termion"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "417813675a504dfbbf21bfde32c03e5bf9f2413999962b479023c02848c1c7a5"
+dependencies = [
+ "libc",
+ "libredox 0.0.2",
+ "numtoa",
+ "redox_termios",
+]
+
 [[package]]
 name = "thiserror"
 version = "1.0.50"
@@ -2987,6 +3032,18 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-tungstenite"
+version = "0.20.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c"
+dependencies = [
+ "futures-util",
+ "log",
+ "tokio",
+ "tungstenite",
+]
+
 [[package]]
 name = "tokio-util"
 version = "0.7.10"
@@ -3158,6 +3215,25 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
 
+[[package]]
+name = "tungstenite"
+version = "0.20.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9"
+dependencies = [
+ "byteorder",
+ "bytes",
+ "data-encoding",
+ "http",
+ "httparse",
+ "log",
+ "rand",
+ "sha1",
+ "thiserror",
+ "url",
+ "utf-8",
+]
+
 [[package]]
 name = "typenum"
 version = "1.17.0"
@@ -3291,6 +3367,12 @@ version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
 
+[[package]]
+name = "utf-8"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
+
 [[package]]
 name = "utf8parse"
 version = "0.2.1"
diff --git a/Cargo.toml b/Cargo.toml
index 39213904..deee7c5a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -28,7 +28,7 @@ dotenvy = "0.15"
 futures = "0.3"
 indexmap = { version = "2.0", features = ["serde"] }
 k8s-openapi = { version = "0.20", default-features = false, features = ["v1_28"] }
-kube = { version = "0.87", default-features = false, features = ["client", "rustls-tls"] }
+kube = { version = "0.87", default-features = false, features = ["client", "rustls-tls", "ws"] }
 lazy_static = "1.4"
 once_cell = "1.18"
 phf = "0.11"
diff --git a/docs/modules/stackablectl/partials/commands/index.adoc b/docs/modules/stackablectl/partials/commands/index.adoc
index b54a62c6..3a68baf8 100644
--- a/docs/modules/stackablectl/partials/commands/index.adoc
+++ b/docs/modules/stackablectl/partials/commands/index.adoc
@@ -13,6 +13,7 @@ Commands:
   demo         Interact with demos, which are end-to-end usage demonstrations of the Stackable data platform
   completions  Generate shell completions for this tool
   cache        Interact with locally cached files
+  debug
   help         Print this message or the help of the given subcommand(s)
 
 Options:
diff --git a/extra/completions/_stackablectl b/extra/completions/_stackablectl
index 06b06a23..60be51e5 100644
--- a/extra/completions/_stackablectl
+++ b/extra/completions/_stackablectl
@@ -1122,6 +1122,31 @@ esac
     ;;
 esac
 ;;
+(debug)
+_arguments "${_arguments_options[@]}" \
+'-n+[]:NAMESPACE: ' \
+'--namespace=[]:NAMESPACE: ' \
+'--image=[]:IMAGE: ' \
+'-l+[Log level this application uses]:LOG_LEVEL: ' \
+'--log-level=[Log level this application uses]:LOG_LEVEL: ' \
+'*-d+[Provide one or more additional (custom) demo file(s)]:DEMO_FILE:_files' \
+'*--demo-file=[Provide one or more additional (custom) demo file(s)]:DEMO_FILE:_files' \
+'*-s+[Provide one or more additional (custom) stack file(s)]:STACK_FILE:_files' \
+'*--stack-file=[Provide one or more additional (custom) stack file(s)]:STACK_FILE:_files' \
+'*-r+[Provide one or more additional (custom) release file(s)]:RELEASE_FILE:_files' \
+'*--release-file=[Provide one or more additional (custom) release file(s)]:RELEASE_FILE:_files' \
+'--helm-repo-stable=[Provide a custom Helm stable repository URL]:URL:_urls' \
+'--helm-repo-test=[Provide a custom Helm test repository URL]:URL:_urls' \
+'--helm-repo-dev=[Provide a custom Helm dev repository URL]:URL:_urls' \
+'--no-cache[Do not cache the remote (default) demo, stack and release files]' \
+'--offline[Do not request any remote files via the network]' \
+'-h[Print help (see more with '\''--help'\'')]' \
+'--help[Print help (see more with '\''--help'\'')]' \
+'-V[Print version]' \
+'--version[Print version]' \
+':pod:' \
+&& ret=0
+;;
 (help)
 _arguments "${_arguments_options[@]}" \
 ":: :_stackablectl__help_commands" \
@@ -1334,6 +1359,10 @@ _arguments "${_arguments_options[@]}" \
     ;;
 esac
 ;;
+(debug)
+_arguments "${_arguments_options[@]}" \
+&& ret=0
+;;
 (help)
 _arguments "${_arguments_options[@]}" \
 && ret=0
@@ -1357,6 +1386,7 @@ _stackablectl_commands() {
 'demo:Interact with demos, which are end-to-end usage demonstrations of the Stackable data platform' \
 'completions:Generate shell completions for this tool' \
 'cache:Interact with locally cached files' \
+'debug:' \
 'help:Print this message or the help of the given subcommand(s)' \
     )
     _describe -t commands 'stackablectl commands' commands "$@"
@@ -1442,6 +1472,16 @@ _stackablectl__stacklet__help__credentials_commands() {
     local commands; commands=()
     _describe -t commands 'stackablectl stacklet help credentials commands' commands "$@"
 }
+(( $+functions[_stackablectl__debug_commands] )) ||
+_stackablectl__debug_commands() {
+    local commands; commands=()
+    _describe -t commands 'stackablectl debug commands' commands "$@"
+}
+(( $+functions[_stackablectl__help__debug_commands] )) ||
+_stackablectl__help__debug_commands() {
+    local commands; commands=()
+    _describe -t commands 'stackablectl help debug commands' commands "$@"
+}
 (( $+functions[_stackablectl__demo_commands] )) ||
 _stackablectl__demo_commands() {
     local commands; commands=(
@@ -1590,6 +1630,7 @@ _stackablectl__help_commands() {
 'demo:Interact with demos, which are end-to-end usage demonstrations of the Stackable data platform' \
 'completions:Generate shell completions for this tool' \
 'cache:Interact with locally cached files' \
+'debug:' \
 'help:Print this message or the help of the given subcommand(s)' \
     )
     _describe -t commands 'stackablectl help commands' commands "$@"
diff --git a/extra/completions/stackablectl.bash b/extra/completions/stackablectl.bash
index d24888ae..75cc4eab 100644
--- a/extra/completions/stackablectl.bash
+++ b/extra/completions/stackablectl.bash
@@ -18,6 +18,9 @@ _stackablectl() {
             stackablectl,completions)
                 cmd="stackablectl__completions"
                 ;;
+            stackablectl,debug)
+                cmd="stackablectl__debug"
+                ;;
             stackablectl,demo)
                 cmd="stackablectl__demo"
                 ;;
@@ -108,6 +111,9 @@ _stackablectl() {
             stackablectl__help,completions)
                 cmd="stackablectl__help__completions"
                 ;;
+            stackablectl__help,debug)
+                cmd="stackablectl__help__debug"
+                ;;
             stackablectl__help,demo)
                 cmd="stackablectl__help__demo"
                 ;;
@@ -307,7 +313,7 @@ _stackablectl() {
 
     case "${cmd}" in
         stackablectl)
-            opts="-l -d -s -r -h -V --log-level --no-cache --offline --demo-file --stack-file --release-file --helm-repo-stable --helm-repo-test --helm-repo-dev --help --version operator release stack stacklet demo completions cache help"
+            opts="-l -d -s -r -h -V --log-level --no-cache --offline --demo-file --stack-file --release-file --helm-repo-stable --helm-repo-test --helm-repo-dev --help --version operator release stack stacklet demo completions cache debug help"
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -896,6 +902,76 @@ _stackablectl() {
             COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
             return 0
             ;;
+        stackablectl__debug)
+            opts="-n -l -d -s -r -h -V --namespace --image --log-level --no-cache --offline --demo-file --stack-file --release-file --helm-repo-stable --helm-repo-test --helm-repo-dev --help --version <POD>"
+            if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
+                COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+                return 0
+            fi
+            case "${prev}" in
+                --namespace)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -n)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --image)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --log-level)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -l)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --demo-file)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -d)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --stack-file)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -s)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --release-file)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -r)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --helm-repo-stable)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --helm-repo-test)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --helm-repo-dev)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                *)
+                    COMPREPLY=()
+                    ;;
+            esac
+            COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+            return 0
+            ;;
         stackablectl__demo)
             opts="-l -d -s -r -h -V --log-level --no-cache --offline --demo-file --stack-file --release-file --helm-repo-stable --helm-repo-test --helm-repo-dev --help --version list describe install help"
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
@@ -1263,7 +1339,7 @@ _stackablectl() {
             return 0
             ;;
         stackablectl__help)
-            opts="operator release stack stacklet demo completions cache help"
+            opts="operator release stack stacklet demo completions cache debug help"
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -1374,6 +1450,20 @@ _stackablectl() {
             COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
             return 0
             ;;
+        stackablectl__help__debug)
+            opts=""
+            if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then
+                COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+                return 0
+            fi
+            case "${prev}" in
+                *)
+                    COMPREPLY=()
+                    ;;
+            esac
+            COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+            return 0
+            ;;
         stackablectl__help__demo)
             opts="list describe install"
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then
diff --git a/extra/completions/stackablectl.fish b/extra/completions/stackablectl.fish
index c34bb539..50c4db3a 100644
--- a/extra/completions/stackablectl.fish
+++ b/extra/completions/stackablectl.fish
@@ -16,6 +16,7 @@ complete -c stackablectl -n "__fish_use_subcommand" -f -a "stacklet" -d 'Interac
 complete -c stackablectl -n "__fish_use_subcommand" -f -a "demo" -d 'Interact with demos, which are end-to-end usage demonstrations of the Stackable data platform'
 complete -c stackablectl -n "__fish_use_subcommand" -f -a "completions" -d 'Generate shell completions for this tool'
 complete -c stackablectl -n "__fish_use_subcommand" -f -a "cache" -d 'Interact with locally cached files'
+complete -c stackablectl -n "__fish_use_subcommand" -f -a "debug"
 complete -c stackablectl -n "__fish_use_subcommand" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
 complete -c stackablectl -n "__fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from list; and not __fish_seen_subcommand_from describe; and not __fish_seen_subcommand_from install; and not __fish_seen_subcommand_from uninstall; and not __fish_seen_subcommand_from installed; and not __fish_seen_subcommand_from help" -s l -l log-level -d 'Log level this application uses' -r
 complete -c stackablectl -n "__fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from list; and not __fish_seen_subcommand_from describe; and not __fish_seen_subcommand_from install; and not __fish_seen_subcommand_from uninstall; and not __fish_seen_subcommand_from installed; and not __fish_seen_subcommand_from help" -s d -l demo-file -d 'Provide one or more additional (custom) demo file(s)' -r -F
@@ -441,14 +442,28 @@ complete -c stackablectl -n "__fish_seen_subcommand_from cache; and __fish_seen_
 complete -c stackablectl -n "__fish_seen_subcommand_from cache; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from list; and not __fish_seen_subcommand_from clean; and not __fish_seen_subcommand_from help" -f -a "list" -d 'List cached files'
 complete -c stackablectl -n "__fish_seen_subcommand_from cache; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from list; and not __fish_seen_subcommand_from clean; and not __fish_seen_subcommand_from help" -f -a "clean" -d 'Clean cached files'
 complete -c stackablectl -n "__fish_seen_subcommand_from cache; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from list; and not __fish_seen_subcommand_from clean; and not __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
-complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from help" -f -a "operator" -d 'Interact with single operator instead of the full platform'
-complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from help" -f -a "release" -d 'Interact with all operators of the platform which are released together'
-complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from help" -f -a "stack" -d 'Interact with stacks, which are ready-to-use product combinations'
-complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from help" -f -a "stacklet" -d 'Interact with deployed stacklets, which are bundles of resources and containers required to run the product'
-complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from help" -f -a "demo" -d 'Interact with demos, which are end-to-end usage demonstrations of the Stackable data platform'
-complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from help" -f -a "completions" -d 'Generate shell completions for this tool'
-complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from help" -f -a "cache" -d 'Interact with locally cached files'
-complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
+complete -c stackablectl -n "__fish_seen_subcommand_from debug" -s n -l namespace -r
+complete -c stackablectl -n "__fish_seen_subcommand_from debug" -l image -r
+complete -c stackablectl -n "__fish_seen_subcommand_from debug" -s l -l log-level -d 'Log level this application uses' -r
+complete -c stackablectl -n "__fish_seen_subcommand_from debug" -s d -l demo-file -d 'Provide one or more additional (custom) demo file(s)' -r -F
+complete -c stackablectl -n "__fish_seen_subcommand_from debug" -s s -l stack-file -d 'Provide one or more additional (custom) stack file(s)' -r -F
+complete -c stackablectl -n "__fish_seen_subcommand_from debug" -s r -l release-file -d 'Provide one or more additional (custom) release file(s)' -r -F
+complete -c stackablectl -n "__fish_seen_subcommand_from debug" -l helm-repo-stable -d 'Provide a custom Helm stable repository URL' -r -f
+complete -c stackablectl -n "__fish_seen_subcommand_from debug" -l helm-repo-test -d 'Provide a custom Helm test repository URL' -r -f
+complete -c stackablectl -n "__fish_seen_subcommand_from debug" -l helm-repo-dev -d 'Provide a custom Helm dev repository URL' -r -f
+complete -c stackablectl -n "__fish_seen_subcommand_from debug" -l no-cache -d 'Do not cache the remote (default) demo, stack and release files'
+complete -c stackablectl -n "__fish_seen_subcommand_from debug" -l offline -d 'Do not request any remote files via the network'
+complete -c stackablectl -n "__fish_seen_subcommand_from debug" -s h -l help -d 'Print help (see more with \'--help\')'
+complete -c stackablectl -n "__fish_seen_subcommand_from debug" -s V -l version -d 'Print version'
+complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from help" -f -a "operator" -d 'Interact with single operator instead of the full platform'
+complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from help" -f -a "release" -d 'Interact with all operators of the platform which are released together'
+complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from help" -f -a "stack" -d 'Interact with stacks, which are ready-to-use product combinations'
+complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from help" -f -a "stacklet" -d 'Interact with deployed stacklets, which are bundles of resources and containers required to run the product'
+complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from help" -f -a "demo" -d 'Interact with demos, which are end-to-end usage demonstrations of the Stackable data platform'
+complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from help" -f -a "completions" -d 'Generate shell completions for this tool'
+complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from help" -f -a "cache" -d 'Interact with locally cached files'
+complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from help" -f -a "debug"
+complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
 complete -c stackablectl -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from list; and not __fish_seen_subcommand_from describe; and not __fish_seen_subcommand_from install; and not __fish_seen_subcommand_from uninstall; and not __fish_seen_subcommand_from installed" -f -a "list" -d 'List available operators'
 complete -c stackablectl -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from list; and not __fish_seen_subcommand_from describe; and not __fish_seen_subcommand_from install; and not __fish_seen_subcommand_from uninstall; and not __fish_seen_subcommand_from installed" -f -a "describe" -d 'Print out detailed operator information'
 complete -c stackablectl -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from list; and not __fish_seen_subcommand_from describe; and not __fish_seen_subcommand_from install; and not __fish_seen_subcommand_from uninstall; and not __fish_seen_subcommand_from installed" -f -a "install" -d 'Install one or more operators'
diff --git a/extra/man/stackablectl.1 b/extra/man/stackablectl.1
index 28315bf3..48daac8f 100644
--- a/extra/man/stackablectl.1
+++ b/extra/man/stackablectl.1
@@ -92,6 +92,8 @@ Generate shell completions for this tool
 stackablectl\-cache(1)
 Interact with locally cached files
 .TP
+stackablectl\-debug(1)
+.TP
 stackablectl\-help(1)
 Print this message or the help of the given subcommand(s)
 .SH VERSION
diff --git a/rust/stackablectl/Cargo.toml b/rust/stackablectl/Cargo.toml
index 1e1c681a..a1ba2a28 100644
--- a/rust/stackablectl/Cargo.toml
+++ b/rust/stackablectl/Cargo.toml
@@ -29,5 +29,8 @@ serde.workspace = true
 snafu.workspace = true
 tera.workspace = true
 tokio.workspace = true
+tokio.features = ["io-std"]
 tracing-subscriber.workspace = true
 tracing.workspace = true
+futures.workspace = true
+termion = "3.0.0"
diff --git a/rust/stackablectl/README.md b/rust/stackablectl/README.md
index 54e6de31..a74ca108 100644
--- a/rust/stackablectl/README.md
+++ b/rust/stackablectl/README.md
@@ -22,6 +22,7 @@ Commands:
   demo         Interact with demos, which are end-to-end usage demonstrations of the Stackable data platform
   completions  Generate shell completions for this tool
   cache        Interact with locally cached files
+  debug
   help         Print this message or the help of the given subcommand(s)
 
 Options:
diff --git a/rust/stackablectl/src/cli/mod.rs b/rust/stackablectl/src/cli/mod.rs
index bf3a50a5..ef5b2cf0 100644
--- a/rust/stackablectl/src/cli/mod.rs
+++ b/rust/stackablectl/src/cli/mod.rs
@@ -17,7 +17,7 @@ use stackable_cockpit::{
 
 use crate::{
     args::{CommonFileArgs, CommonRepoArgs},
-    cmds::{cache, completions, demo, operator, release, stack, stacklet},
+    cmds::{cache, completions, debug, demo, operator, release, stack, stacklet},
     constants::{
         ENV_KEY_DEMO_FILES, ENV_KEY_RELEASE_FILES, ENV_KEY_STACK_FILES, REMOTE_DEMO_FILE,
         REMOTE_RELEASE_FILE, REMOTE_STACK_FILE, USER_DIR_APPLICATION_NAME,
@@ -49,6 +49,9 @@ pub enum Error {
     #[snafu(display("cache command error"))]
     Cache { source: cache::CmdError },
 
+    #[snafu(display("debug command error"))]
+    Debug { source: debug::CmdError },
+
     #[snafu(display("helm error"))]
     Helm { source: helm::Error },
 }
@@ -191,6 +194,7 @@ impl Cli {
             Commands::Demo(args) => args.run(self, cache).await.context(DemoSnafu),
             Commands::Completions(args) => args.run().context(CompletionsSnafu),
             Commands::Cache(args) => args.run(self, cache).await.context(CacheSnafu),
+            Commands::Debug(args) => args.run(self).await.context(DebugSnafu),
         }
     }
 
@@ -240,6 +244,8 @@ CRDs."
 
     /// Interact with locally cached files
     Cache(cache::CacheArgs),
+
+    Debug(debug::DebugArgs),
 }
 
 #[derive(Clone, Debug, Default, ValueEnum)]
diff --git a/rust/stackablectl/src/cmds/debug.rs b/rust/stackablectl/src/cmds/debug.rs
new file mode 100644
index 00000000..5fa28519
--- /dev/null
+++ b/rust/stackablectl/src/cmds/debug.rs
@@ -0,0 +1,125 @@
+use std::pin::pin;
+
+use clap::Args;
+use rand::Rng;
+use snafu::{ResultExt, Snafu};
+use stackable_operator::{
+    k8s_openapi::api::core::v1::{ContainerStatus, EphemeralContainer, Pod, PodSpec},
+    kube::{
+        self,
+        api::{AttachParams, PatchParams},
+    },
+};
+use termion::raw::IntoRawMode;
+use tracing::info;
+
+use crate::cli::Cli;
+
+#[derive(Debug, Snafu)]
+pub enum CmdError {
+    Attach { source: kube::Error },
+}
+
+#[derive(Debug, Args)]
+pub struct DebugArgs {
+    #[clap(long, short)]
+    namespace: String,
+    pod: String,
+    #[clap(long)]
+    image: String,
+}
+
+impl DebugArgs {
+    pub async fn run(&self, _cli: &Cli) -> Result<String, CmdError> {
+        let kube = kube::Client::try_default().await.unwrap();
+        let pods = kube::Api::<Pod>::namespaced(kube, &self.namespace);
+        let mut rng = rand::thread_rng();
+        let mut debug_container_name = "sble-debug-".to_string();
+        for _ in 0..5 {
+            debug_container_name.push(rng.gen_range('a'..='z'));
+        }
+        info!(
+            container.name = debug_container_name,
+            "Creating debug container"
+        );
+        let pod = Pod {
+            spec: Some(PodSpec {
+                ephemeral_containers: Some(vec![EphemeralContainer {
+                    name: debug_container_name.clone(),
+                    image: Some(self.image.clone()),
+                    tty: Some(true),
+                    stdin: Some(true),
+                    ..Default::default()
+                }]),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+        pods.patch_ephemeral_containers(
+            &self.pod,
+            &PatchParams::default(),
+            &kube::api::Patch::Strategic(pod),
+        )
+        .await
+        .unwrap();
+        info!(
+            container.name = debug_container_name,
+            "Waiting for container to start"
+        );
+        let ready_pod =
+            kube::runtime::wait::await_condition(pods.clone(), &self.pod, |pod: Option<&Pod>| {
+                pod.and_then(debug_container_status_of_pod(&debug_container_name))
+                    .and_then(|c| Some(c.state.as_ref()?.waiting.is_none()))
+                    .unwrap_or_default()
+            })
+            .await
+            .unwrap();
+        dbg!(ready_pod
+            .as_ref()
+            .and_then(debug_container_status_of_pod(&debug_container_name)));
+        info!(
+            container.name = debug_container_name,
+            "Attaching to container"
+        );
+        let mut attachment = pods
+            .attach(
+                &self.pod,
+                &AttachParams::interactive_tty().container(debug_container_name),
+            )
+            .await
+            .context(AttachSnafu)?;
+        info!("Attached to container, if the shell line looks empty, press ENTER!");
+        {
+            let _raw = std::io::stdout().into_raw_mode().unwrap();
+            futures::future::select(
+                pin!(tokio::io::copy(
+                    &mut attachment.stdout().unwrap(),
+                    &mut tokio::io::stdout()
+                )),
+                pin!(tokio::io::copy(
+                    &mut tokio::io::stdin(),
+                    &mut attachment.stdin().unwrap()
+                )),
+            )
+            .await
+            .factor_first()
+            .0
+            .unwrap();
+        }
+        // FIXME: Terminate the process to avoid Tokio hogging stdin forever
+        std::process::exit(0);
+    }
+}
+
+fn debug_container_status_of_pod(
+    debug_container_name: &str,
+) -> impl for<'a> Fn(&'a Pod) -> Option<&'a ContainerStatus> + '_ {
+    move |pod: &Pod| {
+        pod.status
+            .as_ref()?
+            .ephemeral_container_statuses
+            .as_ref()?
+            .iter()
+            .find(|c| c.name == debug_container_name)
+    }
+}
diff --git a/rust/stackablectl/src/cmds/mod.rs b/rust/stackablectl/src/cmds/mod.rs
index a19d1112..039e1fd8 100644
--- a/rust/stackablectl/src/cmds/mod.rs
+++ b/rust/stackablectl/src/cmds/mod.rs
@@ -1,5 +1,6 @@
 pub mod cache;
 pub mod completions;
+pub mod debug;
 pub mod demo;
 pub mod operator;
 pub mod release;

From 7b2a24961777b77cc886701e34613e047b6ce1a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= <nat@nullable.se>
Date: Fri, 9 Feb 2024 21:10:50 +0100
Subject: [PATCH 02/18] Add support for copying env from a target container, as
 well as running a custom command

---
 extra/completions/_stackablectl     |  3 ++
 extra/completions/stackablectl.bash | 10 +++++-
 extra/completions/stackablectl.fish |  1 +
 rust/stackablectl/src/cmds/debug.rs | 48 +++++++++++++++++++++++++----
 4 files changed, 55 insertions(+), 7 deletions(-)

diff --git a/extra/completions/_stackablectl b/extra/completions/_stackablectl
index 60be51e5..304a9ff8 100644
--- a/extra/completions/_stackablectl
+++ b/extra/completions/_stackablectl
@@ -1126,6 +1126,8 @@ esac
 _arguments "${_arguments_options[@]}" \
 '-n+[]:NAMESPACE: ' \
 '--namespace=[]:NAMESPACE: ' \
+'-c+[]:CONTAINER: ' \
+'--container=[]:CONTAINER: ' \
 '--image=[]:IMAGE: ' \
 '-l+[Log level this application uses]:LOG_LEVEL: ' \
 '--log-level=[Log level this application uses]:LOG_LEVEL: ' \
@@ -1145,6 +1147,7 @@ _arguments "${_arguments_options[@]}" \
 '-V[Print version]' \
 '--version[Print version]' \
 ':pod:' \
+'*::cmd:' \
 && ret=0
 ;;
 (help)
diff --git a/extra/completions/stackablectl.bash b/extra/completions/stackablectl.bash
index 75cc4eab..e213b976 100644
--- a/extra/completions/stackablectl.bash
+++ b/extra/completions/stackablectl.bash
@@ -903,7 +903,7 @@ _stackablectl() {
             return 0
             ;;
         stackablectl__debug)
-            opts="-n -l -d -s -r -h -V --namespace --image --log-level --no-cache --offline --demo-file --stack-file --release-file --helm-repo-stable --helm-repo-test --helm-repo-dev --help --version <POD>"
+            opts="-n -c -l -d -s -r -h -V --namespace --container --image --log-level --no-cache --offline --demo-file --stack-file --release-file --helm-repo-stable --helm-repo-test --helm-repo-dev --help --version <POD> [CMD]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -917,6 +917,14 @@ _stackablectl() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
+                --container)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -c)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
                 --image)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
diff --git a/extra/completions/stackablectl.fish b/extra/completions/stackablectl.fish
index 50c4db3a..0197b451 100644
--- a/extra/completions/stackablectl.fish
+++ b/extra/completions/stackablectl.fish
@@ -443,6 +443,7 @@ complete -c stackablectl -n "__fish_seen_subcommand_from cache; and __fish_seen_
 complete -c stackablectl -n "__fish_seen_subcommand_from cache; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from list; and not __fish_seen_subcommand_from clean; and not __fish_seen_subcommand_from help" -f -a "clean" -d 'Clean cached files'
 complete -c stackablectl -n "__fish_seen_subcommand_from cache; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from list; and not __fish_seen_subcommand_from clean; and not __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
 complete -c stackablectl -n "__fish_seen_subcommand_from debug" -s n -l namespace -r
+complete -c stackablectl -n "__fish_seen_subcommand_from debug" -s c -l container -r
 complete -c stackablectl -n "__fish_seen_subcommand_from debug" -l image -r
 complete -c stackablectl -n "__fish_seen_subcommand_from debug" -s l -l log-level -d 'Log level this application uses' -r
 complete -c stackablectl -n "__fish_seen_subcommand_from debug" -s d -l demo-file -d 'Provide one or more additional (custom) demo file(s)' -r -F
diff --git a/rust/stackablectl/src/cmds/debug.rs b/rust/stackablectl/src/cmds/debug.rs
index 5fa28519..68caaea1 100644
--- a/rust/stackablectl/src/cmds/debug.rs
+++ b/rust/stackablectl/src/cmds/debug.rs
@@ -11,7 +11,7 @@ use stackable_operator::{
     },
 };
 use termion::raw::IntoRawMode;
-use tracing::info;
+use tracing::{error, info};
 
 use crate::cli::Cli;
 
@@ -25,8 +25,12 @@ pub struct DebugArgs {
     #[clap(long, short)]
     namespace: String,
     pod: String,
+    #[clap(long, short)]
+    container: String,
     #[clap(long)]
     image: String,
+    #[clap(last = true)]
+    cmd: Option<Vec<String>>,
 }
 
 impl DebugArgs {
@@ -42,13 +46,29 @@ impl DebugArgs {
             container.name = debug_container_name,
             "Creating debug container"
         );
-        let pod = Pod {
+        let pod = pods.get(&self.pod).await.unwrap();
+        let template_container = pod
+            .spec
+            .as_ref()
+            .and_then(|spec| spec.containers.iter().find(|c| c.name == self.container))
+            .unwrap();
+        let pod_patch = Pod {
             spec: Some(PodSpec {
                 ephemeral_containers: Some(vec![EphemeralContainer {
                     name: debug_container_name.clone(),
                     image: Some(self.image.clone()),
                     tty: Some(true),
                     stdin: Some(true),
+
+                    command: self.cmd.clone(),
+                    args: self.cmd.is_some().then(Vec::new),
+
+                    // copy environment from template
+                    env: template_container.env.clone(),
+                    env_from: template_container.env_from.clone(),
+                    volume_mounts: template_container.volume_mounts.clone(),
+                    volume_devices: template_container.volume_devices.clone(),
+
                     ..Default::default()
                 }]),
                 ..Default::default()
@@ -58,7 +78,7 @@ impl DebugArgs {
         pods.patch_ephemeral_containers(
             &self.pod,
             &PatchParams::default(),
-            &kube::api::Patch::Strategic(pod),
+            &kube::api::Patch::Strategic(pod_patch),
         )
         .await
         .unwrap();
@@ -68,15 +88,31 @@ impl DebugArgs {
         );
         let ready_pod =
             kube::runtime::wait::await_condition(pods.clone(), &self.pod, |pod: Option<&Pod>| {
-                pod.and_then(debug_container_status_of_pod(&debug_container_name))
+                let container = pod.and_then(debug_container_status_of_pod(&debug_container_name));
+                container
                     .and_then(|c| Some(c.state.as_ref()?.waiting.is_none()))
                     .unwrap_or_default()
+                    || container
+                        .and_then(|c| c.last_state.as_ref()?.terminated.as_ref())
+                        .is_some()
             })
             .await
             .unwrap();
-        dbg!(ready_pod
+        let debug_container_status = dbg!(ready_pod
+            .as_ref()
+            .and_then(debug_container_status_of_pod(&debug_container_name)))
+        .unwrap();
+        if let Some(termination) = debug_container_status
+            .last_state
             .as_ref()
-            .and_then(debug_container_status_of_pod(&debug_container_name)));
+            .and_then(|state| state.terminated.as_ref())
+        {
+            error!(
+                error = termination.message,
+                exit_code = termination.exit_code,
+                "Debug container failed to start!"
+            );
+        }
         info!(
             container.name = debug_container_name,
             "Attaching to container"

From 87fc7ba3923f53abf3813505bfe8e0ee2cd14199 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= <nat@nullable.se>
Date: Mon, 12 Feb 2024 15:32:07 +0100
Subject: [PATCH 03/18] Send TTY window size

---
 rust/stackablectl/src/cmds/debug.rs | 57 ++++++++++++++++++++---------
 1 file changed, 39 insertions(+), 18 deletions(-)

diff --git a/rust/stackablectl/src/cmds/debug.rs b/rust/stackablectl/src/cmds/debug.rs
index 68caaea1..c70c05fd 100644
--- a/rust/stackablectl/src/cmds/debug.rs
+++ b/rust/stackablectl/src/cmds/debug.rs
@@ -1,16 +1,16 @@
-use std::pin::pin;
-
 use clap::Args;
+use futures::{channel::mpsc::Sender, FutureExt, SinkExt, TryFutureExt};
 use rand::Rng;
 use snafu::{ResultExt, Snafu};
 use stackable_operator::{
     k8s_openapi::api::core::v1::{ContainerStatus, EphemeralContainer, Pod, PodSpec},
     kube::{
         self,
-        api::{AttachParams, PatchParams},
+        api::{AttachParams, PatchParams, TerminalSize},
     },
 };
-use termion::raw::IntoRawMode;
+use termion::{raw::IntoRawMode, terminal_size};
+use tokio::signal::unix::SignalKind;
 use tracing::{error, info};
 
 use crate::cli::Cli;
@@ -98,10 +98,10 @@ impl DebugArgs {
             })
             .await
             .unwrap();
-        let debug_container_status = dbg!(ready_pod
+        let debug_container_status = ready_pod
             .as_ref()
-            .and_then(debug_container_status_of_pod(&debug_container_name)))
-        .unwrap();
+            .and_then(debug_container_status_of_pod(&debug_container_name))
+            .unwrap();
         if let Some(termination) = debug_container_status
             .last_state
             .as_ref()
@@ -127,18 +127,18 @@ impl DebugArgs {
         info!("Attached to container, if the shell line looks empty, press ENTER!");
         {
             let _raw = std::io::stdout().into_raw_mode().unwrap();
-            futures::future::select(
-                pin!(tokio::io::copy(
-                    &mut attachment.stdout().unwrap(),
-                    &mut tokio::io::stdout()
-                )),
-                pin!(tokio::io::copy(
-                    &mut tokio::io::stdin(),
-                    &mut attachment.stdin().unwrap()
-                )),
-            )
+            futures::future::select_all([
+                update_terminal_size(attachment.terminal_size().unwrap())
+                    .map(Ok)
+                    .boxed(),
+                tokio::io::copy(&mut attachment.stdout().unwrap(), &mut tokio::io::stdout())
+                    .map_ok(drop)
+                    .boxed(),
+                tokio::io::copy(&mut tokio::io::stdin(), &mut attachment.stdin().unwrap())
+                    .map_ok(drop)
+                    .boxed(),
+            ])
             .await
-            .factor_first()
             .0
             .unwrap();
         }
@@ -147,6 +147,27 @@ impl DebugArgs {
     }
 }
 
+async fn update_terminal_size(mut tx: Sender<TerminalSize>) {
+    let mut signal = tokio::signal::unix::signal(SignalKind::window_change()).unwrap();
+    {
+        let (width, height) = terminal_size().unwrap();
+        // Make TTY apps re-render by force-changing the terminal size
+        // Start by sending an invalid size so that it's a change no matter
+        // whether the size has actually changed.
+        tx.send(TerminalSize {
+            width: width - 1,
+            height,
+        })
+        .await
+        .unwrap();
+        tx.send(TerminalSize { width, height }).await.unwrap();
+    }
+    while let Some(()) = signal.recv().await {
+        let (width, height) = terminal_size().unwrap();
+        tx.send(TerminalSize { width, height }).await.unwrap();
+    }
+}
+
 fn debug_container_status_of_pod(
     debug_container_name: &str,
 ) -> impl for<'a> Fn(&'a Pod) -> Option<&'a ContainerStatus> + '_ {

From 10be291f5622ebf5f8c6750c6022ea45718206f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= <nat@nullable.se>
Date: Mon, 12 Feb 2024 16:56:26 +0100
Subject: [PATCH 04/18] Read asynchronously from stdin

---
 Cargo.lock                          | 13 +++---
 rust/stackablectl/Cargo.toml        |  1 +
 rust/stackablectl/src/cmds/debug.rs | 70 +++++++++++++++++++++++++++--
 3 files changed, 74 insertions(+), 10 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index d6a1a732..2dbbc1db 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1461,9 +1461,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
 [[package]]
 name = "libc"
-version = "0.2.150"
+version = "0.2.153"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c"
+checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
 
 [[package]]
 name = "libloading"
@@ -2772,6 +2772,7 @@ dependencies = [
  "futures",
  "indexmap 2.1.0",
  "lazy_static",
+ "libc",
  "rand",
  "reqwest",
  "semver",
@@ -2974,9 +2975,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.33.0"
+version = "1.36.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653"
+checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931"
 dependencies = [
  "backtrace",
  "bytes",
@@ -3002,9 +3003,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-macros"
-version = "2.1.0"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
+checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
 dependencies = [
  "proc-macro2",
  "quote",
diff --git a/rust/stackablectl/Cargo.toml b/rust/stackablectl/Cargo.toml
index a1ba2a28..ffda21ef 100644
--- a/rust/stackablectl/Cargo.toml
+++ b/rust/stackablectl/Cargo.toml
@@ -34,3 +34,4 @@ tracing-subscriber.workspace = true
 tracing.workspace = true
 futures.workspace = true
 termion = "3.0.0"
+libc = "0.2.153"
diff --git a/rust/stackablectl/src/cmds/debug.rs b/rust/stackablectl/src/cmds/debug.rs
index c70c05fd..827f15eb 100644
--- a/rust/stackablectl/src/cmds/debug.rs
+++ b/rust/stackablectl/src/cmds/debug.rs
@@ -1,3 +1,9 @@
+use std::{
+    io::{Read, Stdin},
+    os::fd::AsRawFd,
+    task::{ready, Poll},
+};
+
 use clap::Args;
 use futures::{channel::mpsc::Sender, FutureExt, SinkExt, TryFutureExt};
 use rand::Rng;
@@ -10,7 +16,10 @@ use stackable_operator::{
     },
 };
 use termion::{raw::IntoRawMode, terminal_size};
-use tokio::signal::unix::SignalKind;
+use tokio::{
+    io::{unix::AsyncFd, AsyncRead},
+    signal::unix::SignalKind,
+};
 use tracing::{error, info};
 
 use crate::cli::Cli;
@@ -134,7 +143,7 @@ impl DebugArgs {
                 tokio::io::copy(&mut attachment.stdout().unwrap(), &mut tokio::io::stdout())
                     .map_ok(drop)
                     .boxed(),
-                tokio::io::copy(&mut tokio::io::stdin(), &mut attachment.stdin().unwrap())
+                tokio::io::copy(&mut AsyncStdin::new(), &mut attachment.stdin().unwrap())
                     .map_ok(drop)
                     .boxed(),
             ])
@@ -142,8 +151,61 @@ impl DebugArgs {
             .0
             .unwrap();
         }
-        // FIXME: Terminate the process to avoid Tokio hogging stdin forever
-        std::process::exit(0);
+        Ok(String::new())
+    }
+}
+
+/// Does true non-blocking reads of stdin, so that we can cancel properly on shutdown.
+/// The compromise is that it does not handle having things piped into it very well, since their write sides
+/// will also be turned non-blocking.
+///
+/// Also, `AsyncStdin` will currently _not_ restore the mode of stdin when dropped.
+// FIXME: restore status
+struct AsyncStdin {
+    fd: AsyncFd<Stdin>,
+}
+
+impl AsyncStdin {
+    fn new() -> Self {
+        let stdin = std::io::stdin();
+        // Make stdin non-blocking
+        {
+            let old_flags = unsafe { libc::fcntl(stdin.as_raw_fd(), libc::F_GETFL) };
+            if old_flags == -1 {
+                panic!("{:?}", std::io::Error::last_os_error());
+            }
+            let status = unsafe {
+                libc::fcntl(
+                    stdin.as_raw_fd(),
+                    libc::F_SETFL,
+                    old_flags | libc::O_NONBLOCK,
+                )
+            };
+            if status == -1 {
+                panic!("{:?}", std::io::Error::last_os_error());
+            }
+        };
+        Self {
+            fd: AsyncFd::new(stdin).unwrap(),
+        }
+    }
+}
+
+impl AsyncRead for AsyncStdin {
+    fn poll_read(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &mut tokio::io::ReadBuf<'_>,
+    ) -> Poll<std::io::Result<()>> {
+        let mut ready = ready!(self.fd.poll_read_ready_mut(cx)?);
+        match ready.try_io(|r| {
+            let read = r.get_mut().read(buf.initialize_unfilled())?;
+            buf.advance(read);
+            Ok(())
+        }) {
+            Ok(res) => Poll::Ready(res),
+            Err(_would_block) => Poll::Pending,
+        }
     }
 }
 

From d7a85a06ef20a3775c1ce486521ef25e9fb7705c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= <nat@nullable.se>
Date: Fri, 16 Feb 2024 13:16:58 +0100
Subject: [PATCH 05/18] Default to the default namespace

---
 rust/stackablectl/src/cmds/debug.rs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/rust/stackablectl/src/cmds/debug.rs b/rust/stackablectl/src/cmds/debug.rs
index 827f15eb..701b4014 100644
--- a/rust/stackablectl/src/cmds/debug.rs
+++ b/rust/stackablectl/src/cmds/debug.rs
@@ -32,7 +32,7 @@ pub enum CmdError {
 #[derive(Debug, Args)]
 pub struct DebugArgs {
     #[clap(long, short)]
-    namespace: String,
+    namespace: Option<String>,
     pod: String,
     #[clap(long, short)]
     container: String,
@@ -45,7 +45,10 @@ pub struct DebugArgs {
 impl DebugArgs {
     pub async fn run(&self, _cli: &Cli) -> Result<String, CmdError> {
         let kube = kube::Client::try_default().await.unwrap();
-        let pods = kube::Api::<Pod>::namespaced(kube, &self.namespace);
+        let pods = match &self.namespace {
+            Some(ns) => kube::Api::<Pod>::namespaced(kube, ns),
+            None => kube::Api::<Pod>::default_namespaced(kube),
+        };
         let mut rng = rand::thread_rng();
         let mut debug_container_name = "sble-debug-".to_string();
         for _ in 0..5 {

From b6ad59f0369bfd0b09ef2bdc6f5337dd62802da4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= <nat@nullable.se>
Date: Fri, 16 Feb 2024 13:17:44 +0100
Subject: [PATCH 06/18] Fix stdin not being re-registered correctly with the
 waker

---
 rust/stackablectl/src/cmds/debug.rs | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/rust/stackablectl/src/cmds/debug.rs b/rust/stackablectl/src/cmds/debug.rs
index 701b4014..40b19da5 100644
--- a/rust/stackablectl/src/cmds/debug.rs
+++ b/rust/stackablectl/src/cmds/debug.rs
@@ -200,14 +200,19 @@ impl AsyncRead for AsyncStdin {
         cx: &mut std::task::Context<'_>,
         buf: &mut tokio::io::ReadBuf<'_>,
     ) -> Poll<std::io::Result<()>> {
-        let mut ready = ready!(self.fd.poll_read_ready_mut(cx)?);
-        match ready.try_io(|r| {
-            let read = r.get_mut().read(buf.initialize_unfilled())?;
-            buf.advance(read);
-            Ok(())
-        }) {
-            Ok(res) => Poll::Ready(res),
-            Err(_would_block) => Poll::Pending,
+        loop {
+            let mut ready = ready!(self.fd.poll_read_ready_mut(cx)?);
+            break match ready.try_io(|r| {
+                let read = r.get_mut().read(buf.initialize_unfilled())?;
+                buf.advance(read);
+                Ok(())
+            }) {
+                Ok(res) => Poll::Ready(res),
+                Err(_would_block) => {
+                    // Try to poll again, so that we re-register the waker
+                    continue;
+                }
+            };
         }
     }
 }

From ce59f16674e3ac70c0a5b31eaed0b42129d599b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= <nat@nullable.se>
Date: Mon, 19 Feb 2024 16:41:40 +0100
Subject: [PATCH 07/18] Add debug error messages

---
 Cargo.lock                          |   2 +
 Cargo.toml                          |   2 +-
 rust/stackablectl/src/cmds/debug.rs | 189 +++++++++++++++++++++-------
 3 files changed, 149 insertions(+), 44 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 2dbbc1db..d61d164e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2600,6 +2600,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6"
 dependencies = [
  "doc-comment",
+ "futures-core",
+ "pin-project",
  "snafu-derive 0.7.5",
 ]
 
diff --git a/Cargo.toml b/Cargo.toml
index deee7c5a..ca22e305 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -42,7 +42,7 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
 sha2 = "0.10"
-snafu = "0.7"
+snafu = { version = "0.7", features = ["futures"] }
 stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.61.0" }
 tera = "1.18"
 tokio = { version = "1.29.0", features = ["rt-multi-thread", "macros", "fs", "process"] }
diff --git a/rust/stackablectl/src/cmds/debug.rs b/rust/stackablectl/src/cmds/debug.rs
index 40b19da5..e103aec2 100644
--- a/rust/stackablectl/src/cmds/debug.rs
+++ b/rust/stackablectl/src/cmds/debug.rs
@@ -5,14 +5,18 @@ use std::{
 };
 
 use clap::Args;
-use futures::{channel::mpsc::Sender, FutureExt, SinkExt, TryFutureExt};
+use futures::{
+    channel::mpsc::{self, Sender},
+    FutureExt, SinkExt, TryFutureExt,
+};
 use rand::Rng;
-use snafu::{ResultExt, Snafu};
+use snafu::{futures::TryFutureExt as _, OptionExt, ResultExt, Snafu};
 use stackable_operator::{
     k8s_openapi::api::core::v1::{ContainerStatus, EphemeralContainer, Pod, PodSpec},
     kube::{
         self,
         api::{AttachParams, PatchParams, TerminalSize},
+        runtime::reflector::ObjectRef,
     },
 };
 use termion::{raw::IntoRawMode, terminal_size};
@@ -26,8 +30,63 @@ use crate::cli::Cli;
 
 #[derive(Debug, Snafu)]
 pub enum CmdError {
-    Attach { source: kube::Error },
+    #[snafu(display("failed to create Kubernetes client"))]
+    KubeClientCreate { source: kube::Error },
+    #[snafu(display("failed to get {pod}"))]
+    GetPod {
+        source: kube::Error,
+        pod: ObjectRef<Pod>,
+    },
+    #[snafu(display("{pod} has no container {container:?}"))]
+    FindTemplateContainer {
+        pod: ObjectRef<Pod>,
+        container: String,
+    },
+    #[snafu(display("failed to create ephemeral debug container {container:?} on {pod}"))]
+    CreateDebugContainer {
+        source: kube::Error,
+        pod: ObjectRef<Pod>,
+        container: String,
+    },
+    #[snafu(display("debug container {container:?} on {pod} never became ready"))]
+    AwaitDebugContainerReadiness {
+        source: kube::runtime::wait::Error,
+        pod: ObjectRef<Pod>,
+        container: String,
+    },
+    #[snafu(display("failed to get status of debug container {container:?} on {pod}"))]
+    FindDebugContainerStatus {
+        pod: ObjectRef<Pod>,
+        container: String,
+    },
+    #[snafu(display("failed to attach to container {container:?} on {pod}"))]
+    AttachContainer {
+        source: kube::Error,
+        pod: ObjectRef<Pod>,
+        container: String,
+    },
+    #[snafu(display("failed to enable raw local TTY input"))]
+    SetRawTtyMode { source: std::io::Error },
+    #[snafu(display("failed to turn stdin async"))]
+    AsyncifyStdin { source: std::io::Error },
+    #[snafu(display("failed to initialize AsyncFd for stdin"))]
+    AsyncFdStdin { source: std::io::Error },
+    #[snafu(display("container has no terminal size channel"))]
+    NoTerminalSizeChannel,
+    #[snafu(display("failed to read terminal size"))]
+    GetTerminalSize { source: std::io::Error },
+    #[snafu(display("failed to update terminal size"))]
+    UpdateTerminalSize { source: mpsc::SendError },
+    #[snafu(display("container has no stdin channel"))]
+    NoStdinChannel,
+    #[snafu(display("container has no stdout channel"))]
+    NoStdoutChannel,
+    #[snafu(display("failed to forward stdin to container"))]
+    ForwardStdin { source: std::io::Error },
+    #[snafu(display("failed to forward stdout from container"))]
+    ForwardStdout { source: std::io::Error },
 }
+type Result<T, E = CmdError> = std::result::Result<T, E>;
 
 #[derive(Debug, Args)]
 pub struct DebugArgs {
@@ -44,26 +103,32 @@ pub struct DebugArgs {
 
 impl DebugArgs {
     pub async fn run(&self, _cli: &Cli) -> Result<String, CmdError> {
-        let kube = kube::Client::try_default().await.unwrap();
-        let pods = match &self.namespace {
-            Some(ns) => kube::Api::<Pod>::namespaced(kube, ns),
-            None => kube::Api::<Pod>::default_namespaced(kube),
-        };
-        let mut rng = rand::thread_rng();
-        let mut debug_container_name = "sble-debug-".to_string();
-        for _ in 0..5 {
-            debug_container_name.push(rng.gen_range('a'..='z'));
-        }
+        let kube = kube::Client::try_default()
+            .await
+            .context(KubeClientCreateSnafu)?;
+        let namespace = self
+            .namespace
+            .as_deref()
+            .unwrap_or_else(|| kube.default_namespace());
+        let pods = kube::Api::<Pod>::namespaced(kube.clone(), namespace);
+        let debug_container_name = generate_debug_container_name();
         info!(
             container.name = debug_container_name,
             "Creating debug container"
         );
-        let pod = pods.get(&self.pod).await.unwrap();
+        let pod_ref = || ObjectRef::<Pod>::new(&self.pod).within(namespace);
+        let pod = pods
+            .get(&self.pod)
+            .await
+            .with_context(|_| GetPodSnafu { pod: pod_ref() })?;
         let template_container = pod
             .spec
             .as_ref()
             .and_then(|spec| spec.containers.iter().find(|c| c.name == self.container))
-            .unwrap();
+            .with_context(|| FindTemplateContainerSnafu {
+                pod: pod_ref(),
+                container: &self.container,
+            })?;
         let pod_patch = Pod {
             spec: Some(PodSpec {
                 ephemeral_containers: Some(vec![EphemeralContainer {
@@ -93,7 +158,10 @@ impl DebugArgs {
             &kube::api::Patch::Strategic(pod_patch),
         )
         .await
-        .unwrap();
+        .with_context(|_| CreateDebugContainerSnafu {
+            pod: pod_ref(),
+            container: &self.container,
+        })?;
         info!(
             container.name = debug_container_name,
             "Waiting for container to start"
@@ -109,11 +177,17 @@ impl DebugArgs {
                         .is_some()
             })
             .await
-            .unwrap();
+            .with_context(|_| AwaitDebugContainerReadinessSnafu {
+                pod: pod_ref(),
+                container: &self.container,
+            })?;
         let debug_container_status = ready_pod
             .as_ref()
             .and_then(debug_container_status_of_pod(&debug_container_name))
-            .unwrap();
+            .with_context(|| FindDebugContainerStatusSnafu {
+                pod: pod_ref(),
+                container: &self.container,
+            })?;
         if let Some(termination) = debug_container_status
             .last_state
             .as_ref()
@@ -135,29 +209,53 @@ impl DebugArgs {
                 &AttachParams::interactive_tty().container(debug_container_name),
             )
             .await
-            .context(AttachSnafu)?;
+            .with_context(|_| AttachContainerSnafu {
+                pod: pod_ref(),
+                container: &self.container,
+            })?;
         info!("Attached to container, if the shell line looks empty, press ENTER!");
         {
-            let _raw = std::io::stdout().into_raw_mode().unwrap();
+            let _raw = std::io::stdout()
+                .into_raw_mode()
+                .context(SetRawTtyModeSnafu)?;
             futures::future::select_all([
-                update_terminal_size(attachment.terminal_size().unwrap())
-                    .map(Ok)
-                    .boxed(),
-                tokio::io::copy(&mut attachment.stdout().unwrap(), &mut tokio::io::stdout())
-                    .map_ok(drop)
-                    .boxed(),
-                tokio::io::copy(&mut AsyncStdin::new(), &mut attachment.stdin().unwrap())
-                    .map_ok(drop)
-                    .boxed(),
+                update_terminal_size(
+                    attachment
+                        .terminal_size()
+                        .context(NoTerminalSizeChannelSnafu)?,
+                )
+                .boxed(),
+                tokio::io::copy(
+                    &mut attachment.stdout().context(NoStdoutChannelSnafu)?,
+                    &mut tokio::io::stdout(),
+                )
+                .map_ok(drop)
+                .context(ForwardStdoutSnafu)
+                .boxed(),
+                tokio::io::copy(
+                    &mut AsyncStdin::new()?,
+                    &mut attachment.stdin().context(NoStdinChannelSnafu)?,
+                )
+                .map_ok(drop)
+                .context(ForwardStdinSnafu)
+                .boxed(),
             ])
             .await
-            .0
-            .unwrap();
+            .0?;
         }
         Ok(String::new())
     }
 }
 
+fn generate_debug_container_name() -> String {
+    let mut rng = rand::thread_rng();
+    let mut name = "sble-debug-".to_string();
+    for _ in 0..5 {
+        name.push(rng.gen_range('a'..='z'));
+    }
+    name
+}
+
 /// Does true non-blocking reads of stdin, so that we can cancel properly on shutdown.
 /// The compromise is that it does not handle having things piped into it very well, since their write sides
 /// will also be turned non-blocking.
@@ -169,13 +267,13 @@ struct AsyncStdin {
 }
 
 impl AsyncStdin {
-    fn new() -> Self {
+    fn new() -> Result<Self> {
         let stdin = std::io::stdin();
         // Make stdin non-blocking
         {
             let old_flags = unsafe { libc::fcntl(stdin.as_raw_fd(), libc::F_GETFL) };
             if old_flags == -1 {
-                panic!("{:?}", std::io::Error::last_os_error());
+                return Err(std::io::Error::last_os_error()).context(AsyncifyStdinSnafu);
             }
             let status = unsafe {
                 libc::fcntl(
@@ -185,12 +283,12 @@ impl AsyncStdin {
                 )
             };
             if status == -1 {
-                panic!("{:?}", std::io::Error::last_os_error());
+                return Err(std::io::Error::last_os_error()).context(AsyncifyStdinSnafu);
             }
         };
-        Self {
-            fd: AsyncFd::new(stdin).unwrap(),
-        }
+        Ok(Self {
+            fd: AsyncFd::new(stdin).context(AsyncFdStdinSnafu)?,
+        })
     }
 }
 
@@ -217,10 +315,10 @@ impl AsyncRead for AsyncStdin {
     }
 }
 
-async fn update_terminal_size(mut tx: Sender<TerminalSize>) {
+async fn update_terminal_size(mut tx: Sender<TerminalSize>) -> Result<()> {
     let mut signal = tokio::signal::unix::signal(SignalKind::window_change()).unwrap();
     {
-        let (width, height) = terminal_size().unwrap();
+        let (width, height) = terminal_size().context(GetTerminalSizeSnafu)?;
         // Make TTY apps re-render by force-changing the terminal size
         // Start by sending an invalid size so that it's a change no matter
         // whether the size has actually changed.
@@ -229,13 +327,18 @@ async fn update_terminal_size(mut tx: Sender<TerminalSize>) {
             height,
         })
         .await
-        .unwrap();
-        tx.send(TerminalSize { width, height }).await.unwrap();
+        .context(UpdateTerminalSizeSnafu)?;
+        tx.send(TerminalSize { width, height })
+            .await
+            .context(UpdateTerminalSizeSnafu)?;
     }
     while let Some(()) = signal.recv().await {
-        let (width, height) = terminal_size().unwrap();
-        tx.send(TerminalSize { width, height }).await.unwrap();
+        let (width, height) = terminal_size().context(GetTerminalSizeSnafu)?;
+        tx.send(TerminalSize { width, height })
+            .await
+            .context(UpdateTerminalSizeSnafu)?;
     }
+    Ok(())
 }
 
 fn debug_container_status_of_pod(

From 2e262704416b52a28c898a8c98e8c7f730a045f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= <nat@nullable.se>
Date: Wed, 28 Feb 2024 14:39:42 +0100
Subject: [PATCH 08/18] Let AsyncStdin reset its state once shut down

---
 rust/stackablectl/src/cmds/debug.rs | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/rust/stackablectl/src/cmds/debug.rs b/rust/stackablectl/src/cmds/debug.rs
index e103aec2..b35eb957 100644
--- a/rust/stackablectl/src/cmds/debug.rs
+++ b/rust/stackablectl/src/cmds/debug.rs
@@ -260,21 +260,28 @@ fn generate_debug_container_name() -> String {
 /// The compromise is that it does not handle having things piped into it very well, since their write sides
 /// will also be turned non-blocking.
 ///
-/// Also, `AsyncStdin` will currently _not_ restore the mode of stdin when dropped.
-// FIXME: restore status
+/// Only a single instance of AsyncStdin should exist at any one time.
+///
+/// Also, `AsyncStdin` will restore the mode of stdin when dropped.
 struct AsyncStdin {
     fd: AsyncFd<Stdin>,
+    old_flags: i32,
 }
 
 impl AsyncStdin {
+    #[tracing::instrument]
     fn new() -> Result<Self> {
         let stdin = std::io::stdin();
         // Make stdin non-blocking
+        let old_flags;
         {
-            let old_flags = unsafe { libc::fcntl(stdin.as_raw_fd(), libc::F_GETFL) };
+            old_flags = unsafe { libc::fcntl(stdin.as_raw_fd(), libc::F_GETFL) };
             if old_flags == -1 {
                 return Err(std::io::Error::last_os_error()).context(AsyncifyStdinSnafu);
             }
+            if old_flags & libc::O_NONBLOCK != 0 {
+                tracing::warn!("stdin is already non-blocking (did you try to create multiple AsyncStdin instances?)");
+            }
             let status = unsafe {
                 libc::fcntl(
                     stdin.as_raw_fd(),
@@ -288,10 +295,23 @@ impl AsyncStdin {
         };
         Ok(Self {
             fd: AsyncFd::new(stdin).context(AsyncFdStdinSnafu)?,
+            old_flags,
         })
     }
 }
 
+impl Drop for AsyncStdin {
+    fn drop(&mut self) {
+        let status = unsafe { libc::fcntl(self.fd.as_raw_fd(), libc::F_SETFL, self.old_flags) };
+        if status == -1 {
+            panic!(
+                "unable to revert stdin flags: {}",
+                std::io::Error::last_os_error()
+            );
+        }
+    }
+}
+
 impl AsyncRead for AsyncStdin {
     fn poll_read(
         mut self: std::pin::Pin<&mut Self>,

From 97545385deb3c44532465545a520223fd8948381 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= <nat@nullable.se>
Date: Wed, 28 Feb 2024 14:53:11 +0100
Subject: [PATCH 09/18] Docs, demarcate as experimental

---
 .../stackablectl/partials/commands/index.adoc |  18 +-
 extra/completions/_stackablectl               |  42 ++--
 extra/completions/stackablectl.bash           | 190 +++++++++---------
 extra/completions/stackablectl.fish           |  48 ++---
 extra/man/stackablectl.1                      |   3 +-
 rust/stackablectl/README.md                   |  18 +-
 rust/stackablectl/src/cli/mod.rs              |   8 +-
 rust/stackablectl/src/cmds/debug.rs           |   7 +
 8 files changed, 173 insertions(+), 161 deletions(-)

diff --git a/docs/modules/stackablectl/partials/commands/index.adoc b/docs/modules/stackablectl/partials/commands/index.adoc
index 3a68baf8..9c30403d 100644
--- a/docs/modules/stackablectl/partials/commands/index.adoc
+++ b/docs/modules/stackablectl/partials/commands/index.adoc
@@ -6,15 +6,15 @@ Command line tool to interact with the Stackable Data Platform
 Usage: stackablectl [OPTIONS] <COMMAND>
 
 Commands:
-  operator     Interact with single operator instead of the full platform
-  release      Interact with all operators of the platform which are released together
-  stack        Interact with stacks, which are ready-to-use product combinations
-  stacklet     Interact with deployed stacklets, which are bundles of resources and containers required to run the product
-  demo         Interact with demos, which are end-to-end usage demonstrations of the Stackable data platform
-  completions  Generate shell completions for this tool
-  cache        Interact with locally cached files
-  debug
-  help         Print this message or the help of the given subcommand(s)
+  operator            Interact with single operator instead of the full platform
+  release             Interact with all operators of the platform which are released together
+  stack               Interact with stacks, which are ready-to-use product combinations
+  stacklet            Interact with deployed stacklets, which are bundles of resources and containers required to run the product
+  demo                Interact with demos, which are end-to-end usage demonstrations of the Stackable data platform
+  completions         Generate shell completions for this tool
+  cache               Interact with locally cached files
+  experimental-debug  EXPERIMENTAL: Launch a debug container for a Pod
+  help                Print this message or the help of the given subcommand(s)
 
 Options:
   -l, --log-level <LOG_LEVEL>
diff --git a/extra/completions/_stackablectl b/extra/completions/_stackablectl
index 304a9ff8..04feeaa7 100644
--- a/extra/completions/_stackablectl
+++ b/extra/completions/_stackablectl
@@ -1122,13 +1122,13 @@ esac
     ;;
 esac
 ;;
-(debug)
+(experimental-debug)
 _arguments "${_arguments_options[@]}" \
-'-n+[]:NAMESPACE: ' \
-'--namespace=[]:NAMESPACE: ' \
-'-c+[]:CONTAINER: ' \
-'--container=[]:CONTAINER: ' \
-'--image=[]:IMAGE: ' \
+'-n+[The namespace of the Pod being debugged]:NAMESPACE: ' \
+'--namespace=[The namespace of the Pod being debugged]:NAMESPACE: ' \
+'-c+[The target container to debug]:CONTAINER: ' \
+'--container=[The target container to debug]:CONTAINER: ' \
+'--image=[The debug container image]:IMAGE: ' \
 '-l+[Log level this application uses]:LOG_LEVEL: ' \
 '--log-level=[Log level this application uses]:LOG_LEVEL: ' \
 '*-d+[Provide one or more additional (custom) demo file(s)]:DEMO_FILE:_files' \
@@ -1146,8 +1146,8 @@ _arguments "${_arguments_options[@]}" \
 '--help[Print help (see more with '\''--help'\'')]' \
 '-V[Print version]' \
 '--version[Print version]' \
-':pod:' \
-'*::cmd:' \
+':pod -- The Pod to debug:' \
+'*::cmd -- The command to run in the debug container:' \
 && ret=0
 ;;
 (help)
@@ -1362,7 +1362,7 @@ _arguments "${_arguments_options[@]}" \
     ;;
 esac
 ;;
-(debug)
+(experimental-debug)
 _arguments "${_arguments_options[@]}" \
 && ret=0
 ;;
@@ -1389,7 +1389,7 @@ _stackablectl_commands() {
 'demo:Interact with demos, which are end-to-end usage demonstrations of the Stackable data platform' \
 'completions:Generate shell completions for this tool' \
 'cache:Interact with locally cached files' \
-'debug:' \
+'experimental-debug:EXPERIMENTAL\: Launch a debug container for a Pod' \
 'help:Print this message or the help of the given subcommand(s)' \
     )
     _describe -t commands 'stackablectl commands' commands "$@"
@@ -1475,16 +1475,6 @@ _stackablectl__stacklet__help__credentials_commands() {
     local commands; commands=()
     _describe -t commands 'stackablectl stacklet help credentials commands' commands "$@"
 }
-(( $+functions[_stackablectl__debug_commands] )) ||
-_stackablectl__debug_commands() {
-    local commands; commands=()
-    _describe -t commands 'stackablectl debug commands' commands "$@"
-}
-(( $+functions[_stackablectl__help__debug_commands] )) ||
-_stackablectl__help__debug_commands() {
-    local commands; commands=()
-    _describe -t commands 'stackablectl help debug commands' commands "$@"
-}
 (( $+functions[_stackablectl__demo_commands] )) ||
 _stackablectl__demo_commands() {
     local commands; commands=(
@@ -1564,6 +1554,16 @@ _stackablectl__stack__help__describe_commands() {
     local commands; commands=()
     _describe -t commands 'stackablectl stack help describe commands' commands "$@"
 }
+(( $+functions[_stackablectl__experimental-debug_commands] )) ||
+_stackablectl__experimental-debug_commands() {
+    local commands; commands=()
+    _describe -t commands 'stackablectl experimental-debug commands' commands "$@"
+}
+(( $+functions[_stackablectl__help__experimental-debug_commands] )) ||
+_stackablectl__help__experimental-debug_commands() {
+    local commands; commands=()
+    _describe -t commands 'stackablectl help experimental-debug commands' commands "$@"
+}
 (( $+functions[_stackablectl__completions__fish_commands] )) ||
 _stackablectl__completions__fish_commands() {
     local commands; commands=()
@@ -1633,7 +1633,7 @@ _stackablectl__help_commands() {
 'demo:Interact with demos, which are end-to-end usage demonstrations of the Stackable data platform' \
 'completions:Generate shell completions for this tool' \
 'cache:Interact with locally cached files' \
-'debug:' \
+'experimental-debug:EXPERIMENTAL\: Launch a debug container for a Pod' \
 'help:Print this message or the help of the given subcommand(s)' \
     )
     _describe -t commands 'stackablectl help commands' commands "$@"
diff --git a/extra/completions/stackablectl.bash b/extra/completions/stackablectl.bash
index e213b976..376f0118 100644
--- a/extra/completions/stackablectl.bash
+++ b/extra/completions/stackablectl.bash
@@ -18,12 +18,12 @@ _stackablectl() {
             stackablectl,completions)
                 cmd="stackablectl__completions"
                 ;;
-            stackablectl,debug)
-                cmd="stackablectl__debug"
-                ;;
             stackablectl,demo)
                 cmd="stackablectl__demo"
                 ;;
+            stackablectl,experimental-debug)
+                cmd="stackablectl__experimental__debug"
+                ;;
             stackablectl,help)
                 cmd="stackablectl__help"
                 ;;
@@ -111,12 +111,12 @@ _stackablectl() {
             stackablectl__help,completions)
                 cmd="stackablectl__help__completions"
                 ;;
-            stackablectl__help,debug)
-                cmd="stackablectl__help__debug"
-                ;;
             stackablectl__help,demo)
                 cmd="stackablectl__help__demo"
                 ;;
+            stackablectl__help,experimental-debug)
+                cmd="stackablectl__help__experimental__debug"
+                ;;
             stackablectl__help,help)
                 cmd="stackablectl__help__help"
                 ;;
@@ -313,7 +313,7 @@ _stackablectl() {
 
     case "${cmd}" in
         stackablectl)
-            opts="-l -d -s -r -h -V --log-level --no-cache --offline --demo-file --stack-file --release-file --helm-repo-stable --helm-repo-test --helm-repo-dev --help --version operator release stack stacklet demo completions cache debug help"
+            opts="-l -d -s -r -h -V --log-level --no-cache --offline --demo-file --stack-file --release-file --helm-repo-stable --helm-repo-test --helm-repo-dev --help --version operator release stack stacklet demo completions cache experimental-debug help"
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -902,84 +902,6 @@ _stackablectl() {
             COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
             return 0
             ;;
-        stackablectl__debug)
-            opts="-n -c -l -d -s -r -h -V --namespace --container --image --log-level --no-cache --offline --demo-file --stack-file --release-file --helm-repo-stable --helm-repo-test --helm-repo-dev --help --version <POD> [CMD]..."
-            if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
-                COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
-                return 0
-            fi
-            case "${prev}" in
-                --namespace)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                -n)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                --container)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                -c)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                --image)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                --log-level)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                -l)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                --demo-file)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                -d)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                --stack-file)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                -s)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                --release-file)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                -r)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                --helm-repo-stable)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                --helm-repo-test)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                --helm-repo-dev)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                *)
-                    COMPREPLY=()
-                    ;;
-            esac
-            COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
-            return 0
-            ;;
         stackablectl__demo)
             opts="-l -d -s -r -h -V --log-level --no-cache --offline --demo-file --stack-file --release-file --helm-repo-stable --helm-repo-test --helm-repo-dev --help --version list describe install help"
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
@@ -1346,8 +1268,86 @@ _stackablectl() {
             COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
             return 0
             ;;
+        stackablectl__experimental__debug)
+            opts="-n -c -l -d -s -r -h -V --namespace --container --image --log-level --no-cache --offline --demo-file --stack-file --release-file --helm-repo-stable --helm-repo-test --helm-repo-dev --help --version <POD> [CMD]..."
+            if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
+                COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+                return 0
+            fi
+            case "${prev}" in
+                --namespace)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -n)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --container)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -c)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --image)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --log-level)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -l)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --demo-file)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -d)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --stack-file)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -s)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --release-file)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -r)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --helm-repo-stable)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --helm-repo-test)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --helm-repo-dev)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                *)
+                    COMPREPLY=()
+                    ;;
+            esac
+            COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+            return 0
+            ;;
         stackablectl__help)
-            opts="operator release stack stacklet demo completions cache debug help"
+            opts="operator release stack stacklet demo completions cache experimental-debug help"
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -1458,8 +1458,8 @@ _stackablectl() {
             COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
             return 0
             ;;
-        stackablectl__help__debug)
-            opts=""
+        stackablectl__help__demo)
+            opts="list describe install"
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -1472,9 +1472,9 @@ _stackablectl() {
             COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
             return 0
             ;;
-        stackablectl__help__demo)
-            opts="list describe install"
-            if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then
+        stackablectl__help__demo__describe)
+            opts=""
+            if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
             fi
@@ -1486,7 +1486,7 @@ _stackablectl() {
             COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
             return 0
             ;;
-        stackablectl__help__demo__describe)
+        stackablectl__help__demo__install)
             opts=""
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
@@ -1500,7 +1500,7 @@ _stackablectl() {
             COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
             return 0
             ;;
-        stackablectl__help__demo__install)
+        stackablectl__help__demo__list)
             opts=""
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
@@ -1514,9 +1514,9 @@ _stackablectl() {
             COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
             return 0
             ;;
-        stackablectl__help__demo__list)
+        stackablectl__help__experimental__debug)
             opts=""
-            if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then
+            if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
             fi
diff --git a/extra/completions/stackablectl.fish b/extra/completions/stackablectl.fish
index 0197b451..d79a3803 100644
--- a/extra/completions/stackablectl.fish
+++ b/extra/completions/stackablectl.fish
@@ -16,7 +16,7 @@ complete -c stackablectl -n "__fish_use_subcommand" -f -a "stacklet" -d 'Interac
 complete -c stackablectl -n "__fish_use_subcommand" -f -a "demo" -d 'Interact with demos, which are end-to-end usage demonstrations of the Stackable data platform'
 complete -c stackablectl -n "__fish_use_subcommand" -f -a "completions" -d 'Generate shell completions for this tool'
 complete -c stackablectl -n "__fish_use_subcommand" -f -a "cache" -d 'Interact with locally cached files'
-complete -c stackablectl -n "__fish_use_subcommand" -f -a "debug"
+complete -c stackablectl -n "__fish_use_subcommand" -f -a "experimental-debug" -d 'EXPERIMENTAL: Launch a debug container for a Pod'
 complete -c stackablectl -n "__fish_use_subcommand" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
 complete -c stackablectl -n "__fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from list; and not __fish_seen_subcommand_from describe; and not __fish_seen_subcommand_from install; and not __fish_seen_subcommand_from uninstall; and not __fish_seen_subcommand_from installed; and not __fish_seen_subcommand_from help" -s l -l log-level -d 'Log level this application uses' -r
 complete -c stackablectl -n "__fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from list; and not __fish_seen_subcommand_from describe; and not __fish_seen_subcommand_from install; and not __fish_seen_subcommand_from uninstall; and not __fish_seen_subcommand_from installed; and not __fish_seen_subcommand_from help" -s d -l demo-file -d 'Provide one or more additional (custom) demo file(s)' -r -F
@@ -442,29 +442,29 @@ complete -c stackablectl -n "__fish_seen_subcommand_from cache; and __fish_seen_
 complete -c stackablectl -n "__fish_seen_subcommand_from cache; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from list; and not __fish_seen_subcommand_from clean; and not __fish_seen_subcommand_from help" -f -a "list" -d 'List cached files'
 complete -c stackablectl -n "__fish_seen_subcommand_from cache; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from list; and not __fish_seen_subcommand_from clean; and not __fish_seen_subcommand_from help" -f -a "clean" -d 'Clean cached files'
 complete -c stackablectl -n "__fish_seen_subcommand_from cache; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from list; and not __fish_seen_subcommand_from clean; and not __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
-complete -c stackablectl -n "__fish_seen_subcommand_from debug" -s n -l namespace -r
-complete -c stackablectl -n "__fish_seen_subcommand_from debug" -s c -l container -r
-complete -c stackablectl -n "__fish_seen_subcommand_from debug" -l image -r
-complete -c stackablectl -n "__fish_seen_subcommand_from debug" -s l -l log-level -d 'Log level this application uses' -r
-complete -c stackablectl -n "__fish_seen_subcommand_from debug" -s d -l demo-file -d 'Provide one or more additional (custom) demo file(s)' -r -F
-complete -c stackablectl -n "__fish_seen_subcommand_from debug" -s s -l stack-file -d 'Provide one or more additional (custom) stack file(s)' -r -F
-complete -c stackablectl -n "__fish_seen_subcommand_from debug" -s r -l release-file -d 'Provide one or more additional (custom) release file(s)' -r -F
-complete -c stackablectl -n "__fish_seen_subcommand_from debug" -l helm-repo-stable -d 'Provide a custom Helm stable repository URL' -r -f
-complete -c stackablectl -n "__fish_seen_subcommand_from debug" -l helm-repo-test -d 'Provide a custom Helm test repository URL' -r -f
-complete -c stackablectl -n "__fish_seen_subcommand_from debug" -l helm-repo-dev -d 'Provide a custom Helm dev repository URL' -r -f
-complete -c stackablectl -n "__fish_seen_subcommand_from debug" -l no-cache -d 'Do not cache the remote (default) demo, stack and release files'
-complete -c stackablectl -n "__fish_seen_subcommand_from debug" -l offline -d 'Do not request any remote files via the network'
-complete -c stackablectl -n "__fish_seen_subcommand_from debug" -s h -l help -d 'Print help (see more with \'--help\')'
-complete -c stackablectl -n "__fish_seen_subcommand_from debug" -s V -l version -d 'Print version'
-complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from help" -f -a "operator" -d 'Interact with single operator instead of the full platform'
-complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from help" -f -a "release" -d 'Interact with all operators of the platform which are released together'
-complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from help" -f -a "stack" -d 'Interact with stacks, which are ready-to-use product combinations'
-complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from help" -f -a "stacklet" -d 'Interact with deployed stacklets, which are bundles of resources and containers required to run the product'
-complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from help" -f -a "demo" -d 'Interact with demos, which are end-to-end usage demonstrations of the Stackable data platform'
-complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from help" -f -a "completions" -d 'Generate shell completions for this tool'
-complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from help" -f -a "cache" -d 'Interact with locally cached files'
-complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from help" -f -a "debug"
-complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
+complete -c stackablectl -n "__fish_seen_subcommand_from experimental-debug" -s n -l namespace -d 'The namespace of the Pod being debugged' -r
+complete -c stackablectl -n "__fish_seen_subcommand_from experimental-debug" -s c -l container -d 'The target container to debug' -r
+complete -c stackablectl -n "__fish_seen_subcommand_from experimental-debug" -l image -d 'The debug container image' -r
+complete -c stackablectl -n "__fish_seen_subcommand_from experimental-debug" -s l -l log-level -d 'Log level this application uses' -r
+complete -c stackablectl -n "__fish_seen_subcommand_from experimental-debug" -s d -l demo-file -d 'Provide one or more additional (custom) demo file(s)' -r -F
+complete -c stackablectl -n "__fish_seen_subcommand_from experimental-debug" -s s -l stack-file -d 'Provide one or more additional (custom) stack file(s)' -r -F
+complete -c stackablectl -n "__fish_seen_subcommand_from experimental-debug" -s r -l release-file -d 'Provide one or more additional (custom) release file(s)' -r -F
+complete -c stackablectl -n "__fish_seen_subcommand_from experimental-debug" -l helm-repo-stable -d 'Provide a custom Helm stable repository URL' -r -f
+complete -c stackablectl -n "__fish_seen_subcommand_from experimental-debug" -l helm-repo-test -d 'Provide a custom Helm test repository URL' -r -f
+complete -c stackablectl -n "__fish_seen_subcommand_from experimental-debug" -l helm-repo-dev -d 'Provide a custom Helm dev repository URL' -r -f
+complete -c stackablectl -n "__fish_seen_subcommand_from experimental-debug" -l no-cache -d 'Do not cache the remote (default) demo, stack and release files'
+complete -c stackablectl -n "__fish_seen_subcommand_from experimental-debug" -l offline -d 'Do not request any remote files via the network'
+complete -c stackablectl -n "__fish_seen_subcommand_from experimental-debug" -s h -l help -d 'Print help (see more with \'--help\')'
+complete -c stackablectl -n "__fish_seen_subcommand_from experimental-debug" -s V -l version -d 'Print version'
+complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from experimental-debug; and not __fish_seen_subcommand_from help" -f -a "operator" -d 'Interact with single operator instead of the full platform'
+complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from experimental-debug; and not __fish_seen_subcommand_from help" -f -a "release" -d 'Interact with all operators of the platform which are released together'
+complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from experimental-debug; and not __fish_seen_subcommand_from help" -f -a "stack" -d 'Interact with stacks, which are ready-to-use product combinations'
+complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from experimental-debug; and not __fish_seen_subcommand_from help" -f -a "stacklet" -d 'Interact with deployed stacklets, which are bundles of resources and containers required to run the product'
+complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from experimental-debug; and not __fish_seen_subcommand_from help" -f -a "demo" -d 'Interact with demos, which are end-to-end usage demonstrations of the Stackable data platform'
+complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from experimental-debug; and not __fish_seen_subcommand_from help" -f -a "completions" -d 'Generate shell completions for this tool'
+complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from experimental-debug; and not __fish_seen_subcommand_from help" -f -a "cache" -d 'Interact with locally cached files'
+complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from experimental-debug; and not __fish_seen_subcommand_from help" -f -a "experimental-debug" -d 'EXPERIMENTAL: Launch a debug container for a Pod'
+complete -c stackablectl -n "__fish_seen_subcommand_from help; and not __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from release; and not __fish_seen_subcommand_from stack; and not __fish_seen_subcommand_from stacklet; and not __fish_seen_subcommand_from demo; and not __fish_seen_subcommand_from completions; and not __fish_seen_subcommand_from cache; and not __fish_seen_subcommand_from experimental-debug; and not __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
 complete -c stackablectl -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from list; and not __fish_seen_subcommand_from describe; and not __fish_seen_subcommand_from install; and not __fish_seen_subcommand_from uninstall; and not __fish_seen_subcommand_from installed" -f -a "list" -d 'List available operators'
 complete -c stackablectl -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from list; and not __fish_seen_subcommand_from describe; and not __fish_seen_subcommand_from install; and not __fish_seen_subcommand_from uninstall; and not __fish_seen_subcommand_from installed" -f -a "describe" -d 'Print out detailed operator information'
 complete -c stackablectl -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from operator; and not __fish_seen_subcommand_from list; and not __fish_seen_subcommand_from describe; and not __fish_seen_subcommand_from install; and not __fish_seen_subcommand_from uninstall; and not __fish_seen_subcommand_from installed" -f -a "install" -d 'Install one or more operators'
diff --git a/extra/man/stackablectl.1 b/extra/man/stackablectl.1
index 48daac8f..79cb633a 100644
--- a/extra/man/stackablectl.1
+++ b/extra/man/stackablectl.1
@@ -92,7 +92,8 @@ Generate shell completions for this tool
 stackablectl\-cache(1)
 Interact with locally cached files
 .TP
-stackablectl\-debug(1)
+stackablectl\-experimental\-debug(1)
+EXPERIMENTAL: Launch a debug container for a Pod
 .TP
 stackablectl\-help(1)
 Print this message or the help of the given subcommand(s)
diff --git a/rust/stackablectl/README.md b/rust/stackablectl/README.md
index a74ca108..57ebe7b2 100644
--- a/rust/stackablectl/README.md
+++ b/rust/stackablectl/README.md
@@ -15,15 +15,15 @@ Command line tool to interact with the Stackable Data Platform
 Usage: stackablectl [OPTIONS] <COMMAND>
 
 Commands:
-  operator     Interact with single operator instead of the full platform
-  release      Interact with all operators of the platform which are released together
-  stack        Interact with stacks, which are ready-to-use product combinations
-  stacklet     Interact with deployed stacklets, which are bundles of resources and containers required to run the product
-  demo         Interact with demos, which are end-to-end usage demonstrations of the Stackable data platform
-  completions  Generate shell completions for this tool
-  cache        Interact with locally cached files
-  debug
-  help         Print this message or the help of the given subcommand(s)
+  operator            Interact with single operator instead of the full platform
+  release             Interact with all operators of the platform which are released together
+  stack               Interact with stacks, which are ready-to-use product combinations
+  stacklet            Interact with deployed stacklets, which are bundles of resources and containers required to run the product
+  demo                Interact with demos, which are end-to-end usage demonstrations of the Stackable data platform
+  completions         Generate shell completions for this tool
+  cache               Interact with locally cached files
+  experimental-debug  EXPERIMENTAL: Launch a debug container for a Pod
+  help                Print this message or the help of the given subcommand(s)
 
 Options:
   -l, --log-level <LOG_LEVEL>
diff --git a/rust/stackablectl/src/cli/mod.rs b/rust/stackablectl/src/cli/mod.rs
index ef5b2cf0..268e1a1c 100644
--- a/rust/stackablectl/src/cli/mod.rs
+++ b/rust/stackablectl/src/cli/mod.rs
@@ -194,7 +194,7 @@ impl Cli {
             Commands::Demo(args) => args.run(self, cache).await.context(DemoSnafu),
             Commands::Completions(args) => args.run().context(CompletionsSnafu),
             Commands::Cache(args) => args.run(self, cache).await.context(CacheSnafu),
-            Commands::Debug(args) => args.run(self).await.context(DebugSnafu),
+            Commands::ExperimentalDebug(args) => args.run(self).await.context(DebugSnafu),
         }
     }
 
@@ -245,7 +245,11 @@ CRDs."
     /// Interact with locally cached files
     Cache(cache::CacheArgs),
 
-    Debug(debug::DebugArgs),
+    /// EXPERIMENTAL: Launch a debug container for a Pod
+    #[command(long_about = "EXPERIMENTAL: Launch a debug container for a Pod.
+
+This container will have access to the same data volumes as the primary container.")]
+    ExperimentalDebug(debug::DebugArgs),
 }
 
 #[derive(Clone, Debug, Default, ValueEnum)]
diff --git a/rust/stackablectl/src/cmds/debug.rs b/rust/stackablectl/src/cmds/debug.rs
index b35eb957..7662fc42 100644
--- a/rust/stackablectl/src/cmds/debug.rs
+++ b/rust/stackablectl/src/cmds/debug.rs
@@ -90,13 +90,20 @@ type Result<T, E = CmdError> = std::result::Result<T, E>;
 
 #[derive(Debug, Args)]
 pub struct DebugArgs {
+    /// The namespace of the Pod being debugged
     #[clap(long, short)]
     namespace: Option<String>,
+    /// The Pod to debug
     pod: String,
+    /// The target container to debug
+    ///
+    /// Volumes and environment variables will be copied from this container.
     #[clap(long, short)]
     container: String,
+    /// The debug container image
     #[clap(long)]
     image: String,
+    /// The command to run in the debug container
     #[clap(last = true)]
     cmd: Option<Vec<String>>,
 }

From 7bbc56755ede9a6a6ce1c30a97242ffa03c5e170 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= <nat@nullable.se>
Date: Wed, 28 Feb 2024 15:10:38 +0100
Subject: [PATCH 10/18] Move debug container name to span

---
 rust/stackablectl/src/cmds/debug.rs | 180 ++++++++++++++--------------
 1 file changed, 89 insertions(+), 91 deletions(-)

diff --git a/rust/stackablectl/src/cmds/debug.rs b/rust/stackablectl/src/cmds/debug.rs
index 7662fc42..51bd5a89 100644
--- a/rust/stackablectl/src/cmds/debug.rs
+++ b/rust/stackablectl/src/cmds/debug.rs
@@ -24,7 +24,7 @@ use tokio::{
     io::{unix::AsyncFd, AsyncRead},
     signal::unix::SignalKind,
 };
-use tracing::{error, info};
+use tracing::{error, info, info_span, warn, Instrument};
 
 use crate::cli::Cli;
 
@@ -119,109 +119,105 @@ impl DebugArgs {
             .unwrap_or_else(|| kube.default_namespace());
         let pods = kube::Api::<Pod>::namespaced(kube.clone(), namespace);
         let debug_container_name = generate_debug_container_name();
-        info!(
-            container.name = debug_container_name,
-            "Creating debug container"
-        );
-        let pod_ref = || ObjectRef::<Pod>::new(&self.pod).within(namespace);
-        let pod = pods
-            .get(&self.pod)
-            .await
-            .with_context(|_| GetPodSnafu { pod: pod_ref() })?;
-        let template_container = pod
-            .spec
-            .as_ref()
-            .and_then(|spec| spec.containers.iter().find(|c| c.name == self.container))
-            .with_context(|| FindTemplateContainerSnafu {
-                pod: pod_ref(),
-                container: &self.container,
-            })?;
-        let pod_patch = Pod {
-            spec: Some(PodSpec {
-                ephemeral_containers: Some(vec![EphemeralContainer {
-                    name: debug_container_name.clone(),
-                    image: Some(self.image.clone()),
-                    tty: Some(true),
-                    stdin: Some(true),
+        let span = info_span!("debug", container.debug.name = debug_container_name);
+        async {
+            info!("Creating debug container");
+            let pod_ref = || ObjectRef::<Pod>::new(&self.pod).within(namespace);
+            let pod = pods
+                .get(&self.pod)
+                .await
+                .with_context(|_| GetPodSnafu { pod: pod_ref() })?;
+            let template_container = pod
+                .spec
+                .as_ref()
+                .and_then(|spec| spec.containers.iter().find(|c| c.name == self.container))
+                .with_context(|| FindTemplateContainerSnafu {
+                    pod: pod_ref(),
+                    container: &self.container,
+                })?;
+            let pod_patch = Pod {
+                spec: Some(PodSpec {
+                    ephemeral_containers: Some(vec![EphemeralContainer {
+                        name: debug_container_name.clone(),
+                        image: Some(self.image.clone()),
+                        tty: Some(true),
+                        stdin: Some(true),
 
-                    command: self.cmd.clone(),
-                    args: self.cmd.is_some().then(Vec::new),
+                        command: self.cmd.clone(),
+                        args: self.cmd.is_some().then(Vec::new),
 
-                    // copy environment from template
-                    env: template_container.env.clone(),
-                    env_from: template_container.env_from.clone(),
-                    volume_mounts: template_container.volume_mounts.clone(),
-                    volume_devices: template_container.volume_devices.clone(),
+                        // copy environment from template
+                        env: template_container.env.clone(),
+                        env_from: template_container.env_from.clone(),
+                        volume_mounts: template_container.volume_mounts.clone(),
+                        volume_devices: template_container.volume_devices.clone(),
 
+                        ..Default::default()
+                    }]),
                     ..Default::default()
-                }]),
+                }),
                 ..Default::default()
-            }),
-            ..Default::default()
-        };
-        pods.patch_ephemeral_containers(
-            &self.pod,
-            &PatchParams::default(),
-            &kube::api::Patch::Strategic(pod_patch),
-        )
-        .await
-        .with_context(|_| CreateDebugContainerSnafu {
-            pod: pod_ref(),
-            container: &self.container,
-        })?;
-        info!(
-            container.name = debug_container_name,
-            "Waiting for container to start"
-        );
-        let ready_pod =
-            kube::runtime::wait::await_condition(pods.clone(), &self.pod, |pod: Option<&Pod>| {
-                let container = pod.and_then(debug_container_status_of_pod(&debug_container_name));
-                container
-                    .and_then(|c| Some(c.state.as_ref()?.waiting.is_none()))
-                    .unwrap_or_default()
-                    || container
-                        .and_then(|c| c.last_state.as_ref()?.terminated.as_ref())
-                        .is_some()
-            })
+            };
+            pods.patch_ephemeral_containers(
+                &self.pod,
+                &PatchParams::default(),
+                &kube::api::Patch::Strategic(pod_patch),
+            )
             .await
-            .with_context(|_| AwaitDebugContainerReadinessSnafu {
+            .with_context(|_| CreateDebugContainerSnafu {
                 pod: pod_ref(),
                 container: &self.container,
             })?;
-        let debug_container_status = ready_pod
-            .as_ref()
-            .and_then(debug_container_status_of_pod(&debug_container_name))
-            .with_context(|| FindDebugContainerStatusSnafu {
-                pod: pod_ref(),
-                container: &self.container,
-            })?;
-        if let Some(termination) = debug_container_status
-            .last_state
-            .as_ref()
-            .and_then(|state| state.terminated.as_ref())
-        {
-            error!(
-                error = termination.message,
-                exit_code = termination.exit_code,
-                "Debug container failed to start!"
-            );
-        }
-        info!(
-            container.name = debug_container_name,
-            "Attaching to container"
-        );
-        let mut attachment = pods
-            .attach(
+            info!("Waiting for container to start");
+            let ready_pod = kube::runtime::wait::await_condition(
+                pods.clone(),
                 &self.pod,
-                &AttachParams::interactive_tty().container(debug_container_name),
+                |pod: Option<&Pod>| {
+                    let container =
+                        pod.and_then(debug_container_status_of_pod(&debug_container_name));
+                    container
+                        .and_then(|c| Some(c.state.as_ref()?.waiting.is_none()))
+                        .unwrap_or_default()
+                        || container
+                            .and_then(|c| c.last_state.as_ref()?.terminated.as_ref())
+                            .is_some()
+                },
             )
             .await
-            .with_context(|_| AttachContainerSnafu {
+            .with_context(|_| AwaitDebugContainerReadinessSnafu {
                 pod: pod_ref(),
                 container: &self.container,
             })?;
-        info!("Attached to container, if the shell line looks empty, press ENTER!");
-        {
+            let debug_container_status = ready_pod
+                .as_ref()
+                .and_then(debug_container_status_of_pod(&debug_container_name))
+                .with_context(|| FindDebugContainerStatusSnafu {
+                    pod: pod_ref(),
+                    container: &self.container,
+                })?;
+            if let Some(termination) = debug_container_status
+                .last_state
+                .as_ref()
+                .and_then(|state| state.terminated.as_ref())
+            {
+                error!(
+                    error = termination.message,
+                    exit_code = termination.exit_code,
+                    "Debug container failed to start!"
+                );
+            }
+            info!("Attaching to container");
+            let mut attachment = pods
+                .attach(
+                    &self.pod,
+                    &AttachParams::interactive_tty().container(debug_container_name),
+                )
+                .await
+                .with_context(|_| AttachContainerSnafu {
+                    pod: pod_ref(),
+                    container: &self.container,
+                })?;
+            info!("Attached to container, if the shell line looks empty, press ENTER!");
             let _raw = std::io::stdout()
                 .into_raw_mode()
                 .context(SetRawTtyModeSnafu)?;
@@ -249,8 +245,10 @@ impl DebugArgs {
             ])
             .await
             .0?;
+            Ok(String::new())
         }
-        Ok(String::new())
+        .instrument(span)
+        .await
     }
 }
 
@@ -287,7 +285,7 @@ impl AsyncStdin {
                 return Err(std::io::Error::last_os_error()).context(AsyncifyStdinSnafu);
             }
             if old_flags & libc::O_NONBLOCK != 0 {
-                tracing::warn!("stdin is already non-blocking (did you try to create multiple AsyncStdin instances?)");
+                warn!("stdin is already non-blocking (did you try to create multiple AsyncStdin instances?)");
             }
             let status = unsafe {
                 libc::fcntl(

From 70af91adf90bbfcca7f8ecdadb4ad14495077fd2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= <nat@nullable.se>
Date: Wed, 28 Feb 2024 15:20:03 +0100
Subject: [PATCH 11/18] Changelog

---
 rust/stackablectl/CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/rust/stackablectl/CHANGELOG.md b/rust/stackablectl/CHANGELOG.md
index fa744333..053244ca 100644
--- a/rust/stackablectl/CHANGELOG.md
+++ b/rust/stackablectl/CHANGELOG.md
@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+### Added
+
+- Added experimental `debug` command ([#204]).
+
 ### Changed
 
 - Operators are now installed in parallel when installing a release ([#202]).
@@ -14,6 +18,7 @@ All notable changes to this project will be documented in this file.
 
 [#181]: https://github.com/stackabletech/stackable-cockpit/pull/181
 [#202]: https://github.com/stackabletech/stackable-cockpit/pull/202
+[#204]: https://github.com/stackabletech/stackable-cockpit/pull/204
 
 ## [23.11.3] - 2024-01-03
 

From 4069084d8fe20952f8c0173a6776e2d3654123e1 Mon Sep 17 00:00:00 2001
From: Nick Larsen <nick.larsen@stackable.tech>
Date: Mon, 4 Mar 2024 12:23:51 +0100
Subject: [PATCH 12/18] ci: use install-nix-action

---
 .github/workflows/pr_cockpit.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr_cockpit.yml b/.github/workflows/pr_cockpit.yml
index a0cee7e6..bda9a16a 100644
--- a/.github/workflows/pr_cockpit.yml
+++ b/.github/workflows/pr_cockpit.yml
@@ -87,7 +87,7 @@ jobs:
         uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
         with:
           submodules: recursive
-
+      - uses: cachix/install-nix-action@6004951b182f8860210c8d6f0d808ec5b1a33d28 # tag=v25
       - name: Setup Rust
         uses: dtolnay/rust-toolchain@0e66bd3e6b38ec0ad5312288c83e47c143e6b09e # v1
         with:

From 4d6ed4b6946acdff72938a58bf3fe294b4fad9bb Mon Sep 17 00:00:00 2001
From: Nick Larsen <nick.larsen@stackable.tech>
Date: Mon, 4 Mar 2024 12:46:58 +0100
Subject: [PATCH 13/18] fix(nix): add shebang to script
 (stackabletech/operator-templating#325)

---
 default.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/default.nix b/default.nix
index ec006975..9f18c20a 100644
--- a/default.nix
+++ b/default.nix
@@ -142,6 +142,7 @@ rec {
 
   regenerateNixLockfiles = pkgs.writeScriptBin "regenerate-nix-lockfiles"
     ''
+      #!/usr/bin/env bash
       set -euo pipefail
       echo Running crate2nix
       ${crate2nix}/bin/crate2nix generate

From 3fa1b7ee84a851a54c4b2bb4a5281ee52606cf0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= <nat@nullable.se>
Date: Tue, 12 Mar 2024 20:29:01 +0100
Subject: [PATCH 14/18] Move dependencies into root Cargo.toml

---
 Cargo.toml                   | 4 +++-
 rust/stackablectl/Cargo.toml | 5 ++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index ca22e305..4eb9bb41 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -30,6 +30,7 @@ indexmap = { version = "2.0", features = ["serde"] }
 k8s-openapi = { version = "0.20", default-features = false, features = ["v1_28"] }
 kube = { version = "0.87", default-features = false, features = ["client", "rustls-tls", "ws"] }
 lazy_static = "1.4"
+libc = "0.2.153"
 once_cell = "1.18"
 phf = "0.11"
 phf_codegen = "0.11"
@@ -45,7 +46,8 @@ sha2 = "0.10"
 snafu = { version = "0.7", features = ["futures"] }
 stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.61.0" }
 tera = "1.18"
-tokio = { version = "1.29.0", features = ["rt-multi-thread", "macros", "fs", "process"] }
+termion = "3.0.0"
+tokio = { version = "1.29.0", features = ["rt-multi-thread", "macros", "fs", "process", "io-std"] }
 tower-http = "0.4"
 tracing = "0.1"
 tracing-subscriber = "0.3"
diff --git a/rust/stackablectl/Cargo.toml b/rust/stackablectl/Cargo.toml
index ffda21ef..5306cfdf 100644
--- a/rust/stackablectl/Cargo.toml
+++ b/rust/stackablectl/Cargo.toml
@@ -29,9 +29,8 @@ serde.workspace = true
 snafu.workspace = true
 tera.workspace = true
 tokio.workspace = true
-tokio.features = ["io-std"]
 tracing-subscriber.workspace = true
 tracing.workspace = true
 futures.workspace = true
-termion = "3.0.0"
-libc = "0.2.153"
+termion.workspace = true
+libc.workspace = true

From 1567a8da411f86178862147014e7e1e14ca068a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= <nat@nullable.se>
Date: Tue, 12 Mar 2024 20:32:14 +0100
Subject: [PATCH 15/18] Uglify formatting

---
 rust/stackablectl/src/cmds/debug.rs | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/rust/stackablectl/src/cmds/debug.rs b/rust/stackablectl/src/cmds/debug.rs
index 51bd5a89..9cc0a794 100644
--- a/rust/stackablectl/src/cmds/debug.rs
+++ b/rust/stackablectl/src/cmds/debug.rs
@@ -32,57 +32,73 @@ use crate::cli::Cli;
 pub enum CmdError {
     #[snafu(display("failed to create Kubernetes client"))]
     KubeClientCreate { source: kube::Error },
+
     #[snafu(display("failed to get {pod}"))]
     GetPod {
         source: kube::Error,
         pod: ObjectRef<Pod>,
     },
+
     #[snafu(display("{pod} has no container {container:?}"))]
     FindTemplateContainer {
         pod: ObjectRef<Pod>,
         container: String,
     },
+
     #[snafu(display("failed to create ephemeral debug container {container:?} on {pod}"))]
     CreateDebugContainer {
         source: kube::Error,
         pod: ObjectRef<Pod>,
         container: String,
     },
+
     #[snafu(display("debug container {container:?} on {pod} never became ready"))]
     AwaitDebugContainerReadiness {
         source: kube::runtime::wait::Error,
         pod: ObjectRef<Pod>,
         container: String,
     },
+
     #[snafu(display("failed to get status of debug container {container:?} on {pod}"))]
     FindDebugContainerStatus {
         pod: ObjectRef<Pod>,
         container: String,
     },
+
     #[snafu(display("failed to attach to container {container:?} on {pod}"))]
     AttachContainer {
         source: kube::Error,
         pod: ObjectRef<Pod>,
         container: String,
     },
+
     #[snafu(display("failed to enable raw local TTY input"))]
     SetRawTtyMode { source: std::io::Error },
+
     #[snafu(display("failed to turn stdin async"))]
     AsyncifyStdin { source: std::io::Error },
+
     #[snafu(display("failed to initialize AsyncFd for stdin"))]
     AsyncFdStdin { source: std::io::Error },
+
     #[snafu(display("container has no terminal size channel"))]
     NoTerminalSizeChannel,
+
     #[snafu(display("failed to read terminal size"))]
     GetTerminalSize { source: std::io::Error },
+
     #[snafu(display("failed to update terminal size"))]
     UpdateTerminalSize { source: mpsc::SendError },
+
     #[snafu(display("container has no stdin channel"))]
     NoStdinChannel,
+
     #[snafu(display("container has no stdout channel"))]
     NoStdoutChannel,
+
     #[snafu(display("failed to forward stdin to container"))]
     ForwardStdin { source: std::io::Error },
+
     #[snafu(display("failed to forward stdout from container"))]
     ForwardStdout { source: std::io::Error },
 }
@@ -93,16 +109,20 @@ pub struct DebugArgs {
     /// The namespace of the Pod being debugged
     #[clap(long, short)]
     namespace: Option<String>,
+
     /// The Pod to debug
     pod: String,
+
     /// The target container to debug
     ///
     /// Volumes and environment variables will be copied from this container.
     #[clap(long, short)]
     container: String,
+
     /// The debug container image
     #[clap(long)]
     image: String,
+
     /// The command to run in the debug container
     #[clap(last = true)]
     cmd: Option<Vec<String>>,

From 335e1ac0c4fa14b68c223c487704b37bbde1f171 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= <nat@nullable.se>
Date: Tue, 12 Mar 2024 20:35:00 +0100
Subject: [PATCH 16/18] Change the container prefix from sble-debug- to
 stackablectl-debug-

---
 rust/stackablectl/src/cmds/debug.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rust/stackablectl/src/cmds/debug.rs b/rust/stackablectl/src/cmds/debug.rs
index 9cc0a794..c0e64b8a 100644
--- a/rust/stackablectl/src/cmds/debug.rs
+++ b/rust/stackablectl/src/cmds/debug.rs
@@ -274,7 +274,7 @@ impl DebugArgs {
 
 fn generate_debug_container_name() -> String {
     let mut rng = rand::thread_rng();
-    let mut name = "sble-debug-".to_string();
+    let mut name = "stackablectl-debug-".to_string();
     for _ in 0..5 {
         name.push(rng.gen_range('a'..='z'));
     }

From 9d26417dd1a7c1223b579d7099efefff8ebf0cbb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= <nat@nullable.se>
Date: Tue, 12 Mar 2024 21:20:28 +0100
Subject: [PATCH 17/18] Default to the target container's image

---
 rust/stackablectl/src/cmds/debug.rs | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/rust/stackablectl/src/cmds/debug.rs b/rust/stackablectl/src/cmds/debug.rs
index c0e64b8a..50345274 100644
--- a/rust/stackablectl/src/cmds/debug.rs
+++ b/rust/stackablectl/src/cmds/debug.rs
@@ -120,8 +120,10 @@ pub struct DebugArgs {
     container: String,
 
     /// The debug container image
+    ///
+    /// Defaults to the image of the target container if not specified.
     #[clap(long)]
-    image: String,
+    image: Option<String>,
 
     /// The command to run in the debug container
     #[clap(last = true)]
@@ -159,7 +161,10 @@ impl DebugArgs {
                 spec: Some(PodSpec {
                     ephemeral_containers: Some(vec![EphemeralContainer {
                         name: debug_container_name.clone(),
-                        image: Some(self.image.clone()),
+                        image: self
+                            .image
+                            .clone()
+                            .or_else(|| template_container.image.clone()),
                         tty: Some(true),
                         stdin: Some(true),
 

From bc4b32a6194c6082d249daf5439fd83628bbc2cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= <nat@nullable.se>
Date: Tue, 12 Mar 2024 21:26:44 +0100
Subject: [PATCH 18/18] Run debug container as root

---
 rust/stackablectl/src/cmds/debug.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/rust/stackablectl/src/cmds/debug.rs b/rust/stackablectl/src/cmds/debug.rs
index 50345274..a9742a82 100644
--- a/rust/stackablectl/src/cmds/debug.rs
+++ b/rust/stackablectl/src/cmds/debug.rs
@@ -12,6 +12,7 @@ use futures::{
 use rand::Rng;
 use snafu::{futures::TryFutureExt as _, OptionExt, ResultExt, Snafu};
 use stackable_operator::{
+    builder::SecurityContextBuilder,
     k8s_openapi::api::core::v1::{ContainerStatus, EphemeralContainer, Pod, PodSpec},
     kube::{
         self,
@@ -171,6 +172,8 @@ impl DebugArgs {
                         command: self.cmd.clone(),
                         args: self.cmd.is_some().then(Vec::new),
 
+                        security_context: Some(SecurityContextBuilder::run_as_root()),
+
                         // copy environment from template
                         env: template_container.env.clone(),
                         env_from: template_container.env_from.clone(),