feat: allow setting exit ip address (#584)

Co-authored-by: Barnabas Busa <busa.barnabas@gmail.com>

.github/tests/mix-public.yaml

@@ -0,0 +1,18 @@
+participants:
+  - el_type: geth
+    cl_type: teku
+  - el_type: nethermind
+    cl_type: prysm
+  - el_type: erigon
+    cl_type: nimbus
+  - el_type: besu
+    cl_type: lighthouse
+  - el_type: reth
+    cl_type: lodestar
+  - el_type: nimbus
+    cl_type: teku
+  - el_type: ethereumjs
+    cl_type: grandine
+additional_services: []
+port_publisher:
+  public_port_start: 50000

.github/tests/mix.yaml

@@ -13,4 +13,4 @@ participants:
     cl_type: teku
   - el_type: ethereumjs
     cl_type: grandine
-additional_services: []
+additional_services: []

.github/workflows/per-pr.yml

@@ -33,6 +33,7 @@ jobs:
             "./.github/tests/mev-mock.yaml",
             "./.github/tests/mix-with-tools.yaml",
             "./.github/tests/mix-persistence.yaml",
+            "./.github/tests/mix-public.yaml",
             "./network_params.yaml"
           ]
     runs-on: ubuntu-latest

README.md

@@ -752,6 +752,17 @@ global_node_selectors: {}
 # This will open up http ports to your validator services!
 # Defaults to false
 keymanager_enabled: false
+
+# Global paarameter to set the exit ip address of services and public ports
+port_publisher:
+  # if you have a service that you want to expose on a specific interfact; set that IP here
+  # if you set it to auto it gets the public ip from ident.me and sets it
+  # Defaults to constants.PRIVATE_IP_ADDRESS_PLACEHOLDER
+  # The default value just means its the IP address of the container in which the service is running
+  nat_exit_ip: KURTOSIS_IP_ADDR_PLACEHOLDER
+  # The start value gets used as a seed for TCP and UDP discovery ports for el/cl client
+  # Defaults to None - no public ports
+  public_port_start: None
 ```
 
 #### Example configurations

main.star

@@ -115,6 +115,7 @@ def run(plan, args={}):
         global_node_selectors,
         keymanager_enabled,
         parallel_keystore_generation,
+        args_with_right_defaults.port_publisher,
     )
 
     plan.print(

network_params.yaml

@@ -138,3 +138,6 @@ xatu_sentry_params:
 global_tolerations: []
 global_node_selectors: {}
 keymanager_enabled: false
+port_publisher:
+  public_port_start: null
+  nat_exit_ip: KURTOSIS_IP_ADDR_PLACEHOLDER

src/blobber/blobber_launcher.star

@@ -1,5 +1,6 @@
 shared_utils = import_module("../shared_utils/shared_utils.star")
 input_parser = import_module("../package_io/input_parser.star")
+constants = import_module("../package_io/constants.star")
 cl_context = import_module("../cl/cl_context.star")
 
 blobber_context = import_module("../blobber/blobber_context.star")
@@ -10,8 +11,6 @@ BLOBBER_BEACON_PORT_UDP_ID = "discovery-udp"
 BLOBBER_VALIDATOR_PROXY_PORT_NUM = 5000
 BLOBBER_VALIDATOR_PROXY_PORT_ID = "http"
 
-PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
-
 DEFAULT_BLOBBER_IMAGE = "ethpandaops/blobber:1.1.0"
 
 VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/validator-keys"
@@ -76,7 +75,8 @@ def get_config(
         "--cl={0}".format(beacon_http_url),
         "--validator-key-folder={0}".format(validator_root_dirpath),
         "--enable-unsafe-mode",
-        "--external-ip={0}".format(PRIVATE_IP_ADDRESS_PLACEHOLDER),
+        # Does this get affected by public ip address changes?
+        "--external-ip={0}".format(constants.PRIVATE_IP_ADDRESS_PLACEHOLDER),
         "--validator-proxy-port-start={0}".format(BLOBBER_VALIDATOR_PROXY_PORT_NUM),
     ]
 
@@ -90,7 +90,7 @@ def get_config(
             VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid
         },
         cmd=cmd,
-        private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
+        private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
         min_cpu=MIN_CPU,
         max_cpu=MAX_CPU,
         min_memory=MIN_MEMORY,

src/cl/cl_launcher.star

@@ -31,6 +31,7 @@ def launch(
     validator_data,
     prysm_password_relative_filepath,
     prysm_password_artifact_uuid,
+    port_publisher,
 ):
     plan.print("Launching CL network")
 
@@ -175,6 +176,7 @@ def launch(
                 node_selectors,
                 participant.use_separate_vc,
                 participant.keymanager_enabled,
+                port_publisher,
             )
         else:
             boot_cl_client_ctx = all_cl_contexts
@@ -208,6 +210,7 @@ def launch(
                 node_selectors,
                 participant.use_separate_vc,
                 participant.keymanager_enabled,
+                port_publisher,
             )
 
         # Add participant cl additional prometheus labels

src/cl/grandine/grandine_launcher.star

@@ -29,22 +29,23 @@ BEACON_METRICS_PATH = "/metrics"
 
 MIN_PEERS = 1
 
-PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
-
-BEACON_USED_PORTS = {
-    BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
-        BEACON_DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL
-    ),
-    BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
-        BEACON_DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL
-    ),
-    BEACON_HTTP_PORT_ID: shared_utils.new_port_spec(
-        BEACON_HTTP_PORT_NUM, shared_utils.TCP_PROTOCOL
-    ),
-    BEACON_METRICS_PORT_ID: shared_utils.new_port_spec(
-        BEACON_METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL
-    ),
-}
+
+def get_used_ports(discovery_port):
+    beacon_used_ports = {
+        BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
+            discovery_port, shared_utils.TCP_PROTOCOL
+        ),
+        BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
+            discovery_port, shared_utils.UDP_PROTOCOL
+        ),
+        BEACON_HTTP_PORT_ID: shared_utils.new_port_spec(
+            BEACON_HTTP_PORT_NUM, shared_utils.TCP_PROTOCOL
+        ),
+        BEACON_METRICS_PORT_ID: shared_utils.new_port_spec(
+            BEACON_METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL
+        ),
+    }
+    return beacon_used_ports
 
 
 ENTRYPOINT_ARGS = ["sh", "-c"]
@@ -88,6 +89,7 @@ def launch(
     node_selectors,
     use_separate_vc,
     keymanager_enabled,
+    port_publisher,
 ):
     beacon_service_name = "{0}".format(service_name)
     log_level = input_parser.get_client_log_level_or_default(
@@ -148,6 +150,7 @@ def launch(
         cl_volume_size,
         tolerations,
         node_selectors,
+        port_publisher,
     )
 
     beacon_service = plan.add_service(service_name, config)
@@ -227,6 +230,7 @@ def get_beacon_config(
     cl_volume_size,
     tolerations,
     node_selectors,
+    port_publisher,
 ):
     validator_keys_dirpath = ""
     validator_secrets_dirpath = ""
@@ -250,22 +254,39 @@ def get_beacon_config(
             el_context.ip_addr,
             el_context.engine_rpc_port_num,
         )
+
+    public_ports = {}
+    discovery_port = BEACON_DISCOVERY_PORT_NUM
+    if port_publisher.public_port_start:
+        discovery_port = port_publisher.cl_start
+        if bootnode_contexts and len(bootnode_contexts) > 0:
+            discovery_port = discovery_port + len(bootnode_contexts)
+        public_ports = {
+            BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
+                discovery_port, shared_utils.TCP_PROTOCOL
+            ),
+            BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
+                discovery_port, shared_utils.UDP_PROTOCOL
+            ),
+        }
+    used_ports = get_used_ports(discovery_port)
+
     cmd = [
         "--network={0}".format(
             network if network in constants.PUBLIC_NETWORKS else "custom"
         ),
         "--data-dir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER,
         "--http-address=0.0.0.0",
         "--http-port={0}".format(BEACON_HTTP_PORT_NUM),
-        "--libp2p-port={0}".format(BEACON_DISCOVERY_PORT_NUM),
-        "--discovery-port={0}".format(BEACON_DISCOVERY_PORT_NUM),
+        "--libp2p-port={0}".format(discovery_port),
+        "--discovery-port={0}".format(discovery_port),
         "--jwt-secret=" + constants.JWT_MOUNT_PATH_ON_CONTAINER,
         "--eth1-rpc-urls=" + EXECUTION_ENGINE_ENDPOINT,
         # vvvvvvvvvvvvvvvvvvv REMOVE THESE WHEN CONNECTING TO EXTERNAL NET vvvvvvvvvvvvvvvvvvvvv
         "--disable-enr-auto-update",
-        "--enr-address=" + PRIVATE_IP_ADDRESS_PLACEHOLDER,
-        "--enr-udp-port={0}".format(BEACON_DISCOVERY_PORT_NUM),
-        "--enr-tcp-port={0}".format(BEACON_DISCOVERY_PORT_NUM),
+        "--enr-address=" + port_publisher.nat_exit_ip,
+        "--enr-udp-port={0}".format(discovery_port),
+        "--enr-tcp-port={0}".format(discovery_port),
         # ^^^^^^^^^^^^^^^^^^^ REMOVE THESE WHEN CONNECTING TO EXTERNAL NET ^^^^^^^^^^^^^^^^^^^^^
         # vvvvvvvvvvvvvvvvvvv METRICS CONFIG vvvvvvvvvvvvvvvvvvvvv
         "--metrics",
@@ -353,7 +374,7 @@ def get_beacon_config(
     }
 
     ports = {}
-    ports.update(BEACON_USED_PORTS)
+    ports.update(used_ports)
     if node_keystore_files != None and not use_separate_vc:
         cmd.extend(validator_default_cmd)
         files[
@@ -373,10 +394,11 @@ def get_beacon_config(
     return ServiceConfig(
         image=image,
         ports=ports,
+        public_ports=public_ports,
         cmd=cmd,
         env_vars=extra_env_vars,
         files=files,
-        private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
+        private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
         ready_conditions=cl_node_ready_conditions.get_ready_conditions(
             BEACON_HTTP_PORT_ID
         ),

src/cl/lighthouse/lighthouse_launcher.star

@@ -32,26 +32,28 @@ BEACON_MIN_MEMORY = 256
 
 METRICS_PATH = "/metrics"
 
-PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
-
-BEACON_USED_PORTS = {
-    BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
-        BEACON_DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL
-    ),
-    BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
-        BEACON_DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL
-    ),
-    BEACON_HTTP_PORT_ID: shared_utils.new_port_spec(
-        BEACON_HTTP_PORT_NUM,
-        shared_utils.TCP_PROTOCOL,
-        shared_utils.HTTP_APPLICATION_PROTOCOL,
-    ),
-    BEACON_METRICS_PORT_ID: shared_utils.new_port_spec(
-        BEACON_METRICS_PORT_NUM,
-        shared_utils.TCP_PROTOCOL,
-        shared_utils.HTTP_APPLICATION_PROTOCOL,
-    ),
-}
+
+def get_used_ports(discovery_port):
+    beacon_used_ports = {
+        BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
+            discovery_port, shared_utils.TCP_PROTOCOL
+        ),
+        BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
+            discovery_port, shared_utils.UDP_PROTOCOL
+        ),
+        BEACON_HTTP_PORT_ID: shared_utils.new_port_spec(
+            BEACON_HTTP_PORT_NUM,
+            shared_utils.TCP_PROTOCOL,
+            shared_utils.HTTP_APPLICATION_PROTOCOL,
+        ),
+        BEACON_METRICS_PORT_ID: shared_utils.new_port_spec(
+            BEACON_METRICS_PORT_NUM,
+            shared_utils.TCP_PROTOCOL,
+            shared_utils.HTTP_APPLICATION_PROTOCOL,
+        ),
+    }
+    return beacon_used_ports
+
 
 VERBOSITY_LEVELS = {
     constants.GLOBAL_LOG_LEVEL.error: "error",
@@ -90,8 +92,9 @@ def launch(
     participant_tolerations,
     global_tolerations,
     node_selectors,
-    use_separate_vc=True,
-    keymanager_enabled=False,
+    use_separate_vc,
+    keymanager_enabled,
+    port_publisher,
 ):
     beacon_service_name = "{0}".format(service_name)
 
@@ -148,6 +151,7 @@ def launch(
         cl_volume_size,
         tolerations,
         node_selectors,
+        port_publisher,
     )
 
     beacon_service = plan.add_service(beacon_service_name, beacon_config)
@@ -242,6 +246,7 @@ def get_beacon_config(
     cl_volume_size,
     tolerations,
     node_selectors,
+    port_publisher,
 ):
     # If snooper is enabled use the snooper engine context, otherwise use the execution client context
     if snooper_enabled:
@@ -255,6 +260,22 @@ def get_beacon_config(
             el_context.engine_rpc_port_num,
         )
 
+    public_ports = {}
+    discovery_port = BEACON_DISCOVERY_PORT_NUM
+    if port_publisher.public_port_start:
+        discovery_port = port_publisher.cl_start
+        if boot_cl_client_ctxs and len(boot_cl_client_ctxs) > 0:
+            discovery_port = discovery_port + len(boot_cl_client_ctxs)
+        public_ports = {
+            BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
+                discovery_port, shared_utils.TCP_PROTOCOL
+            ),
+            BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
+                discovery_port, shared_utils.UDP_PROTOCOL
+            ),
+        }
+    used_ports = get_used_ports(discovery_port)
+
     # NOTE: If connecting to the merge devnet remotely we DON'T want the following flags; when they're not set, the node's external IP address is auto-detected
     #  from the peers it communicates with but when they're set they basically say "override the autodetection and
     #  use what I specify instead." This requires having a know external IP address and port, which we definitely won't
@@ -270,13 +291,13 @@ def get_beacon_config(
         "--datadir=" + BEACON_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER,
         # vvvvvvvvvvvvvvvvvvv REMOVE THESE WHEN CONNECTING TO EXTERNAL NET vvvvvvvvvvvvvvvvvvvvv
         "--disable-enr-auto-update",
-        "--enr-address=" + PRIVATE_IP_ADDRESS_PLACEHOLDER,
-        "--enr-udp-port={0}".format(BEACON_DISCOVERY_PORT_NUM),
-        "--enr-tcp-port={0}".format(BEACON_DISCOVERY_PORT_NUM),
+        "--enr-address=" + port_publisher.nat_exit_ip,
+        "--enr-udp-port={0}".format(discovery_port),
+        "--enr-tcp-port={0}".format(discovery_port),
         # ^^^^^^^^^^^^^^^^^^^ REMOVE THESE WHEN CONNECTING TO EXTERNAL NET ^^^^^^^^^^^^^^^^^^^^^
         "--listen-address=0.0.0.0",
         "--port={0}".format(
-            BEACON_DISCOVERY_PORT_NUM
+            discovery_port
         ),  # NOTE: Remove for connecting to external net!
         "--http",
         "--http-address=0.0.0.0",
@@ -368,11 +389,12 @@ def get_beacon_config(
     env.update(extra_env_vars)
     return ServiceConfig(
         image=image,
-        ports=BEACON_USED_PORTS,
+        ports=used_ports,
+        public_ports=public_ports,
         cmd=cmd,
         files=files,
         env_vars=env,
-        private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
+        private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
         ready_conditions=cl_node_ready_conditions.get_ready_conditions(
             BEACON_HTTP_PORT_ID
         ),