From 0fb8bc4a863d8e5fbe63af88b15ea9bf89f08e9d Mon Sep 17 00:00:00 2001
From: Cheick Keita <chkeita@microsoft.com>
Date: Thu, 15 Dec 2022 19:39:17 -0800
Subject: [PATCH] Bug fixes and documentation (#2694)

---
 docs/unmnaged-nodes.md                        | 80 +++++++++++++++++++
 .../ApiService/Functions/AgentRegistration.cs |  4 +-
 .../ApiService/OneFuzzTypes/Model.cs          |  3 +-
 .../ApiService/ServiceConfiguration.cs        |  2 +
 src/ApiService/ApiService/onefuzzlib/Creds.cs |  6 +-
 .../ApiService/onefuzzlib/NodeOperations.cs   |  4 +
 .../Fakes/TestServiceConfiguration.cs         |  2 +
 src/agent/onefuzz-agent/src/config.rs         | 11 ++-
 src/agent/onefuzz-agent/src/main.rs           | 23 +++---
 src/agent/onefuzz/src/auth.rs                 |  4 +-
 src/cli/onefuzz/api.py                        | 11 +++
 src/deployment/deploylib/registration.py      |  5 +-
 src/pytypes/onefuzztypes/models.py            |  3 +
 src/runtime-tools/linux/set-env.sh            |  6 ++
 src/runtime-tools/win64/set-env.ps1           |  6 ++
 15 files changed, 147 insertions(+), 23 deletions(-)
 create mode 100644 docs/unmnaged-nodes.md
 create mode 100644 src/runtime-tools/linux/set-env.sh
 create mode 100644 src/runtime-tools/win64/set-env.ps1

diff --git a/docs/unmnaged-nodes.md b/docs/unmnaged-nodes.md
new file mode 100644
index 0000000000..0850fb0c68
--- /dev/null
+++ b/docs/unmnaged-nodes.md
@@ -0,0 +1,80 @@
+# Unmanaged Nodes
+The default mode of OneFuzz is to run the agents inside scalesets managed by the the Onefuzz instance. But it is possible to run outside of the Instance infrastructure.
+This is the unmanaged scenario. In this mode, the user can use their own resource to participate in the fuzzing.
+
+## Set-up
+These are the steps to run an unmanaged node
+
+
+### Create an Application Registration in Azure Active Directory
+We will create the authentication method for the unmanaged node.
+From the [azure cli](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) create a new **application registration**:
+```cmd
+az ad app create --display-name <registration_name>
+```
+Then use the application `app_id` in the result to create the associated **service principal**:
+
+```cmd
+az ad sp create --id <app_id>
+```
+Take note of the `id` returned by this request. We will call it the `principal_id`.
+
+Next, create a `client_secret`:
+
+```
+az ad app credential reset --id <pp_id> --append
+```
+Take note of the `password` returned.
+
+### Authorize the application in OneFuzz
+From the OneFuzz `deployment` folder run the following script using the `app_id` from above:
+``` cmd
+python .\deploylib\registration.py register_app <onefuzz_instance_id> <subscription_id> --app_id <app_id> --role UnmanagedNode
+```
+
+### Create an unmanaged pool
+Using the OneFuzz CLI:
+``` cmd
+onefuzz pools create <pool_name> <os> --unmanaged --object_id <principal_id>
+```
+
+### Download the agent binaries and the agent configuration
+Download a zip file containing the agent binaries:
+```
+onefuzz tools get <destination_folder>
+```
+Extract the zip file in a folder of your choice.
+
+Download the configuration file for the agent:
+
+```
+onefuzz pools get_config <pool_name>
+```
+
+Under the `client_credential` section of the agent config file, update `client_id` and `client_secret`:
+```json
+{
+    "client_id": "<app_id>",
+    "client_secret": "<password>",
+}
+```
+Save the config to the file.
+
+### Start the agent.
+Navigate to the folder corresponding to your OS.
+Set the necessary environment variable by running the script `set-env.ps1` (for Windows) or `set-env.sh` (for Linux).
+Run the agent with the following command. If you need more nodes use a different `machine_guid` for each one:
+```cmd
+onefuzz-agent run --machine_id <machine_guid> -c <path_to_config_file> --reset_lock
+```
+
+### Verify that the agent is registered to OneFuzz
+
+Using the OneFuzz CLI run the following command:
+
+```
+onefuzz nodes get <machine_guid>
+```
+
+This should return one entry. Verify that the `pool_name` matched the pool name created earlier.
+From here you will be able to schedule jobs on that pool and they will be running.
\ No newline at end of file
diff --git a/src/ApiService/ApiService/Functions/AgentRegistration.cs b/src/ApiService/ApiService/Functions/AgentRegistration.cs
index 0236372d85..ce380c7c78 100644
--- a/src/ApiService/ApiService/Functions/AgentRegistration.cs
+++ b/src/ApiService/ApiService/Functions/AgentRegistration.cs
@@ -152,7 +152,9 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
             MachineId: machineId,
             ScalesetId: scalesetId,
             InstanceId: instanceId,
-            Version: version
+            Version: version,
+            Os: os ?? pool.Os,
+            Managed: pool.Managed
             );
 
         var r = await _context.NodeOperations.Replace(node);
diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs
index ff274f0a31..83ed622734 100644
--- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs
+++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs
@@ -110,7 +110,8 @@ public record Node
 
     bool ReimageRequested = false,
     bool DeleteRequested = false,
-    bool DebugKeepNode = false
+    bool DebugKeepNode = false,
+    bool Managed = true
 ) : StatefulEntityBase<NodeState>(State) {
 
     public List<NodeTasks>? Tasks { get; set; }
diff --git a/src/ApiService/ApiService/ServiceConfiguration.cs b/src/ApiService/ApiService/ServiceConfiguration.cs
index 57f056122c..9b50c53f23 100644
--- a/src/ApiService/ApiService/ServiceConfiguration.cs
+++ b/src/ApiService/ApiService/ServiceConfiguration.cs
@@ -32,6 +32,7 @@ public interface IServiceConfig {
     public ResourceIdentifier? OneFuzzFuncStorage { get; }
     public string? OneFuzzInstance { get; }
     public string? OneFuzzInstanceName { get; }
+    public string? OneFuzzEndpoint { get; }
     public string? OneFuzzKeyvault { get; }
 
     public string? OneFuzzMonitor { get; }
@@ -117,6 +118,7 @@ public ResourceIdentifier? OneFuzzFuncStorage {
 
     public string? OneFuzzInstance { get => GetEnv("ONEFUZZ_INSTANCE"); }
     public string? OneFuzzInstanceName { get => GetEnv("ONEFUZZ_INSTANCE_NAME"); }
+    public string? OneFuzzEndpoint { get => GetEnv("ONEFUZZ_ENDPOINT"); }
     public string? OneFuzzKeyvault { get => GetEnv("ONEFUZZ_KEYVAULT"); }
     public string? OneFuzzMonitor { get => GetEnv("ONEFUZZ_MONITOR"); }
     public string? OneFuzzOwner { get => GetEnv("ONEFUZZ_OWNER"); }
diff --git a/src/ApiService/ApiService/onefuzzlib/Creds.cs b/src/ApiService/ApiService/onefuzzlib/Creds.cs
index d12b48d5a6..5e085861f5 100644
--- a/src/ApiService/ApiService/onefuzzlib/Creds.cs
+++ b/src/ApiService/ApiService/onefuzzlib/Creds.cs
@@ -105,8 +105,10 @@ public Async.Task<Region> GetBaseRegion() {
         });
     }
 
-    public Uri GetInstanceUrl()
-        => new($"https://{GetInstanceName()}.azurewebsites.net");
+    public Uri GetInstanceUrl() {
+        var onefuzzEndpoint = _config.OneFuzzEndpoint;
+        return onefuzzEndpoint != null ? new Uri(onefuzzEndpoint) : new($"https://{GetInstanceName()}.azurewebsites.net");
+    }
 
     public record ScaleSetIdentity(string principalId);
 
diff --git a/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs b/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs
index 5449ee247c..b3e7acae83 100644
--- a/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs
+++ b/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs
@@ -337,6 +337,10 @@ public async Async.Task CleanupBusyNodesWithoutWork() {
     }
 
     public async Async.Task<Node> ToReimage(Node node, bool done = false) {
+        if (!node.Managed) {
+            _logTracer.Info($"skip reimage for unmanaged node: {node.MachineId:Tag:MachineId}");
+            return node;
+        }
 
         var nodeState = node.State;
         if (done) {
diff --git a/src/ApiService/IntegrationTests/Fakes/TestServiceConfiguration.cs b/src/ApiService/IntegrationTests/Fakes/TestServiceConfiguration.cs
index 72a1622d44..eb6d487e8e 100644
--- a/src/ApiService/IntegrationTests/Fakes/TestServiceConfiguration.cs
+++ b/src/ApiService/IntegrationTests/Fakes/TestServiceConfiguration.cs
@@ -31,6 +31,8 @@ public TestServiceConfiguration(string tablePrefix) {
 
     // -- Remainder not implemented --
 
+    public string? OneFuzzEndpoint => throw new System.NotImplementedException();
+
     public LogDestination[] LogDestinations { get => throw new System.NotImplementedException(); set => throw new System.NotImplementedException(); }
 
     public SeverityLevel LogSeverityLevel => throw new System.NotImplementedException();
diff --git a/src/agent/onefuzz-agent/src/config.rs b/src/agent/onefuzz-agent/src/config.rs
index 3e50ab7ae6..d26d54e7f0 100644
--- a/src/agent/onefuzz-agent/src/config.rs
+++ b/src/agent/onefuzz-agent/src/config.rs
@@ -80,7 +80,7 @@ struct RawStaticConfig {
 }
 
 impl StaticConfig {
-    pub async fn new(data: &[u8]) -> Result<Self> {
+    pub async fn new(data: &[u8], machine_identity: Option<MachineIdentity>) -> Result<Self> {
         let config: RawStaticConfig = serde_json::from_slice(data)?;
 
         let credentials = match config.client_credentials {
@@ -104,7 +104,7 @@ impl StaticConfig {
                 managed.into()
             }
         };
-        let machine_identity = match config.machine_identity {
+        let machine_identity = match machine_identity.or(config.machine_identity) {
             Some(machine_identity) => machine_identity,
             None => MachineIdentity::from_metadata().await?,
         };
@@ -125,11 +125,14 @@ impl StaticConfig {
         Ok(config)
     }
 
-    pub async fn from_file(config_path: impl AsRef<Path>) -> Result<Self> {
+    pub async fn from_file(
+        config_path: impl AsRef<Path>,
+        machine_identity: Option<MachineIdentity>,
+    ) -> Result<Self> {
         let config_path = config_path.as_ref();
         let data = std::fs::read(config_path)
             .with_context(|| format!("unable to read config file: {}", config_path.display()))?;
-        Self::new(&data).await
+        Self::new(&data, machine_identity).await
     }
 
     pub fn from_env() -> Result<Self> {
diff --git a/src/agent/onefuzz-agent/src/main.rs b/src/agent/onefuzz-agent/src/main.rs
index 12428db2ba..38ca6ef621 100644
--- a/src/agent/onefuzz-agent/src/main.rs
+++ b/src/agent/onefuzz-agent/src/main.rs
@@ -170,8 +170,6 @@ fn run(opt: RunOpt) -> Result<()> {
     if opt.redirect_output.is_some() {
         return redirect(opt);
     }
-    let opt_machine_id = opt.machine_id;
-    let opt_machine_name = opt.machine_name.clone();
     let rt = tokio::runtime::Runtime::new()?;
     let reset_lock = opt.reset_node_lock;
     let config = rt.block_on(load_config(opt));
@@ -184,15 +182,6 @@ fn run(opt: RunOpt) -> Result<()> {
 
     let config = config?;
 
-    let config = StaticConfig {
-        machine_identity: MachineIdentity {
-            machine_id: opt_machine_id.unwrap_or(config.machine_identity.machine_id),
-            machine_name: opt_machine_name.unwrap_or(config.machine_identity.machine_name),
-            ..config.machine_identity
-        },
-        ..config
-    };
-
     if reset_lock {
         done::remove_done_lock(config.machine_identity.machine_id)?;
     } else if done::is_agent_done(config.machine_identity.machine_id)? {
@@ -218,10 +207,18 @@ fn run(opt: RunOpt) -> Result<()> {
 }
 
 async fn load_config(opt: RunOpt) -> Result<StaticConfig> {
-    info!("loading supervisor agent config");
+    info!("loading supervisor agent config: {:?}", opt);
+    let opt_machine_id = opt.machine_id;
+    let opt_machine_name = opt.machine_name.clone();
+
+    let machine_identity = opt_machine_id.map(|machine_id| MachineIdentity {
+        machine_id,
+        machine_name: opt_machine_name.unwrap_or(format!("{}", machine_id)),
+        scaleset_name: None,
+    });
 
     let config = match &opt.config_path {
-        Some(config_path) => StaticConfig::from_file(config_path).await?,
+        Some(config_path) => StaticConfig::from_file(config_path, machine_identity).await?,
         None => StaticConfig::from_env()?,
     };
 
diff --git a/src/agent/onefuzz/src/auth.rs b/src/agent/onefuzz/src/auth.rs
index d25a3807f9..2def380c3b 100644
--- a/src/agent/onefuzz/src/auth.rs
+++ b/src/agent/onefuzz/src/auth.rs
@@ -113,9 +113,11 @@ impl ClientCredentials {
     pub async fn access_token(&self) -> Result<AccessToken> {
         let (authority, scope) = {
             let url = Url::parse(&self.resource.clone())?;
-            let host = url.host_str().ok_or_else(|| {
+            let port = url.port().map(|p| format!(":{}", p)).unwrap_or_default();
+            let host_name = url.host_str().ok_or_else(|| {
                 anyhow::format_err!("resource URL does not have a host string: {}", url)
             })?;
+            let host = format!("{}{}", host_name, port);
             if let Some(domain) = &self.multi_tenant_domain {
                 let instance: Vec<&str> = host.split('.').collect();
                 (
diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py
index f7659356d7..a7b579c57f 100644
--- a/src/cli/onefuzz/api.py
+++ b/src/cli/onefuzz/api.py
@@ -14,6 +14,7 @@
 from enum import Enum
 from shutil import which
 from typing import Callable, Dict, List, Optional, Tuple, Type, TypeVar
+from urllib.parse import urlparse
 from uuid import UUID
 
 import semver
@@ -1268,6 +1269,16 @@ def get_config(self, pool_name: primitives.PoolName) -> models.AgentConfig:
         if pool.config is None:
             raise Exception("Missing AgentConfig in response")
 
+        config = pool.config
+        if not pool.managed:
+            config.client_credentials = models.ClientCredentials(  # nosec
+                client_id=uuid.UUID(int=0),
+                client_secret="<client_secret>",
+                resource=self.onefuzz._backend.config.endpoint,
+                tenant=urlparse(self.onefuzz._backend.config.authority).path.strip("/"),
+                multi_tenant_domain=self.onefuzz._backend.config.tenant_domain,
+            )
+
         return pool.config
 
     def shutdown(self, name: str, *, now: bool = False) -> responses.BoolResult:
diff --git a/src/deployment/deploylib/registration.py b/src/deployment/deploylib/registration.py
index 641ae62203..e5c5a42b21 100644
--- a/src/deployment/deploylib/registration.py
+++ b/src/deployment/deploylib/registration.py
@@ -861,10 +861,13 @@ def main() -> None:
         "--registration_name", help="the name of the cli registration"
     )
     register_app_parser = subparsers.add_parser("register_app", parents=[parent_parser])
-    register_app_parser.add_argument("--app_id", help="the application id to register")
+    register_app_parser.add_argument(
+        "--app_id", help="the application id to register", required=True
+    )
     register_app_parser.add_argument(
         "--role",
         help=f"the role of the application to register.  Valid values: {', '.join([member.value for member in OnefuzzAppRole])}",
+        required=True,
     )
 
     args = parser.parse_args()
diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py
index 858dd18c3d..60fa68cc12 100644
--- a/src/pytypes/onefuzztypes/models.py
+++ b/src/pytypes/onefuzztypes/models.py
@@ -328,6 +328,9 @@ class SyncedDir(BaseModel):
 class ClientCredentials(BaseModel):
     client_id: UUID
     client_secret: str
+    resource: str
+    tenant: str
+    multi_tenant_domain: Optional[str]
 
 
 class AgentConfig(BaseModel):
diff --git a/src/runtime-tools/linux/set-env.sh b/src/runtime-tools/linux/set-env.sh
new file mode 100644
index 0000000000..33ec3ddfd7
--- /dev/null
+++ b/src/runtime-tools/linux/set-env.sh
@@ -0,0 +1,6 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+export DOTNET_ROOT=/onefuzz/tools/dotnet
+export DOTNET_CLI_HOME="$DOTNET_ROOT"
+export LLVM_SYMBOLIZER_PATH=/onefuzz/bin/llvm-symbolizer
+export RUST_LOG = "info"
\ No newline at end of file
diff --git a/src/runtime-tools/win64/set-env.ps1 b/src/runtime-tools/win64/set-env.ps1
new file mode 100644
index 0000000000..20546bab07
--- /dev/null
+++ b/src/runtime-tools/win64/set-env.ps1
@@ -0,0 +1,6 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+$env:Path += ";C:\Program Files (x86)\Windows Kits\10\Debuggers\x64\;C:\onefuzz\win64;C:\onefuzz\tools\win64;C:\onefuzz\tools\win64\radamsa;$env:ProgramFiles\LLVM\bin"
+$env:LLVM_SYMBOLIZER_PATH = "C:\Program Files\LLVM\bin\llvm-symbolizer.exe"
+$env:RUST_LOG = "info"