docs: add deployment specific docs and refactor code to match (#213)

docs/README.md

@@ -6,4 +6,10 @@ NR Broker is a customizable software deployment business intellegence tool. It a
 
 Users can search, browse objects, view a graph representation and review activities using a web application. Developers integrate NR Broker into workflows by sending their intentions, requesting access and recording activity using http requests. As such, NR broker works on private on-premise clouds, AWS, OpenShift Clouds and more.
 
-<img src="./images/broker_architecture.png" alt="NR Broker Lifecycle" width="900"/>
+<img src="./images/broker_architecture.png" alt="NR Broker Lifecycle" width="900"/>
+
+## Deployment Specific Information
+
+If this documentation refers to things as 'deployment specific' then you should refer to your team's own documentation. Your deployment should show a link to your documentation on the homepage.
+
+This documentation is generic to all NR Broker installations.

docs/_sidebar.md

@@ -24,5 +24,9 @@
 ** [Adding Internal Users](/operations_internal_user.md)
 ** [Customize Homepage](/dev_customize_connections.md)
 
+* Deploy
+** [Broker Token](/dev_broker_token.md)
+** [Environment Variables](/dev_env_vars.md)
+
 * Reference
 ** [Elastic Common Schema](https://www.elastic.co/guide/en/ecs/current/ecs-reference.html)

docs/dev_broker_token.md

@@ -0,0 +1,47 @@
+# Broker Token
+
+Each Broker cluster uses a Vault Token called the "Broker Token" to authenticate all Vault API calls. Only intentions that pass validation can make use of the token and only for the service declared in the action.
+
+The primary Broker node (determined by the hostname) will renew this token automatically.
+
+There are three scenarios where you may need to create a NR Broker token:
+
+* Initial start of a NR Broker cluster
+* If NR Broker is down for an extended period then the Broker Token will expire
+* If you are rotating the token
+
+## Security
+
+The Broker Token should never be exposed outside of the team responsible for running your Broker and Vault instances.
+
+The token should be renewable and have a TTL (time-to-live) high enough to survive reasonable network or infrastructure outages. A suggested TTL is 24 hours.
+
+Only a user with elevated premissions within Vault should be able to create the Broker Token. The token should be rotated periodically.
+
+## Policy
+
+The Broker Token should only have a minimal set of policies attached to it. The following is a sample policy that allows brokering services on the endpoint 'vs_apps_approle'.
+
+```
+# Authentication policy for global broker
+# Scope: Broker Approle
+
+path "auth/vs_apps_approle/role/+/role-id" {
+  capabilities = ["read"]
+}
+
+path "auth/vs_apps_approle/role/+/secret-id" {
+  capabilities = ["update"]
+}
+
+# Deny attempts to generate new Broker tokens
+path "auth/vs_apps_approle/role/vault_nr-broker_*" {
+  capabilities = ["deny"]
+}
+```
+
+See also: [Environment Variables](/dev_env_vars.md)
+
+## How to Generate
+
+Broker Token generation is deployment specific.

docs/dev_env_vars.md

@@ -0,0 +1,129 @@
+# Backend Environment Variables
+
+The Broker backend is configured using environment variables.
+
+A suggested deployment strategy is to use [envconsul](https://github.com/hashicorp/envconsul) to populate the secrets. To avoid overloading the [Broker Token](/dev_broker_token.md) with access to paths to retrieve secrets related to OIDC, JWT, Kinesis, MongoDB and more, a recommended pattern is to start envconsul using a token with access to fetch the Broker Token and all other secrets from Vault.
+
+## Deployment Info
+
+| Env Var | Default | Secret | Description |
+| --- | --- | --- | --- |
+| HOSTNAME |  |  | The hostname of the server this instance is running on. Used in logs. The instance with a hostname ending in '-0' is the primary node. It will cause issues if there is no primary node or there are multiple primary nodes'. |
+
+## Audit file logging
+
+The audit is written to disk and rotated automatically as a backup to sending it to a Kinesis endpoint.
+
+| Env Var | Default | Secret | Description |
+| --- | --- | --- | --- |
+| AUDIT_LOGSTREAM_DIR | '/tmp' |  | Directory to write logs to |
+| AUDIT_LOGSTREAM_SIZE | '50M' |  | Maximum size of each log files |
+| AUDIT_LOGSTREAM_MAX_LOGS | 7 |  | Maximum number of logs to keep |
+
+## Hashicorp Vault Setup
+
+Hashicorp Vault values used to integrate with the external Vault instance.
+
+See: [Broker Token](/dev_broker_token.md)
+
+| Env Var | Default | Secret | Description |
+| --- | --- | --- | --- |
+| BROKER_TOKEN |  | Yes | The vault token this instance uses to authenticate all Vault API calls with. Broker will renew this token automatically. |
+| VAULT_ADDR |  |  | The Vault address |
+| VAULT_APPROLE_PATH | vs_apps_approle |  | The approle endpoint containing all approles this instance is a broker for. |
+
+## JWT Setup
+
+| Env Var | Default | Secret | Description |
+| --- | --- | --- | --- |
+| JWT_SKIP_VALIDATION | false |  | Skips validating if token is in allowed or blocked list. If true, allows any valid token generated using secret to work. Token generation API adds all tokens to an allow list which blocks any token not generated through it. |
+| JWT_SECRET |  | Yes | The JWT secret used to create and validate all tokens |
+
+## OAUTH Client Setup
+
+OAUTH Client setup used to integrate as an OAUTH client.
+
+| Env Var | Default | Secret | Description |
+| --- | --- | --- | --- |
+| OAUTH2_CLIENT_SESSION_SECRET |  | Yes |  |
+| OAUTH2_CLIENT_PROVIDER_OIDC_ISSUER |  |  |  |
+| OAUTH2_CLIENT_REGISTRATION_LOGIN_CLIENT_ID |  |  |  |
+| OAUTH2_CLIENT_REGISTRATION_LOGIN_CLIENT_SECRET |  | Yes |  |
+| OAUTH2_CLIENT_REGISTRATION_LOGIN_REDIRECT_URI |  |  |  |
+| OAUTH2_CLIENT_REGISTRATION_LOGIN_SCOPE |  |  |  |
+
+## OAUTH Token Mapping
+
+Broker expects to read various values from the user's OAUTH Token. These let you configure where the values are read from. If a value is blank in the Broker database then it may be reading it from the wrong location.
+
+| Env Var | Default | Secret | Description |
+| --- | --- | --- | --- |
+| OAUTH2_CLIENT_MAP_DOMAIN | '' |  | The auth domain |
+| OAUTH2_CLIENT_DOMAIN | 'idp' |  | Backup value for domain. Used if mapped value is not present. |
+| OAUTH2_CLIENT_MAP_EMAIL | 'email' |  | The user's email address |
+| OAUTH2_CLIENT_MAP_GUID | 'idir_user_guid' |  | The user's GUID |
+| OAUTH2_CLIENT_MAP_NAME | 'display_name' |  | The user's display name |
+| OAUTH2_CLIENT_MAP_ROLES | 'client_roles' |  | The client roles the user has. |
+| OAUTH2_CLIENT_MAP_USERNAME | 'idir_username' |  | The user's username |
+
+## AWS Kinesis
+
+AWS configuration used to push the audit log to a Kinesis end point. Consuming the data from the Kinesis endpoint is deployment specific.
+
+| Env Var | Default | Secret | Description |
+| --- | --- | --- | --- |
+| AWS_ACCESS_KEY_ID |  | Yes |  |
+| AWS_SECRET_ACCESS_KEY |  | Yes |  |
+| AWS_SESSION_TOKEN |  | Yes |  |
+| AWS_KINESIS_ROLE_ARN |  | Yes |  |
+| AWS_DEFAULT_REGION | ca-central-1 |  |  |
+
+## Log redirection 
+
+The logs are normally split into different datasets by the field `event.dataset`. See: [Understanding the Audit Log](/audit.md)
+
+If you are testing, the `@metadata.index` field can be used by what processes the logs to override the destination index. Log processing is deployment specific.
+
+| Env Var | Default | Secret | Description |
+| --- | --- | --- | --- |
+| BROKER_AUDIT_INDEX_ACTIVITY |  |  | Set field `@metadata.index` to this value in all `broker.audit` dataset documents. |
+| BROKER_AUDIT_INDEX_HTTP_ACCESS |  |  | Set field `@metadata.index` to this value in all `generic.access` dataset documents. |
+
+## Helmet Configuration
+
+https://helmetjs.github.io
+
+Broker, by default, uses Helmet's default content security policy directives.
+
+| Env Var | Default | Secret | Description |
+| --- | --- | --- | --- |
+| NESTJS_HELMET_HSTS |  |  | If HSTS (HTTP Strict Transport Security) is set to 'off', insecure requests are not upgraded. This option is primarily for local development. This deletes 'upgrade-insecure-requests' from the policy directives. |
+
+## MongoDB Connection
+
+The MongoDB environment variables used to setup the connection.
+
+
+| Env Var | Default | Secret | Description |
+| --- | --- | --- | --- |
+| MONGODB_URL |  |  | The MongoDB URL that can be paramaterized as described in username and password environment variables. |
+| MONGODB_USERNAME |  | Yes | The username to substitute for the text '{{username}}' in MONGODB_URL. |
+| MONGODB_PASSWORD |  | Yes | The password to substitute for the text '{{password}}' in MONGODB_URL. |
+
+## Redis
+
+The Redis environment variables used to setup the connection.
+
+| Env Var | Default | Secret | Description |
+| --- | --- | --- | --- |
+| REDIS_HOST | 'localhost' |  | The Redis server host |
+| REDIS_PORT | 6379 |  | The Redis server port |
+| REDIS_USER | '' | Yes | The Redis user |
+| REDIS_PASSWORD | '' | Yes | The Redis password |
+
+## Temporary
+
+| Env Var | Default | Secret | Description |
+| --- | --- | --- | --- |
+| ACTION_VALIDATE_TEAM_ADMIN |  |  | Users that are members of this team can skip all intention validations.  |
+| ACTION_VALIDATE_TEAM_DBA |  |  | Users that are members of this team can skip database intention validations. |

scripts/setenv-common.sh.tmp

@@ -33,7 +33,6 @@ fi
 )
 
 export BROKER_AUDIT_INDEX_ACTIVITY=audit-test-broker-activity
-export BROKER_AUDIT_INDEX_AUTH=audit-test-broker-auth
 export BROKER_AUDIT_INDEX_HTTP_ACCESS=audit-test-broker-access-external
 
 export JWT_DEFAULT_SUB="something@example.com"

src/audit/audit.service.ts

@@ -33,8 +33,7 @@ const hostInfo = {
  */
 @Injectable()
 export class AuditService {
-  private readonly metadataIntentionActivity = {};
-  private readonly metadataAuth = {};
+  private readonly metadataActivity = {};
   private readonly metadataHttpAccess = {};
 
   /**
@@ -43,15 +42,10 @@ export class AuditService {
    */
   constructor(private readonly stream: AuditStreamerService) {
     if (process.env.BROKER_AUDIT_INDEX_ACTIVITY) {
-      this.metadataIntentionActivity['@metadata'] = {
+      this.metadataActivity['@metadata'] = {
         index: process.env.BROKER_AUDIT_INDEX_ACTIVITY,
       };
     }
-    if (process.env.BROKER_AUDIT_INDEX_AUTH) {
-      this.metadataAuth['@metadata'] = {
-        index: process.env.BROKER_AUDIT_INDEX_AUTH,
-      };
-    }
     if (process.env.BROKER_AUDIT_INDEX_HTTP_ACCESS) {
       this.metadataHttpAccess['@metadata'] = {
         index: process.env.BROKER_AUDIT_INDEX_HTTP_ACCESS,
@@ -97,7 +91,7 @@ export class AuditService {
         map(this.addErrorFunc(exception)),
         map(this.addHostFunc),
         map(this.addLabelsFunc),
-        map(this.addMetadataIntentionActivityFunc()),
+        map(this.addMetadataActivityFunc()),
         map(this.addServiceFunc),
         map(this.addSourceFunc(req)),
         map(this.addTimestampFunc(now)),
@@ -154,7 +148,7 @@ export class AuditService {
           ),
           map(this.addHostFunc),
           map(this.addLabelsFunc),
-          map(this.addMetadataIntentionActivityFunc()),
+          map(this.addMetadataActivityFunc()),
           map(this.addServiceFunc),
           map(this.addSourceFunc(req)),
           map(this.addTimestampFunc(now)),
@@ -204,7 +198,7 @@ export class AuditService {
         map(this.addEcsFunc),
         map(this.addHostFunc),
         map(this.addLabelsFunc),
-        map(this.addMetadataIntentionActivityFunc()),
+        map(this.addMetadataActivityFunc()),
         map(this.addServiceFunc),
         map(this.addSourceFunc(req)),
         map(this.addTimestampFunc(now)),
@@ -250,7 +244,7 @@ export class AuditService {
         map(this.addEcsFunc),
         map(this.addHostFunc),
         map(this.addLabelsFunc),
-        map(this.addMetadataIntentionActivityFunc()),
+        map(this.addMetadataActivityFunc()),
         map(this.addServiceFunc),
         map(this.addSourceFunc(req)),
         map(this.addTimestampFunc(now)),
@@ -293,7 +287,7 @@ export class AuditService {
         map(this.addHostFunc),
         map(this.addLabelsFunc),
         map(this.addAssignFunc(assignObj)),
-        map(this.addMetadataIntentionActivityFunc()),
+        map(this.addMetadataActivityFunc()),
         map(this.addServiceFunc),
         map(this.addSourceFunc(req)),
         map(this.addTimestampFunc(now)),
@@ -331,7 +325,7 @@ export class AuditService {
         map(this.addEcsFunc),
         map(this.addHostFunc),
         map(this.addLabelsFunc),
-        map(this.addMetadataAuthFunc()),
+        map(this.addMetadataActivityFunc()),
         map(this.addServiceFunc),
         map(this.addSourceFunc(req)),
         map(this.addTimestampFunc()),
@@ -368,7 +362,7 @@ export class AuditService {
         map(this.addGraphObjectFunc(set, graphObj)),
         map(this.addHostFunc),
         map(this.addLabelsFunc),
-        map(this.addMetadataAuthFunc()),
+        map(this.addMetadataActivityFunc()),
         map(this.addRequestUserFunc(req)),
         map(this.addServiceFunc),
         map(this.addSourceFunc(req)),
@@ -409,7 +403,7 @@ export class AuditService {
         map(this.addEcsFunc),
         map(this.addHostFunc),
         map(this.addLabelsFunc),
-        map(this.addMetadataAuthFunc()),
+        map(this.addMetadataActivityFunc()),
         map(this.addServiceFunc),
         map(this.addSourceFunc(req)),
         map(this.addTimestampFunc()),
@@ -743,17 +737,8 @@ export class AuditService {
    * This allows dev environments to send data to a temporary index.
    * @returns Function to generate partial ECS document
    */
-  private addMetadataIntentionActivityFunc() {
-    return (ecsObj: any) => merge(ecsObj, this.metadataIntentionActivity);
-  }
-
-  /**
-   * Map function generator for adding authentication activity metadata to ECS document.
-   * This allows dev environments to send data to a temporary index.
-   * @returns Function to generate partial ECS document
-   */
-  private addMetadataAuthFunc() {
-    return (ecsObj: any) => merge(ecsObj, this.metadataAuth);
+  private addMetadataActivityFunc() {
+    return (ecsObj: any) => merge(ecsObj, this.metadataActivity);
   }
 
   /**