From fecff75f7b9408a7c04e592d8cf511c8f5d9cfb6 Mon Sep 17 00:00:00 2001 From: Joshua Karp Date: Tue, 15 Feb 2022 13:20:47 +1100 Subject: [PATCH] NAT Traversal testing utils and test WIPs --- scripts/test-pipelines.sh | 4 +- shell.nix | 1 + tests/nat/endpointDependentNAT.test.ts | 147 ++++ tests/nat/endpointIndependentNAT.test.ts | 262 ++++++ tests/nat/noNAT.test.ts | 239 ++++++ tests/nat/utils.ts | 983 +++++++++++++++++++++++ 6 files changed, 1634 insertions(+), 2 deletions(-) create mode 100644 tests/nat/endpointDependentNAT.test.ts create mode 100644 tests/nat/endpointIndependentNAT.test.ts create mode 100644 tests/nat/noNAT.test.ts create mode 100644 tests/nat/utils.ts diff --git a/scripts/test-pipelines.sh b/scripts/test-pipelines.sh index 323850fdd..8ec7919ca 100755 --- a/scripts/test-pipelines.sh +++ b/scripts/test-pipelines.sh @@ -59,7 +59,7 @@ test $test_dir: interruptible: true script: - > - nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' + nix-shell -I nixpkgs=./pkgs.nix --packages nodejs iproute2 utillinux nftables iptables --run ' npm ci; npm test -- ${test_files[@]}; ' @@ -76,7 +76,7 @@ test index: interruptible: true script: - > - nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' + nix-shell -I nixpkgs=./pkgs.nix --packages nodejs iptables-legacy --run ' npm ci; npm test -- ${test_files[@]}; ' diff --git a/shell.nix b/shell.nix index 318d807aa..2b77427f4 100644 --- a/shell.nix +++ b/shell.nix @@ -11,6 +11,7 @@ in grpc-tools grpcurl utils.pkg + utillinux ]; PKG_CACHE_PATH = utils.pkgCachePath; PKG_IGNORE_TAG = 1; diff --git a/tests/nat/endpointDependentNAT.test.ts b/tests/nat/endpointDependentNAT.test.ts new file mode 100644 index 000000000..1f5a6e22c --- /dev/null +++ b/tests/nat/endpointDependentNAT.test.ts @@ -0,0 +1,147 @@ +import os from 'os'; +import path from 'path'; +import fs from 'fs'; +import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; +import * as testNatUtils from './utils'; + +describe('endpoint dependent NAT traversal', () => { + const logger = new Logger('EDM NAT test', LogLevel.WARN, [ + new StreamHandler(), + ]); + let dataDir: string; + beforeEach(async () => { + dataDir = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'polykey-test-'), + ); + }); + afterEach(async () => { + await fs.promises.rm(dataDir, { + force: true, + recursive: true, + }); + }); + test( + 'Node1 behind EDM NAT connects to Node2', + async () => { + const { + userPid, + agent1Pid, + password, + dataDir, + agent1NodePath, + agent2NodeId, + tearDownNAT, + } = await testNatUtils.setupNAT('edm', 'dmz', logger); + const { exitCode, stdout } = await testNatUtils.pkExecNs( + userPid, + agent1Pid, + ['nodes', 'ping', agent2NodeId, '--format', 'json'], + { + PK_NODE_PATH: agent1NodePath, + PK_PASSWORD: password, + }, + dataDir, + ); + expect(exitCode).toBe(0); + expect(JSON.parse(stdout)).toEqual({ + success: true, + message: 'Node is Active.', + }); + await tearDownNAT(); + }, + global.defaultTimeout * 2, + ); + test( + 'Node1 connects to Node2 behind EDM NAT', + async () => { + const { + userPid, + agent1Pid, + password, + dataDir, + agent1NodePath, + agent2NodeId, + tearDownNAT, + } = await testNatUtils.setupNAT('dmz', 'edm', logger); + const { exitCode, stdout } = await testNatUtils.pkExecNs( + userPid, + agent1Pid, + ['nodes', 'ping', agent2NodeId, '--format', 'json'], + { + PK_NODE_PATH: agent1NodePath, + PK_PASSWORD: password, + }, + dataDir, + ); + expect(exitCode).toBe(0); + expect(JSON.parse(stdout)).toEqual({ + success: true, + message: 'Node is Active.', + }); + await tearDownNAT(); + }, + global.defaultTimeout * 2, + ); + test( + 'Node1 behind EDM NAT cannot connect to Node2 behind EDM NAT', + async () => { + const { + userPid, + agent1Pid, + password, + dataDir, + agent1NodePath, + agent2NodeId, + tearDownNAT, + } = await testNatUtils.setupNAT('edm', 'edm', logger); + const { exitCode, stdout } = await testNatUtils.pkExecNs( + userPid, + agent1Pid, + ['nodes', 'ping', agent2NodeId, '--format', 'json'], + { + PK_NODE_PATH: agent1NodePath, + PK_PASSWORD: password, + }, + dataDir, + ); + expect(exitCode).not.toBe(0); + expect(JSON.parse(stdout)).toEqual({ + success: false, + message: `No response received`, + }); + await tearDownNAT(); + }, + global.defaultTimeout * 2, + ); + test( + 'Node1 behind EDM NAT cannot connect to Node2 behind EIM NAT', + async () => { + const { + userPid, + agent1Pid, + password, + dataDir, + agent1NodePath, + agent2NodeId, + tearDownNAT, + } = await testNatUtils.setupNAT('edm', 'eim', logger); + const { exitCode, stdout } = await testNatUtils.pkExecNs( + userPid, + agent1Pid, + ['nodes', 'ping', agent2NodeId, '--format', 'json'], + { + PK_NODE_PATH: agent1NodePath, + PK_PASSWORD: password, + }, + dataDir, + ); + expect(exitCode).not.toBe(0); + expect(JSON.parse(stdout)).toEqual({ + success: false, + message: `No response received`, + }); + await tearDownNAT(); + }, + global.defaultTimeout * 2, + ); +}); diff --git a/tests/nat/endpointIndependentNAT.test.ts b/tests/nat/endpointIndependentNAT.test.ts new file mode 100644 index 000000000..99df25159 --- /dev/null +++ b/tests/nat/endpointIndependentNAT.test.ts @@ -0,0 +1,262 @@ +import os from 'os'; +import path from 'path'; +import fs from 'fs'; +import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; +import * as testNatUtils from './utils'; + +describe('endpoint independent NAT traversal', () => { + const logger = new Logger('EIM NAT test', LogLevel.WARN, [ + new StreamHandler(), + ]); + let dataDir: string; + beforeEach(async () => { + dataDir = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'polykey-test-'), + ); + }); + afterEach(async () => { + await fs.promises.rm(dataDir, { + force: true, + recursive: true, + }); + }); + test( + 'Node1 behind EIM NAT connects to Node2', + async () => { + const { + userPid, + agent1Pid, + password, + dataDir, + agent1NodePath, + agent2NodeId, + agent2Host, + agent2ProxyPort, + tearDownNAT, + } = await testNatUtils.setupNAT('eim', 'dmz', logger); + // Since node2 is not behind a NAT can directly add its details + await testNatUtils.pkExecNs( + userPid, + agent1Pid, + ['nodes', 'add', agent2NodeId, agent2Host, agent2ProxyPort], + { + PK_NODE_PATH: agent1NodePath, + PK_PASSWORD: password, + }, + dataDir, + ); + const { exitCode, stdout } = await testNatUtils.pkExecNs( + userPid, + agent1Pid, + ['nodes', 'ping', agent2NodeId, '--format', 'json'], + { + PK_NODE_PATH: agent1NodePath, + PK_PASSWORD: password, + }, + dataDir, + ); + expect(exitCode).toBe(0); + expect(JSON.parse(stdout)).toEqual({ + success: true, + message: 'Node is Active.', + }); + await tearDownNAT(); + }, + global.defaultTimeout * 2, + ); + test( + 'Node1 connects to Node2 behind EIM NAT', + async () => { + const { + userPid, + agent1Pid, + agent2Pid, + password, + dataDir, + agent1NodePath, + agent2NodePath, + agent1NodeId, + agent1Host, + agent1ProxyPort, + agent2NodeId, + agent2Host, + agent2ProxyPort, + tearDownNAT, + } = await testNatUtils.setupNAT('dmz', 'eim', logger); + // Since node2 is behind a NAT we need it to contact us first + await testNatUtils.pkExecNs( + userPid, + agent2Pid, + ['nodes', 'add', agent1NodeId, agent1Host, agent1ProxyPort], + { + PK_NODE_PATH: agent2NodePath, + PK_PASSWORD: password, + }, + dataDir, + ); + // This add call can be removed once nodes add connection info of + // connecting nodes + await testNatUtils.pkExecNs( + userPid, + agent1Pid, + ['nodes', 'add', agent2NodeId, agent2Host, agent2ProxyPort], + { + PK_NODE_PATH: agent1NodePath, + PK_PASSWORD: password, + }, + dataDir, + ); + await testNatUtils.pkExecNs( + userPid, + agent2Pid, + ['nodes', 'ping', agent1NodeId, '--format', 'json'], + { + PK_NODE_PATH: agent2NodePath, + PK_PASSWORD: password, + }, + dataDir, + ); + // We should now be able to ping back + const { exitCode, stdout } = await testNatUtils.pkExecNs( + userPid, + agent1Pid, + ['nodes', 'ping', agent2NodeId, '--format', 'json'], + { + PK_NODE_PATH: agent1NodePath, + PK_PASSWORD: password, + }, + dataDir, + ); + expect(exitCode).toBe(0); + expect(JSON.parse(stdout)).toEqual({ + success: true, + message: 'Node is Active.', + }); + await tearDownNAT(); + }, + global.defaultTimeout * 2, + ); + test( + 'Node1 behind EIM NAT connects to Node2 behind EIM NAT', + async () => { + const { + userPid, + agent1Pid, + agent2Pid, + password, + dataDir, + agent1NodePath, + agent2NodePath, + agent1NodeId, + agent2NodeId, + tearDownNAT, + } = await testNatUtils.setupNATWithSeedNode('eim', 'eim', logger); + // Since node2 is behind a NAT we need it to attempt to contact us first + // This won't be successfull, but will allow to get past its router with + // our own ping + await testNatUtils.pkExecNs( + userPid, + agent2Pid, + ['nodes', 'ping', agent1NodeId, '--format', 'json', '-vv'], + { + PK_NODE_PATH: agent2NodePath, + PK_PASSWORD: password, + }, + dataDir, + ); + const { exitCode, stdout } = await testNatUtils.pkExecNs( + userPid, + agent1Pid, + ['nodes', 'ping', agent2NodeId, '--format', 'json', '-vv'], + { + PK_NODE_PATH: agent1NodePath, + PK_PASSWORD: password, + }, + dataDir, + ); + expect(exitCode).toBe(0); + expect(JSON.parse(stdout)).toEqual({ + success: true, + message: 'Node is Active.', + }); + await tearDownNAT(); + }, + global.defaultTimeout * 20, + ); + test.skip( + 'Node1 behind EIM NAT cannot connect to Node2 behind EDM NAT', + async () => { + const { + userPid, + agent1Pid, + agent2Pid, + password, + dataDir, + agent1NodePath, + agent2NodePath, + agent1NodeId, + agent1Host, + agent1ProxyPort, + agent2NodeId, + agent2Host, + agent2ProxyPort, + tearDownNAT, + } = await testNatUtils.setupNAT('eim', 'edm', logger); + // Since one of the nodes uses EDM NAT we cannot punch through + await testNatUtils.pkExecNs( + userPid, + agent1Pid, + ['nodes', 'add', agent2NodeId, agent2Host, agent2ProxyPort], + { + PK_NODE_PATH: agent1NodePath, + PK_PASSWORD: password, + }, + dataDir, + ); + await testNatUtils.pkExecNs( + userPid, + agent2Pid, + ['nodes', 'add', agent1NodeId, agent1Host, agent1ProxyPort], + { + PK_NODE_PATH: agent2NodePath, + PK_PASSWORD: password, + }, + dataDir, + ); + const [ping12, ping21] = await Promise.all([ + testNatUtils.pkExecNs( + userPid, + agent1Pid, + ['nodes', 'ping', agent2NodeId, '--format', 'json'], + { + PK_NODE_PATH: agent1NodePath, + PK_PASSWORD: password, + }, + dataDir, + ), + testNatUtils.pkExecNs( + userPid, + agent2Pid, + ['nodes', 'ping', agent1NodeId, '--format', 'json'], + { + PK_NODE_PATH: agent2NodePath, + PK_PASSWORD: password, + }, + dataDir, + ), + ]); + expect(ping12.exitCode).toBe(1); + expect(JSON.parse(ping12.stdout)).toEqual({ + success: false, + message: 'No response received', + }); + expect(ping21.exitCode).toBe(1); + expect(JSON.parse(ping21.stdout)).toEqual({ + success: false, + message: 'No response received', + }); + await tearDownNAT(); + }, + global.defaultTimeout * 2, + ); +}); diff --git a/tests/nat/noNAT.test.ts b/tests/nat/noNAT.test.ts new file mode 100644 index 000000000..14c63bece --- /dev/null +++ b/tests/nat/noNAT.test.ts @@ -0,0 +1,239 @@ +import os from 'os'; +import path from 'path'; +import fs from 'fs'; +import readline from 'readline'; +import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; +import Status from '@/status/Status'; +import config from '@/config'; +import * as testNatUtils from './utils'; +import * as testBinUtils from '../bin/utils'; + +describe('no NAT', () => { + const logger = new Logger('no NAT test', LogLevel.WARN, [ + new StreamHandler(), + ]); + let dataDir: string; + beforeEach(async () => { + dataDir = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'polykey-test-'), + ); + }); + afterEach(async () => { + await fs.promises.rm(dataDir, { + force: true, + recursive: true, + }); + }); + test( + 'can create an agent in a namespace', + async () => { + const password = 'abc123'; + const usrns = testNatUtils.createUserNamespace(); + const netns = testNatUtils.createNetworkNamespace(usrns.pid); + const agentProcess = await testNatUtils.pkSpawnNs( + usrns.pid, + netns.pid, + [ + 'agent', + 'start', + '--node-path', + path.join(dataDir, 'polykey'), + '--root-key-pair-bits', + '1024', + '--client-host', + '127.0.0.1', + '--proxy-host', + '127.0.0.1', + '--workers', + '0', + '--verbose', + '--format', + 'json', + ], + { + PK_PASSWORD: password, + }, + dataDir, + logger.getChild('agentProcess'), + ); + const rlOut = readline.createInterface(agentProcess.stdout!); + const stdout = await new Promise((resolve, reject) => { + rlOut.once('line', resolve); + rlOut.once('close', reject); + }); + const statusLiveData = JSON.parse(stdout); + expect(statusLiveData).toMatchObject({ + pid: agentProcess.pid, + nodeId: expect.any(String), + clientHost: expect.any(String), + clientPort: expect.any(Number), + agentHost: expect.any(String), + agentPort: expect.any(Number), + forwardHost: expect.any(String), + forwardPort: expect.any(Number), + proxyHost: expect.any(String), + proxyPort: expect.any(Number), + recoveryCode: expect.any(String), + }); + expect( + statusLiveData.recoveryCode.split(' ').length === 12 || + statusLiveData.recoveryCode.split(' ').length === 24, + ).toBe(true); + agentProcess.kill('SIGTERM'); + let exitCode, signal; + [exitCode, signal] = await testBinUtils.processExit(agentProcess); + expect(exitCode).toBe(null); + expect(signal).toBe('SIGTERM'); + // Check for graceful exit + const status = new Status({ + statusPath: path.join(dataDir, 'polykey', config.defaults.statusBase), + statusLockPath: path.join( + dataDir, + 'polykey', + config.defaults.statusLockBase, + ), + fs, + logger, + }); + const statusInfo = (await status.readStatus())!; + expect(statusInfo.status).toBe('DEAD'); + netns.kill('SIGTERM'); + [exitCode, signal] = await testBinUtils.processExit(netns); + expect(exitCode).toBe(null); + expect(signal).toBe('SIGTERM'); + usrns.kill('SIGTERM'); + [exitCode, signal] = await testBinUtils.processExit(usrns); + expect(exitCode).toBe(null); + expect(signal).toBe('SIGTERM'); + }, + global.defaultTimeout * 2, + ); + test( + 'agents in different namespaces can ping each other', + async () => { + const { + userPid, + agent1Pid, + agent2Pid, + password, + dataDir, + agent1NodePath, + agent2NodePath, + agent1NodeId, + agent1Host, + agent1ProxyPort, + agent2NodeId, + agent2Host, + agent2ProxyPort, + tearDownNAT, + } = await testNatUtils.setupNAT('dmz', 'dmz', logger); + // Since neither node is behind a NAT can directly add eachother's + // details using pk nodes add + await testNatUtils.pkExecNs( + userPid, + agent1Pid, + ['nodes', 'add', agent2NodeId, agent2Host, agent2ProxyPort], + { + PK_NODE_PATH: agent1NodePath, + PK_PASSWORD: password, + }, + dataDir, + ); + await testNatUtils.pkExecNs( + userPid, + agent2Pid, + ['nodes', 'add', agent1NodeId, agent1Host, agent1ProxyPort], + { + PK_NODE_PATH: agent2NodePath, + PK_PASSWORD: password, + }, + dataDir, + ); + let exitCode, stdout; + ({ exitCode, stdout } = await testNatUtils.pkExecNs( + userPid, + agent1Pid, + ['nodes', 'ping', agent2NodeId, '--format', 'json', '--verbose'], + { + PK_NODE_PATH: agent1NodePath, + PK_PASSWORD: password, + }, + dataDir, + )); + expect(exitCode).toBe(0); + expect(JSON.parse(stdout)).toEqual({ + success: true, + message: 'Node is Active.', + }); + ({ exitCode, stdout } = await testNatUtils.pkExecNs( + userPid, + agent2Pid, + ['nodes', 'ping', agent1NodeId, '--format', 'json', '--verbose'], + { + PK_NODE_PATH: agent2NodePath, + PK_PASSWORD: password, + }, + dataDir, + )); + expect(exitCode).toBe(0); + expect(JSON.parse(stdout)).toEqual({ + success: true, + message: 'Node is Active.', + }); + await tearDownNAT(); + }, + global.defaultTimeout * 2, + ); + test( + 'agents in different namespaces can ping each other via seed node', + async () => { + const { + userPid, + agent1Pid, + agent2Pid, + password, + dataDir, + agent1NodePath, + agent2NodePath, + agent1NodeId, + agent2NodeId, + tearDownNAT, + } = await testNatUtils.setupNATWithSeedNode('dmz', 'dmz', logger); + // Should be able to ping straight away using the details from the + // seed node + let exitCode, stdout; + ({ exitCode, stdout } = await testNatUtils.pkExecNs( + userPid, + agent1Pid, + ['nodes', 'ping', agent2NodeId, '--format', 'json', '--verbose'], + { + PK_NODE_PATH: agent1NodePath, + PK_PASSWORD: password, + }, + dataDir, + )); + expect(exitCode).toBe(0); + expect(JSON.parse(stdout)).toEqual({ + success: true, + message: 'Node is Active.', + }); + ({ exitCode, stdout } = await testNatUtils.pkExecNs( + userPid, + agent2Pid, + ['nodes', 'ping', agent1NodeId, '--format', 'json', '--verbose'], + { + PK_NODE_PATH: agent2NodePath, + PK_PASSWORD: password, + }, + dataDir, + )); + expect(exitCode).toBe(0); + expect(JSON.parse(stdout)).toEqual({ + success: true, + message: 'Node is Active.', + }); + await tearDownNAT(); + }, + global.defaultTimeout * 2, + ); +}); diff --git a/tests/nat/utils.ts b/tests/nat/utils.ts new file mode 100644 index 000000000..d59446b3e --- /dev/null +++ b/tests/nat/utils.ts @@ -0,0 +1,983 @@ +import type { ChildProcess } from 'child_process'; +import os from 'os'; +import fs from 'fs'; +import path from 'path'; +import process from 'process'; +import child_process from 'child_process'; +import readline from 'readline'; +import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; +import * as testBinUtils from '../bin/utils'; + +type NATType = 'eim' | 'edm' | 'dmz'; + +// Constants for all util functions +// Veth pairs (ends) +const agent1Host = 'agent1'; +const agent2Host = 'agent2'; +const agent1RouterHostInt = 'router1-int'; +const agent1RouterHostExt = 'router1-ext'; +const agent2RouterHostInt = 'router2-int'; +const agent2RouterHostExt = 'router2-ext'; +const router1SeedHost = 'router1-seed'; +const router2SeedHost = 'router2-seed'; +const seedRouter1Host = 'seed-router1'; +const seedRouter2Host = 'seed-router2'; +// Subnets +const agent1HostIp = '10.0.0.2'; +const agent2HostIp = '10.0.0.2'; +const agent1RouterHostIntIp = '10.0.0.1'; +const agent2RouterHostIntIp = '10.0.0.1'; +const agent1RouterHostExtIp = '192.168.0.1'; +const agent2RouterHostExtIp = '192.168.0.2'; +const router1SeedHostIp = '192.168.0.1'; +const seedHostIp = '192.168.0.3'; +const router2SeedHostIp = '192.168.0.2'; +// Subnet mask +const subnetMask = '/24'; +// Ports +const mappedPort = '55555'; + +/** + * Formats the command to enter a namespace to run a process inside it + */ +const nsenter = (usrnsPid: number, netnsPid: number) => { + return `nsenter --target ${usrnsPid} --user --preserve-credentials `.concat( + `nsenter --target ${netnsPid} --net `, + ); +}; + +/** + * Create a user namespace from which network namespaces can be created without + * requiring sudo + */ +function createUserNamespace(): ChildProcess { + return child_process.spawn('unshare', ['--user', '--map-root-user'], { + shell: true, + }); +} + +/** + * Create a network namespace inside a user namespace + */ +function createNetworkNamespace(usrnsPid: number): ChildProcess { + return child_process.spawn( + 'nsenter', + [ + '--target', + usrnsPid.toString(), + '--user', + '--preserve-credentials', + 'unshare', + '--net', + ], + { shell: true }, + ); +} + +/** + * Set up four network namespaces to allow communication between two agents + * each behind a router + * Brings up loopback interfaces, creates and brings up a veth pair + * between each pair of adjacent namespaces, and adds default routing to allow + * cross-communication + */ +function setupNetworkNamespaceInterfaces( + usrnsPid: number, + agent1NetnsPid: number, + router1NetnsPid: number, + router2NetnsPid: number, + agent2NetnsPid: number, +) { + // Bring up loopback + child_process.exec(nsenter(usrnsPid, agent1NetnsPid) + `ip link set lo up`); + child_process.exec(nsenter(usrnsPid, router1NetnsPid) + `ip link set lo up`); + child_process.exec(nsenter(usrnsPid, router2NetnsPid) + `ip link set lo up`); + child_process.exec(nsenter(usrnsPid, agent2NetnsPid) + `ip link set lo up`); + // Create veth pair to link the namespaces + child_process.exec( + nsenter(usrnsPid, agent1NetnsPid) + + `ip link add ${agent1Host} type veth peer name ${agent1RouterHostInt}`, + ); + child_process.exec( + nsenter(usrnsPid, router1NetnsPid) + + `ip link add ${agent1RouterHostExt} type veth peer name ${agent2RouterHostExt}`, + ); + child_process.exec( + nsenter(usrnsPid, router2NetnsPid) + + `ip link add ${agent2RouterHostInt} type veth peer name ${agent2Host}`, + ); + // Link up the ends to the correct namespaces + child_process.exec( + nsenter(usrnsPid, agent1NetnsPid) + + `ip link set dev ${agent1RouterHostInt} netns ${router1NetnsPid}`, + ); + child_process.exec( + nsenter(usrnsPid, router1NetnsPid) + + `ip link set dev ${agent2RouterHostExt} netns ${router2NetnsPid}`, + ); + child_process.exec( + nsenter(usrnsPid, router2NetnsPid) + + `ip link set dev ${agent2Host} netns ${agent2NetnsPid}`, + ); + // Bring up each end + child_process.exec( + nsenter(usrnsPid, agent1NetnsPid) + `ip link set ${agent1Host} up`, + ); + child_process.exec( + nsenter(usrnsPid, router1NetnsPid) + + `ip link set ${agent1RouterHostInt} up`, + ); + child_process.exec( + nsenter(usrnsPid, router1NetnsPid) + + `ip link set ${agent1RouterHostExt} up`, + ); + child_process.exec( + nsenter(usrnsPid, router2NetnsPid) + + `ip link set ${agent2RouterHostExt} up`, + ); + child_process.exec( + nsenter(usrnsPid, router2NetnsPid) + + `ip link set ${agent2RouterHostInt} up`, + ); + child_process.exec( + nsenter(usrnsPid, agent2NetnsPid) + `ip link set ${agent2Host} up`, + ); + // Assign ip addresses to each end + child_process.exec( + nsenter(usrnsPid, agent1NetnsPid) + + `ip addr add ${agent1HostIp}${subnetMask} dev ${agent1Host}`, + ); + child_process.exec( + nsenter(usrnsPid, router1NetnsPid) + + `ip addr add ${agent1RouterHostIntIp}${subnetMask} dev ${agent1RouterHostInt}`, + ); + child_process.exec( + nsenter(usrnsPid, router1NetnsPid) + + `ip addr add ${agent1RouterHostExtIp}${subnetMask} dev ${agent1RouterHostExt}`, + ); + child_process.exec( + nsenter(usrnsPid, router2NetnsPid) + + `ip addr add ${agent2RouterHostExtIp}${subnetMask} dev ${agent2RouterHostExt}`, + ); + child_process.exec( + nsenter(usrnsPid, router2NetnsPid) + + `ip addr add ${agent2RouterHostIntIp}${subnetMask} dev ${agent2RouterHostInt}`, + ); + child_process.exec( + nsenter(usrnsPid, agent2NetnsPid) + + `ip addr add ${agent2HostIp}${subnetMask} dev ${agent2Host}`, + ); + // Add default routing + child_process.exec( + nsenter(usrnsPid, agent1NetnsPid) + + `ip route add default via ${agent1RouterHostIntIp}`, + ); + child_process.exec( + nsenter(usrnsPid, router1NetnsPid) + + `ip route add default via ${agent2RouterHostExtIp}`, + ); + child_process.exec( + nsenter(usrnsPid, router2NetnsPid) + + `ip route add default via ${agent1RouterHostExtIp}`, + ); + child_process.exec( + nsenter(usrnsPid, agent2NetnsPid) + + `ip route add default via ${agent2RouterHostIntIp}`, + ); +} + +/** + * Set up four network namespaces to allow communication between two agents + * each behind a router + * Brings up loopback interfaces, creates and brings up a veth pair + * between each pair of adjacent namespaces, and adds default routing to allow + * cross-communication + */ +function setupSeedNamespaceInterfaces( + usrnsPid: number, + seedNetnsPid: number, + router1NetnsPid: number, + router2NetnsPid: number, +) { + // Bring up loopback + child_process.exec(nsenter(usrnsPid, seedNetnsPid) + `ip link set lo up`); + // Create veth pairs to link the namespaces + child_process.exec( + nsenter(usrnsPid, router1NetnsPid) + + `ip link add ${router1SeedHost} type veth peer name ${seedRouter1Host}`, + ); + child_process.exec( + nsenter(usrnsPid, router2NetnsPid) + + `ip link add ${router2SeedHost} type veth peer name ${seedRouter2Host}`, + ); + // Move seed ends into seed network namespace + child_process.exec( + nsenter(usrnsPid, router1NetnsPid) + + `ip link set dev ${seedRouter1Host} netns ${seedNetnsPid}`, + ); + child_process.exec( + nsenter(usrnsPid, router2NetnsPid) + + `ip link set dev ${seedRouter2Host} netns ${seedNetnsPid}`, + ); + // Bring up each end + child_process.exec( + nsenter(usrnsPid, router1NetnsPid) + `ip link set ${router1SeedHost} up`, + ); + child_process.exec( + nsenter(usrnsPid, seedNetnsPid) + `ip link set ${seedRouter1Host} up`, + ); + child_process.exec( + nsenter(usrnsPid, seedNetnsPid) + `ip link set ${seedRouter2Host} up`, + ); + child_process.exec( + nsenter(usrnsPid, router2NetnsPid) + `ip link set ${router2SeedHost} up`, + ); + // Assign ip addresses to each end + child_process.exec( + nsenter(usrnsPid, router1NetnsPid) + + `ip addr add ${router1SeedHostIp}${subnetMask} dev ${router1SeedHost}`, + ); + child_process.exec( + nsenter(usrnsPid, seedNetnsPid) + + `ip addr add ${seedHostIp}${subnetMask} dev ${seedRouter1Host}`, + ); + child_process.exec( + nsenter(usrnsPid, seedNetnsPid) + + `ip addr add ${seedHostIp}${subnetMask} dev ${seedRouter2Host}`, + ); + child_process.exec( + nsenter(usrnsPid, router2NetnsPid) + + `ip addr add ${router2SeedHostIp}${subnetMask} dev ${router2SeedHost}`, + ); + child_process.exec( + nsenter(usrnsPid, router1NetnsPid) + + `ip route add ${seedHostIp} dev ${router1SeedHost}`, + ); + child_process.exec( + nsenter(usrnsPid, router2NetnsPid) + + `ip route add ${seedHostIp} dev ${router2SeedHost}`, + ); + // Add default routing + child_process.exec( + nsenter(usrnsPid, seedNetnsPid) + + `ip route add ${router1SeedHostIp} dev ${seedRouter1Host}`, + ); + child_process.exec( + nsenter(usrnsPid, seedNetnsPid) + + `ip route add ${router2SeedHostIp} dev ${seedRouter2Host}`, + ); + const router1SeedRouting = nsenter(usrnsPid, router1NetnsPid).concat( + 'iptables --table nat ', + '--append POSTROUTING ', + '--protocol udp ', + `--source ${agent1HostIp}${subnetMask} `, + `--out-interface ${router1SeedHost} `, + '--jump SNAT ', + `--to-source ${router1SeedHostIp}:${mappedPort} `, + '--persistent', + ); + const router2SeedRouting = nsenter(usrnsPid, router2NetnsPid).concat( + 'iptables --table nat ', + '--append POSTROUTING ', + '--protocol udp ', + `--source ${agent2HostIp}${subnetMask} `, + `--out-interface ${router2SeedHost} `, + '--jump SNAT ', + `--to-source ${router2SeedHostIp}:${mappedPort} `, + '--persistent', + ); + child_process.exec(router1SeedRouting); + child_process.exec(router2SeedRouting); +} + +/** + * Runs pk command through subprocess inside a network namespace + * This is used when a subprocess functionality needs to be used + * This is intended for terminating subprocesses + * Both stdout and stderr are the entire output including newlines + * @param env Augments env for command execution + * @param cwd Defaults to temporary directory + */ +async function pkExecNs( + usrnsPid: number, + netnsPid: number, + args: Array = [], + env: Record = {}, + cwd?: string, +): Promise<{ + exitCode: number; + stdout: string; + stderr: string; +}> { + cwd = + cwd ?? (await fs.promises.mkdtemp(path.join(os.tmpdir(), 'polykey-test-'))); + env = { + ...process.env, + ...env, + }; + // Recall that we attempt to connect to all specified seed nodes on agent start. + // Therefore, for testing purposes only, we default the seed nodes as empty + // (if not defined in the env) to ensure no attempted connections. A regular + // PolykeyAgent is expected to initially connect to the mainnet seed nodes + env['PK_SEED_NODES'] = env['PK_SEED_NODES'] ?? ''; + const tsConfigPath = path.resolve( + path.join(global.projectDir, 'tsconfig.json'), + ); + const tsConfigPathsRegisterPath = path.resolve( + path.join(global.projectDir, 'node_modules/tsconfig-paths/register'), + ); + const polykeyPath = path.resolve( + path.join(global.projectDir, 'src/bin/polykey.ts'), + ); + return new Promise((resolve, reject) => { + child_process.execFile( + 'nsenter', + [ + '--target', + usrnsPid.toString(), + '--user', + '--preserve-credentials', + 'nsenter', + '--target', + netnsPid.toString(), + '--net', + 'ts-node', + '--project', + tsConfigPath, + '--require', + tsConfigPathsRegisterPath, + '--compiler', + 'typescript-cached-transpile', + '--transpile-only', + polykeyPath, + ...args, + ], + { + env, + cwd, + windowsHide: true, + }, + (error, stdout, stderr) => { + if (error != null && error.code === undefined) { + // This can only happen when the command is killed + return reject(error); + } else { + // Success and Unsuccessful exits are valid here + return resolve({ + exitCode: error && error.code != null ? error.code : 0, + stdout, + stderr, + }); + } + }, + ); + }); +} + +/** + * Launch pk command through subprocess inside a network namespace + * This is used when a subprocess functionality needs to be used + * This is intended for non-terminating subprocesses + * @param env Augments env for command execution + * @param cwd Defaults to temporary directory + */ +async function pkSpawnNs( + usrnsPid: number, + netnsPid: number, + args: Array = [], + env: Record = {}, + cwd?: string, + logger: Logger = new Logger(pkSpawnNs.name), +): Promise { + cwd = + cwd ?? (await fs.promises.mkdtemp(path.join(os.tmpdir(), 'polykey-test-'))); + env = { + ...process.env, + ...env, + }; + // Recall that we attempt to connect to all specified seed nodes on agent start. + // Therefore, for testing purposes only, we default the seed nodes as empty + // (if not defined in the env) to ensure no attempted connections. A regular + // PolykeyAgent is expected to initially connect to the mainnet seed nodes + env['PK_SEED_NODES'] = env['PK_SEED_NODES'] ?? ''; + const tsConfigPath = path.resolve( + path.join(global.projectDir, 'tsconfig.json'), + ); + const tsConfigPathsRegisterPath = path.resolve( + path.join(global.projectDir, 'node_modules/tsconfig-paths/register'), + ); + const polykeyPath = path.resolve( + path.join(global.projectDir, 'src/bin/polykey.ts'), + ); + const subprocess = child_process.spawn( + 'nsenter', + [ + '--target', + usrnsPid.toString(), + '--user', + '--preserve-credentials', + 'nsenter', + '--target', + netnsPid.toString(), + '--net', + 'ts-node', + '--project', + tsConfigPath, + '--require', + tsConfigPathsRegisterPath, + '--compiler', + 'typescript-cached-transpile', + '--transpile-only', + polykeyPath, + ...args, + ], + { + env, + cwd, + stdio: ['pipe', 'pipe', 'pipe'], + windowsHide: true, + shell: true, + }, + ); + const rlErr = readline.createInterface(subprocess.stderr!); + rlErr.on('line', (l) => { + // The readline library will trim newlines + logger.info(l); + }); + return subprocess; +} + +/** + * Setup routing between an agent and router with no NAT rules + */ +function setupDMZ( + usrnsPid: number, + routerNsPid: number, + agentIp: string, + agentPort: string, + routerExt: string, + routerExtIp: string, +) { + const postroutingCommand = nsenter(usrnsPid, routerNsPid).concat( + 'iptables --table nat ', + '--append POSTROUTING ', + '--protocol udp ', + `--source ${agentIp}${subnetMask} `, + `--out-interface ${routerExt} `, + '--jump SNAT ', + `--to-source ${routerExtIp}:${mappedPort}`, + ); + const preroutingCommand = nsenter(usrnsPid, routerNsPid).concat( + 'iptables --table nat ', + '--append PREROUTING ', + '--protocol udp ', + `--destination-port ${mappedPort} `, + `--in-interface ${routerExt} `, + '--jump DNAT ', + `--to-destination ${agentIp}:${agentPort}`, + ); + child_process.exec(postroutingCommand); + child_process.exec(preroutingCommand); +} + +/** + * Setup Port-Restricted Cone NAT for a namespace (on the router namespace) + */ +function setupNATEndpointIndependentMapping( + usrnsPid: number, + routerNsPid: number, + routerExt: string, +) { + const natCommand = nsenter(usrnsPid, routerNsPid).concat( + 'iptables --table nat ', + '--append POSTROUTING ', + '--protocol udp ', + `--out-interface ${routerExt} `, + '--jump MASQUERADE ', + `--to-ports ${mappedPort}`, + ); + const dropCommand = nsenter(usrnsPid, routerNsPid).concat( + 'iptables --table filter ', + '--append INPUT ', + '--jump DROP', + ); + child_process.exec(natCommand); + child_process.exec(dropCommand); +} + +/** + * Setup Symmetric NAT for a namespace (on the router namespace) + */ +function setupNATEndpointDependentMapping( + usrnsPid: number, + routerNsPid: number, + routerExt: string, +) { + const command = nsenter(usrnsPid, routerNsPid).concat( + 'iptables --table nat ', + '--append POSTROUTING ', + '--protocol udp ', + `--out-interface ${routerExt} `, + '--jump MASQUERADE ', + `--random`, + ); + child_process.exec(command); +} + +async function setupNATWithSeedNode( + agent1NAT: NATType, + agent2NAT: NATType, + logger: Logger = new Logger(setupNAT.name, LogLevel.WARN, [ + new StreamHandler(), + ]), +) { + const dataDir = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'polykey-test-'), + ); + const password = 'password'; + // Create a user namespace containing four network namespaces + // Two agents and two routers + const usrns = createUserNamespace(); + const seedNetns = createNetworkNamespace(usrns.pid); + const agent1Netns = createNetworkNamespace(usrns.pid); + const agent2Netns = createNetworkNamespace(usrns.pid); + const router1Netns = createNetworkNamespace(usrns.pid); + const router2Netns = createNetworkNamespace(usrns.pid); + setupNetworkNamespaceInterfaces( + usrns.pid, + agent1Netns.pid, + router1Netns.pid, + router2Netns.pid, + agent2Netns.pid, + ); + setupSeedNamespaceInterfaces( + usrns.pid, + seedNetns.pid, + router1Netns.pid, + router2Netns.pid, + ); + const seedNode = await pkSpawnNs( + usrns.pid, + seedNetns.pid, + [ + 'agent', + 'start', + '--node-path', + path.join(dataDir, 'seed'), + '--root-key-pair-bits', + '1024', + '--client-host', + '127.0.0.1', + '--proxy-host', + '0.0.0.0', + '--connection-timeout', + '1000', + '--workers', + '0', + '--verbose', + '--format', + 'json', + ], + { + PK_PASSWORD: password, + }, + dataDir, + logger.getChild('seed'), + ); + const rlOutSeed = readline.createInterface(seedNode.stdout!); + const stdoutSeed = await new Promise((resolve, reject) => { + rlOutSeed.once('line', resolve); + rlOutSeed.once('close', reject); + }); + const nodeIdSeed = JSON.parse(stdoutSeed).nodeId; + const proxyPortSeed = JSON.parse(stdoutSeed).proxyPort; + const agent1 = await pkSpawnNs( + usrns.pid, + agent1Netns.pid, + [ + 'agent', + 'start', + '--node-path', + path.join(dataDir, 'agent1'), + '--root-key-pair-bits', + '1024', + '--client-host', + '127.0.0.1', + '--proxy-host', + `${agent1HostIp}`, + '--workers', + '0', + '--connection-timeout', + '1000', + '--seed-nodes', + `${nodeIdSeed}@${seedHostIp}:${proxyPortSeed}`, + '--verbose', + '--format', + 'json', + ], + { + PK_PASSWORD: password, + }, + dataDir, + logger.getChild('agent1'), + ); + const rlOutNode1 = readline.createInterface(agent1.stdout!); + const stdoutNode1 = await new Promise((resolve, reject) => { + rlOutNode1.once('line', resolve); + rlOutNode1.once('close', reject); + }); + const nodeId1 = JSON.parse(stdoutNode1).nodeId; + const agent2 = await pkSpawnNs( + usrns.pid, + agent2Netns.pid, + [ + 'agent', + 'start', + '--node-path', + path.join(dataDir, 'agent2'), + '--root-key-pair-bits', + '1024', + '--client-host', + '127.0.0.1', + '--proxy-host', + `${agent2HostIp}`, + '--workers', + '0', + '--connection-timeout', + '1000', + '--seed-nodes', + `${nodeIdSeed}@${seedHostIp}:${proxyPortSeed}`, + '--verbose', + '--format', + 'json', + ], + { + PK_PASSWORD: password, + }, + dataDir, + logger.getChild('agent2'), + ); + const rlOutNode2 = readline.createInterface(agent2.stdout!); + const stdoutNode2 = await new Promise((resolve, reject) => { + rlOutNode2.once('line', resolve); + rlOutNode2.once('close', reject); + }); + const nodeId2 = JSON.parse(stdoutNode2).nodeId; + // Until nodes add the information of nodes that connect to them must + // do it manually + await pkExecNs( + usrns.pid, + seedNode.pid, + ['nodes', 'add', nodeId1, agent1RouterHostExtIp, mappedPort], + { + PK_NODE_PATH: path.join(dataDir, 'seed'), + PK_PASSWORD: password, + }, + dataDir, + ); + await pkExecNs( + usrns.pid, + seedNode.pid, + ['nodes', 'add', nodeId2, agent2RouterHostExtIp, mappedPort], + { + PK_NODE_PATH: path.join(dataDir, 'seed'), + PK_PASSWORD: password, + }, + dataDir, + ); + // Apply appropriate NAT rules + switch (agent1NAT) { + case 'dmz': { + setupDMZ( + usrns.pid, + router1Netns.pid, + agent1HostIp, + JSON.parse(stdoutNode1).proxyPort, + agent1RouterHostExt, + agent1RouterHostExtIp, + ); + break; + } + case 'eim': { + setupNATEndpointIndependentMapping( + usrns.pid, + router1Netns.pid, + agent1RouterHostExt, + ); + break; + } + case 'edm': { + setupNATEndpointDependentMapping( + usrns.pid, + router1Netns.pid, + agent1RouterHostExt, + ); + break; + } + } + switch (agent2NAT) { + case 'dmz': { + setupDMZ( + usrns.pid, + router2Netns.pid, + agent2HostIp, + JSON.parse(stdoutNode2).proxyPort, + agent2RouterHostExt, + agent2RouterHostExtIp, + ); + break; + } + case 'eim': { + setupNATEndpointIndependentMapping( + usrns.pid, + router2Netns.pid, + agent2RouterHostExt, + ); + break; + } + case 'edm': { + setupNATEndpointDependentMapping( + usrns.pid, + router2Netns.pid, + agent2RouterHostExt, + ); + break; + } + } + return { + userPid: usrns.pid, + agent1Pid: agent1Netns.pid, + agent2Pid: agent2Netns.pid, + password, + dataDir, + agent1NodePath: path.join(dataDir, 'agent1'), + agent2NodePath: path.join(dataDir, 'agent2'), + agent1NodeId: nodeId1, + agent2NodeId: nodeId2, + tearDownNAT: async () => { + agent2.kill('SIGTERM'); + await testBinUtils.processExit(agent2); + agent1.kill('SIGTERM'); + await testBinUtils.processExit(agent1); + seedNode.kill('SIGTERM'); + await testBinUtils.processExit(seedNode); + router2Netns.kill('SIGTERM'); + await testBinUtils.processExit(router2Netns); + router1Netns.kill('SIGTERM'); + await testBinUtils.processExit(router1Netns); + agent2Netns.kill('SIGTERM'); + await testBinUtils.processExit(agent2Netns); + agent1Netns.kill('SIGTERM'); + await testBinUtils.processExit(agent1Netns); + seedNetns.kill('SIGTERM'); + await testBinUtils.processExit(seedNetns); + usrns.kill('SIGTERM'); + await testBinUtils.processExit(usrns); + await fs.promises.rm(dataDir, { + force: true, + recursive: true, + }); + }, + }; +} + +async function setupNAT( + agent1NAT: NATType, + agent2NAT: NATType, + logger: Logger = new Logger(setupNAT.name, LogLevel.WARN, [ + new StreamHandler(), + ]), +) { + const dataDir = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'polykey-test-'), + ); + const password = 'password'; + // Create a user namespace containing four network namespaces + // Two agents and two routers + const usrns = createUserNamespace(); + const agent1Netns = createNetworkNamespace(usrns.pid); + const agent2Netns = createNetworkNamespace(usrns.pid); + const router1Netns = createNetworkNamespace(usrns.pid); + const router2Netns = createNetworkNamespace(usrns.pid); + setupNetworkNamespaceInterfaces( + usrns.pid, + agent1Netns.pid, + router1Netns.pid, + router2Netns.pid, + agent2Netns.pid, + ); + const agent1 = await pkSpawnNs( + usrns.pid, + agent1Netns.pid, + [ + 'agent', + 'start', + '--node-path', + path.join(dataDir, 'agent1'), + '--root-key-pair-bits', + '1024', + '--client-host', + '127.0.0.1', + '--proxy-host', + `${agent1HostIp}`, + '--connection-timeout', + '1000', + '--workers', + '0', + '-vv', + '--format', + 'json', + ], + { + PK_PASSWORD: password, + }, + dataDir, + logger.getChild('agent1'), + ); + const rlOutNode1 = readline.createInterface(agent1.stdout!); + const stdoutNode1 = await new Promise((resolve, reject) => { + rlOutNode1.once('line', resolve); + rlOutNode1.once('close', reject); + }); + const nodeId1 = JSON.parse(stdoutNode1).nodeId; + const proxyPort1 = JSON.parse(stdoutNode1).proxyPort; + const agent2 = await pkSpawnNs( + usrns.pid, + agent2Netns.pid, + [ + 'agent', + 'start', + '--node-path', + path.join(dataDir, 'agent2'), + '--root-key-pair-bits', + '1024', + '--client-host', + '127.0.0.1', + '--proxy-host', + `${agent2HostIp}`, + '--connection-timeout', + '1000', + '--workers', + '0', + '-vv', + '--format', + 'json', + ], + { + PK_PASSWORD: password, + }, + dataDir, + logger.getChild('agent2'), + ); + const rlOutNode2 = readline.createInterface(agent2.stdout!); + const stdoutNode2 = await new Promise((resolve, reject) => { + rlOutNode2.once('line', resolve); + rlOutNode2.once('close', reject); + }); + const nodeId2 = JSON.parse(stdoutNode2).nodeId; + const proxyPort2 = JSON.parse(stdoutNode2).proxyPort; + // Apply appropriate NAT rules + switch (agent1NAT) { + case 'dmz': { + setupDMZ( + usrns.pid, + router1Netns.pid, + agent1HostIp, + proxyPort1, + agent1RouterHostExt, + agent1RouterHostExtIp, + ); + break; + } + case 'eim': { + setupNATEndpointIndependentMapping( + usrns.pid, + router1Netns.pid, + agent1RouterHostExt, + ); + break; + } + case 'edm': { + setupNATEndpointDependentMapping( + usrns.pid, + router1Netns.pid, + agent1RouterHostExt, + ); + break; + } + } + switch (agent2NAT) { + case 'dmz': { + setupDMZ( + usrns.pid, + router2Netns.pid, + agent2HostIp, + proxyPort2, + agent2RouterHostExt, + agent2RouterHostExtIp, + ); + break; + } + case 'eim': { + setupNATEndpointIndependentMapping( + usrns.pid, + router2Netns.pid, + agent2RouterHostExt, + ); + break; + } + case 'edm': { + setupNATEndpointDependentMapping( + usrns.pid, + router2Netns.pid, + agent2RouterHostExt, + ); + break; + } + } + return { + userPid: usrns.pid, + agent1Pid: agent1Netns.pid, + agent2Pid: agent2Netns.pid, + password, + dataDir, + agent1NodePath: path.join(dataDir, 'agent1'), + agent2NodePath: path.join(dataDir, 'agent2'), + agent1NodeId: nodeId1, + agent1Host: agent1RouterHostExtIp, + agent1ProxyPort: mappedPort, + agent2NodeId: nodeId2, + agent2Host: agent2RouterHostExtIp, + agent2ProxyPort: mappedPort, + tearDownNAT: async () => { + agent2.kill('SIGTERM'); + await testBinUtils.processExit(agent2); + agent1.kill('SIGTERM'); + await testBinUtils.processExit(agent1); + router2Netns.kill('SIGTERM'); + await testBinUtils.processExit(router2Netns); + router1Netns.kill('SIGTERM'); + await testBinUtils.processExit(router1Netns); + agent2Netns.kill('SIGTERM'); + await testBinUtils.processExit(agent2Netns); + agent1Netns.kill('SIGTERM'); + await testBinUtils.processExit(agent1Netns); + usrns.kill('SIGTERM'); + await testBinUtils.processExit(usrns); + await fs.promises.rm(dataDir, { + force: true, + recursive: true, + }); + }, + }; +} + +export { + createUserNamespace, + createNetworkNamespace, + setupNetworkNamespaceInterfaces, + pkExecNs, + pkSpawnNs, + setupNAT, + setupNATWithSeedNode, +};