diff --git a/.travis.yml b/.travis.yml
index fb703731..daffde76 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,4 +5,4 @@ cache:
   yarn: true
 script:
   - yarn run validate:json
-  - yarn run validate:schema
+  - yarn run validate:tests
diff --git a/package.json b/package.json
index d2078022..e4434a34 100644
--- a/package.json
+++ b/package.json
@@ -6,8 +6,8 @@
   "license": "MIT",
   "scripts": {
     "validate:json": "jsonlint-cli data/**/*.json",
-    "validate:schema": "jest tests/**-schema.test.js",
-    "validate:schema:watch": "jest --watch tests/**-schema.test.js",
+    "validate:tests": "jest tests/**.test.js",
+    "validate:tests:watch": "jest --watch tests/**.test.js",
     "format": "prettier --loglevel warn --write \"data/**/*.json\"",
     "changelog": "git log --pretty=format:'- %s (%h)' --no-merges",
     "ffg2xws": "node scripts/ffg2xws.js && prettier --write data/ffg-xws.json",
diff --git a/tests/helpers/data.js b/tests/helpers/data.js
new file mode 100644
index 00000000..e703c54e
--- /dev/null
+++ b/tests/helpers/data.js
@@ -0,0 +1,106 @@
+const manifest = require("../../data/manifest.json");
+
+const loadedData = {};
+const mapSlotXWSToName = {
+  astromech: "Astromech",
+  cannon: "Cannon",
+  configuration: "Configuration",
+  crew: "Crew",
+  device: "Device",
+  "force-power": "Force Power",
+  gunner: "Gunner",
+  illicit: "Illicit",
+  missile: "Missile",
+  modification: "Modification",
+  sensor: "Sensor",
+  tacticalrelay: "Tactical Relay",
+  talent: "Talent",
+  tech: "Tech",
+  title: "Title",
+  torpedo: "Torpedo",
+  turret: "Turret"
+};
+
+const loadPilotsAndShips = () => {
+  const allPilots = [];
+  const allShips = [];
+
+  manifest.pilots.forEach(({ ships }) => {
+    ships.forEach(filename => {
+      const { pilots = [], ship } = require(`../../${filename}`);
+      allPilots.push(...pilots);
+      allShips.push(ship);
+    });
+  });
+
+  return { ships: allShips, pilots: allPilots };
+};
+
+const getPilots = () => {
+  if (!loadedData.pilots) {
+    const { pilots } = loadPilotsAndShips();
+    loadedData.pilots = pilots;
+  }
+  return loadedData.pilots;
+};
+
+const getPilotXWSIds = () => {
+  const pilots = getPilots();
+  return pilots.map(p => p.xws).filter(Boolean);
+};
+
+const validatePilotXWSId = id => {
+  const ids = getPilotXWSIds();
+  if (ids.indexOf(id) === -1) {
+    throw new Error(`Pilot xws id "${id}" does not exist`);
+  }
+};
+
+const loadUpgrades = () => {
+  const allUpgrades = [];
+
+  manifest.upgrades.forEach(filename => {
+    const upgrades = require(`../../${filename}`);
+    allUpgrades.push(...upgrades);
+  });
+
+  return { upgrades: allUpgrades };
+};
+
+const getUpgrades = () => {
+  if (!loadedData.upgrades) {
+    const { upgrades } = loadUpgrades();
+    loadedData.upgrades = upgrades;
+  }
+  return loadedData.upgrades;
+};
+
+const getUpgradesXWSIds = () => {
+  const upgrades = getUpgrades();
+  return upgrades.map(u => u.xws).filter(Boolean);
+};
+
+const getUpgradesXWSIdsForSlot = slotName => {
+  const slotId = mapSlotXWSToName[slotName] || slotName;
+  const upgrades = getUpgrades();
+  return upgrades
+    .filter(u => u.sides.some(s => s.type === slotId))
+    .map(u => u.xws)
+    .filter(Boolean);
+};
+
+const validateUpgradeXWSIdForSlot = (id, slotName) => {
+  const ids = getUpgradesXWSIdsForSlot(slotName);
+  if (ids.indexOf(id) === -1) {
+    throw new Error(
+      `Upgrade xws id "${id}" does not exist for slot "${slotName}"`
+    );
+  }
+};
+
+module.exports = {
+  getPilotXWSIds,
+  validatePilotXWSId,
+  getUpgradesXWSIds,
+  validateUpgradeXWSIdForSlot
+};
diff --git a/tests/quick-builds-schema.test.js b/tests/quick-builds-schema.test.js
deleted file mode 100644
index d941b550..00000000
--- a/tests/quick-builds-schema.test.js
+++ /dev/null
@@ -1,28 +0,0 @@
-const path = require("path");
-const { matchers } = require("jest-json-schema");
-expect.extend(matchers);
-
-const { "quick-builds": quickBuildFiles } = require("../data/manifest.json");
-
-const quickBuildsSchema = require("./schemas/quick-build.schema.json");
-
-describe("Quick Builds", () => {
-  quickBuildFiles.forEach(file => {
-    const contents = require(`../${file}`);
-    const filename = path.basename(file, path.extname(file));
-    expect(contents).toHaveProperty("quick-builds");
-    const { "quick-builds": quickBuilds } = contents;
-    describe(`${filename}`, () => {
-      quickBuilds.forEach((qb, i) => {
-        const testName = [
-          `#${i}`,
-          `threat ${qb.threat}`,
-          qb.pilots.map(d => d.id).join(", ")
-        ].join(" - ");
-        test(testName, () => {
-          expect(qb).toMatchSchema(quickBuildsSchema);
-        });
-      });
-    });
-  });
-});
diff --git a/tests/quick-builds.test.js b/tests/quick-builds.test.js
new file mode 100644
index 00000000..ce8bdd96
--- /dev/null
+++ b/tests/quick-builds.test.js
@@ -0,0 +1,55 @@
+const path = require("path");
+const {
+  validatePilotXWSId,
+  validateUpgradeXWSIdForSlot
+} = require("./helpers/data");
+const { matchers } = require("jest-json-schema");
+expect.extend(matchers);
+
+const { "quick-builds": quickBuildFiles } = require("../data/manifest.json");
+
+const quickBuildsSchema = require("./schemas/quick-build.schema.json");
+
+describe("Quick Builds", () => {
+  quickBuildFiles.forEach(file => {
+    const contents = require(`../${file}`);
+
+    test("has a top-level `quick-builds` property", () => {
+      expect(contents).toHaveProperty("quick-builds");
+    });
+
+    // `data/quick-builds/resistance.json` -> `resistance`
+    const faction = path.basename(file, path.extname(file));
+
+    describe(`${faction}`, () => {
+      const { "quick-builds": quickBuilds } = contents;
+
+      quickBuilds.forEach((qb, i) => {
+        // Example: `#64 - threat 2 - deathrain`
+        const testName = [
+          `#${i}`,
+          `threat ${qb.threat}`,
+          qb.pilots.map(d => d.id).join(", ")
+        ].join(" - ");
+
+        describe(testName, () => {
+          test("matches schema", () => {
+            expect(qb).toMatchSchema(quickBuildsSchema);
+          });
+
+          test("uses valid XWS ids", () => {
+            qb.pilots.forEach(pilot => {
+              const { id: pilotId, upgrades = {} } = pilot;
+              validatePilotXWSId(pilotId);
+              Object.entries(upgrades).forEach(([slot, ids]) => {
+                ids.forEach(upgradeId =>
+                  validateUpgradeXWSIdForSlot(upgradeId, slot)
+                );
+              });
+            });
+          });
+        });
+      });
+    });
+  });
+});
diff --git a/tests/xws.test.js b/tests/xws.test.js
new file mode 100644
index 00000000..7b364868
--- /dev/null
+++ b/tests/xws.test.js
@@ -0,0 +1,20 @@
+const { getPilotXWSIds, getUpgradesXWSIds } = require("./helpers/data");
+
+const checkForDuplicates = ids => {
+  ids.reduce((acc, id) => {
+    if (acc.indexOf(id) > -1) {
+      throw new Error(`Duplicate XWS id: "${id}"`);
+    }
+    acc.push(id);
+    return acc;
+  }, []);
+};
+
+describe("XWS", () => {
+  test("no duplicate pilot XWS ids", () => {
+    checkForDuplicates(getPilotXWSIds());
+  });
+  test("no duplicate upgrade XWS ids", () => {
+    checkForDuplicates(getUpgradesXWSIds());
+  });
+});