Skip to content

Commit

Permalink
Add new sync implementation
Browse files Browse the repository at this point in the history
  • Loading branch information
@scott-rc
scott-rc committed Dec 8, 2023
1 parent e9b0a34 commit 6d39212
Show file tree
Hide file tree
Showing 10 changed files with 1,740 additions and 35 deletions.
145 changes: 126 additions & 19 deletions spec/__support__/filesync.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import fs from "fs-extra";
import assert from "node:assert";
import os from "node:os";
import pMap from "p-map";
import { expect, vi, type Assertion } from "vitest";
import { z } from "zod";
import {
Expand All @@ -10,13 +11,16 @@ import {
type FileSyncDeletedEventInput,
} from "../../src/__generated__/graphql.js";
import {
FILE_SYNC_FILES_QUERY,
FILE_SYNC_HASHES_QUERY,
PUBLISH_FILE_SYNC_EVENTS_MUTATION,
REMOTE_FILES_VERSION_QUERY,
REMOTE_FILE_SYNC_EVENTS_SUBSCRIPTION,
} from "../../src/services/app/edit-graphql.js";
import { Directory, swallowEnoent } from "../../src/services/filesync/directory.js";
import { Directory, swallowEnoent, type Hashes } from "../../src/services/filesync/directory.js";
import { FileSync, type File } from "../../src/services/filesync/filesync.js";
import { isEqualHashes } from "../../src/services/filesync/hashes.js";
import { noop } from "../../src/services/util/function.js";
import { isNil } from "../../src/services/util/is.js";
import { defaults, omit } from "../../src/services/util/object.js";
import { PromiseSignal } from "../../src/services/util/promise.js";
import type { PartialExcept } from "../types.js";
Expand Down Expand Up @@ -52,11 +56,6 @@ export type SyncScenarioOptions = {
*/
localFiles: Files;

/**
* The filesVersion Gadget currently has.
*/
gadgetFilesVersion?: 1n | 2n;

/**
* The files Gadget currently has.
* @default { ".gadget/": "" }
Expand Down Expand Up @@ -122,6 +121,11 @@ export type SyncScenario = {
* @returns A mock subscription for {@linkcode REMOTE_FILE_SYNC_EVENTS_SUBSCRIPTION}.
*/
expectGadgetChangesSubscription: () => MockEditGraphQLSubscription<REMOTE_FILE_SYNC_EVENTS_SUBSCRIPTION>;

/**
* Asserts that the local and gadget directories have the same hashes.
*/
expectLocalAndGadgetHashesMatch: () => Promise<void>;
};

/**
Expand All @@ -133,9 +137,9 @@ export type SyncScenario = {
export const makeSyncScenario = async ({
filesVersion1Files,
localFiles,
gadgetFilesVersion = 1n,
gadgetFiles,
}: Partial<SyncScenarioOptions> = {}): Promise<SyncScenario> => {
let gadgetFilesVersion = 1n;
await writeDir(testDirPath("gadget"), { ".gadget/": "", ...gadgetFiles });
const gadgetDir = await Directory.init(testDirPath("gadget"));

Expand All @@ -145,7 +149,8 @@ export const makeSyncScenario = async ({
const filesVersionDirs = new Map<bigint, Directory>();
filesVersionDirs.set(1n, filesVersion1Dir);

if (gadgetFilesVersion === 2n) {
if (!isEqualHashes(await gadgetDir.hashes(), await filesVersion1Dir.hashes())) {
gadgetFilesVersion = 2n;
await fs.copy(gadgetDir.path, testDirPath("fv-2"));
filesVersionDirs.set(2n, await Directory.init(testDirPath("fv-2")));
}
Expand All @@ -155,8 +160,10 @@ export const makeSyncScenario = async ({
await writeDir(testDirPath("local"), localFiles);
await localDir.loadIgnoreFile();

const syncJson: SyncJson = { app: testApp.slug, filesVersion: "1", mtime: Date.now() };
await fs.outputJSON(localDir.absolute(".gadget/sync.json"), syncJson, { spaces: 2 });
if (!localFiles[".gadget/sync.json"]) {
const syncJson: SyncJson = { app: testApp.slug, filesVersion: "1", mtime: Date.now() };
await fs.outputJSON(localDir.absolute(".gadget/sync.json"), syncJson, { spaces: 2 });
}
}

FileSync.init.mockRestore?.();
Expand Down Expand Up @@ -191,21 +198,84 @@ export const makeSyncScenario = async ({
await fs.chmod(gadgetDir.absolute(file.path), file.mode & 0o777);
}

gadgetFilesVersion += 1n;
const newFilesVersionDir = await Directory.init(testDirPath(`fv-${gadgetFilesVersion}`));
await fs.copy(gadgetDir.path, newFilesVersionDir.path);
filesVersionDirs.set(gadgetFilesVersion, newFilesVersionDir);
log.trace("new files version", { gadgetFilesVersion });
const gadgetFilesVersionDir = filesVersionDirs.get(gadgetFilesVersion);
assert(gadgetFilesVersionDir, `filesVersionDir ${gadgetFilesVersion} doesn't exist`);
if (!isEqualHashes(await gadgetDir.hashes(), await gadgetFilesVersionDir.hashes())) {
gadgetFilesVersion += 1n;
const newFilesVersionDir = await Directory.init(testDirPath(`fv-${gadgetFilesVersion}`));
await fs.copy(gadgetDir.path, newFilesVersionDir.path);
filesVersionDirs.set(gadgetFilesVersion, newFilesVersionDir);
log.trace("new files version", { gadgetFilesVersion });
}
};

void nockEditGraphQLResponse({
optional: true,
persist: true,
query: REMOTE_FILES_VERSION_QUERY,
result: () => {
query: FILE_SYNC_HASHES_QUERY,
expectVariables: z.object({ filesVersion: z.string().optional() }).optional(),
result: async (variables) => {
let filesVersion: bigint;
let hashes: Hashes;

if (isNil(variables?.filesVersion)) {
log.trace("sending gadget hashes", { gadgetFilesVersion, variables });
filesVersion = gadgetFilesVersion;
hashes = await gadgetDir.hashes();
} else {
filesVersion = BigInt(variables.filesVersion);
log.trace("sending files version hashes", { filesVersion, variables });
const filesVersionDir = filesVersionDirs.get(filesVersion);
assert(filesVersionDir, `filesVersionDir ${filesync.filesVersion} doesn't exist`);
hashes = await filesVersionDir.hashes();
}

return {
data: {
remoteFilesVersion: String(gadgetFilesVersion),
fileSyncHashes: {
filesVersion: String(filesVersion),
hashes,
},
},
};
},
});

void nockEditGraphQLResponse({
optional: true,
persist: true,
query: FILE_SYNC_FILES_QUERY,
expectVariables: z.object({
filesVersion: z.string().optional(),
paths: z.array(z.string()),
encoding: z.nativeEnum(FileSyncEncoding).optional(),
}),
result: async ({ filesVersion, paths, encoding }) => {
filesVersion ??= String(gadgetFilesVersion);
encoding ??= FileSyncEncoding.Base64;

const filesVersionDir = filesVersionDirs.get(BigInt(filesVersion));
assert(filesVersionDir, `filesVersionDir ${filesync.filesVersion} doesn't exist`);

return {
data: {
fileSyncFiles: {
filesVersion: filesVersion,
files: await pMap(paths, async (filepath) => {
const stats = await fs.stat(filesVersionDir.absolute(filepath));
let content = "";
if (stats.isFile()) {
content = (await fs.readFile(filesVersionDir.absolute(filepath), { encoding })) as string;
}

return {
path: filepath,
mode: stats.mode,
content,
encoding: FileSyncEncoding.Base64,
};
}),
},
},
};
},
Expand Down Expand Up @@ -359,9 +429,46 @@ export const makeSyncScenario = async ({
},

expectGadgetChangesSubscription: () => mockEditGraphQLSubs.expectSubscription(REMOTE_FILE_SYNC_EVENTS_SUBSCRIPTION),

expectLocalAndGadgetHashesMatch: async () => {
const localHashes = await localDir.hashes();
const gadgetHashes = await gadgetDir.hashes();
expect(localHashes).toEqual(gadgetHashes);
},
};
};

/**
* Creates hashes of the given files.
*/
export const makeHashes = async ({
filesVersionFiles,
localFiles,
gadgetFiles,
}: {
filesVersionFiles: Files;
localFiles: Files;
gadgetFiles?: Files;
}): Promise<{ filesVersionHashes: Hashes; localHashes: Hashes; gadgetHashes: Hashes }> => {
const [filesVersionHashes, localHashes, gadgetHashes] = await Promise.all([
writeDir(testDirPath("filesVersion"), filesVersionFiles)
.then(() => Directory.init(testDirPath("filesVersion")))
.then((dir) => dir.hashes()),

writeDir(testDirPath("local"), localFiles)
.then(() => Directory.init(testDirPath("local")))
.then((dir) => dir.hashes()),

!gadgetFiles
? Promise.resolve({})
: writeDir(testDirPath("gadget"), gadgetFiles)
.then(() => Directory.init(testDirPath("gadget")))
.then((dir) => dir.hashes()),
]);

return { filesVersionHashes, localHashes, gadgetHashes };
};

export const defaultFileMode = os.platform() === "win32" ? 0o100666 : 0o100644;
export const defaultDirMode = os.platform() === "win32" ? 0o40666 : 0o40755;

Expand Down
134 changes: 134 additions & 0 deletions spec/services/filesync/conflicts.spec.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
import { describe, expect, it } from "vitest";
import { getConflicts } from "../../../src/services/filesync/conflicts.js";
import { getChanges } from "../../../src/services/filesync/hashes.js";
import { makeHashes } from "../../__support__/filesync.js";

describe("getConflicts", () => {
it("returns conflicting changes", async () => {
const { filesVersionHashes, gadgetHashes, localHashes } = await makeHashes({
filesVersionFiles: {
"foo.js": "// foo",
"bar.js": "// bar",
"baz.js": "// baz",
},
localFiles: {
"foo.js": "// foo (local)",
"bar.js": "// bar (local)",
"qux.js": "// qux (local)",
},
gadgetFiles: {
"foo.js": "// foo (gadget)",
"baz.js": "// baz (gadget)",
"qux.js": "// qux (gadget)",
},
});

const localChanges = getChanges({ from: filesVersionHashes, to: localHashes });
const gadgetChanges = getChanges({ from: filesVersionHashes, to: gadgetHashes });
const conflicts = getConflicts({ localChanges, gadgetChanges });
expect(Object.fromEntries(conflicts)).toEqual({
"foo.js": {
localChange: { type: "update", sourceHash: filesVersionHashes["foo.js"], targetHash: localHashes["foo.js"] },
gadgetChange: { type: "update", sourceHash: filesVersionHashes["foo.js"], targetHash: gadgetHashes["foo.js"] },
},
"bar.js": {
localChange: { type: "update", sourceHash: filesVersionHashes["bar.js"], targetHash: localHashes["bar.js"] },
gadgetChange: { type: "delete", sourceHash: filesVersionHashes["bar.js"] },
},
"baz.js": {
localChange: { type: "delete", sourceHash: filesVersionHashes["baz.js"] },
gadgetChange: { type: "update", sourceHash: filesVersionHashes["baz.js"], targetHash: gadgetHashes["baz.js"] },
},
"qux.js": {
localChange: { type: "create", targetHash: localHashes["qux.js"] },
gadgetChange: { type: "create", targetHash: gadgetHashes["qux.js"] },
},
});
});

it("doesn't return non-conflicting changes", async () => {
const { filesVersionHashes, gadgetHashes, localHashes } = await makeHashes({
filesVersionFiles: {
"foo.js": "// foo",
"bar.js": "// bar",
"baz.js": "// baz",
},
localFiles: {
"foo.js": "// foo (same)",
"bar.js": "// bar (local)",
"baz.js": "// baz",
"qux.js": "// qux (same)",
},
gadgetFiles: {
"foo.js": "// foo (same)",
"bar.js": "// bar",
"baz.js": "// baz (gadget)",
"qux.js": "// qux (same)",
},
});

const localChanges = getChanges({ from: filesVersionHashes, to: localHashes });
const gadgetChanges = getChanges({ from: filesVersionHashes, to: gadgetHashes });
const conflicts = getConflicts({ localChanges, gadgetChanges });
expect(Object.fromEntries(conflicts)).toEqual({});
expect(conflicts.size).toBe(0);
});

it("deletes .gadget/ conflicts from local changes", async () => {
const { filesVersionHashes, gadgetHashes, localHashes } = await makeHashes({
filesVersionFiles: {
".gadget/client.js": "// client",
"foo.js": "// foo",
},
localFiles: {
".gadget/client.js": "// client (local)",
"foo.js": "// foo (local)",
},
gadgetFiles: {
".gadget/client.js": "// client (gadget)",
"foo.js": "// foo (gadget)",
},
});

const localChanges = getChanges({ from: filesVersionHashes, to: localHashes });
expect(Object.fromEntries(localChanges)).toEqual({
".gadget/client.js": {
type: "update",
sourceHash: filesVersionHashes[".gadget/client.js"],
targetHash: localHashes[".gadget/client.js"],
},
"foo.js": { type: "update", sourceHash: filesVersionHashes["foo.js"], targetHash: localHashes["foo.js"] },
});

const gadgetChanges = getChanges({ from: filesVersionHashes, to: gadgetHashes });
expect(Object.fromEntries(gadgetChanges)).toEqual({
".gadget/client.js": {
type: "update",
sourceHash: filesVersionHashes[".gadget/client.js"],
targetHash: gadgetHashes[".gadget/client.js"],
},
"foo.js": { type: "update", sourceHash: filesVersionHashes["foo.js"], targetHash: gadgetHashes["foo.js"] },
});

const conflicts = getConflicts({ localChanges, gadgetChanges });
expect(Object.fromEntries(conflicts)).toEqual({
"foo.js": {
localChange: { type: "update", sourceHash: filesVersionHashes["foo.js"], targetHash: localHashes["foo.js"] },
gadgetChange: { type: "update", sourceHash: filesVersionHashes["foo.js"], targetHash: gadgetHashes["foo.js"] },
},
});

expect(Object.fromEntries(localChanges)).toEqual({
"foo.js": { type: "update", sourceHash: filesVersionHashes["foo.js"], targetHash: localHashes["foo.js"] },
});

expect(Object.fromEntries(gadgetChanges)).toEqual({
".gadget/client.js": {
type: "update",
sourceHash: filesVersionHashes[".gadget/client.js"],
targetHash: gadgetHashes[".gadget/client.js"],
},
"foo.js": { type: "update", sourceHash: filesVersionHashes["foo.js"], targetHash: gadgetHashes["foo.js"] },
});
});
});
Loading

0 comments on commit 6d39212

Please sign in to comment.