Skip to content

Commit

Permalink
Add --local option to kv commands (#3684)
Browse files Browse the repository at this point in the history 
* Add --local to kv commands

* Add tests for kv --local commands

* Move `node:stream/consumers` types to `src`

This where Wrangler stores its other `.d.ts` files.

---------

Co-authored-by: bcoll <bcoll@cloudflare.com>
  • Loading branch information
@jspspike @mrbbot
jspspike and mrbbot authored Aug 1, 2023
1 parent 2d82ef7 commit ff8603b
Show file tree
Hide file tree
Showing 6 changed files with 523 additions and 40 deletions.
5 changes: 5 additions & 0 deletions .changeset/eight-rice-raise.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"wrangler": minor
---

Added --local option for kv commands to interact with local persisted kv entries
290 changes: 282 additions & 8 deletions packages/wrangler/src/__tests__/kv.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -568,7 +568,9 @@ describe("wrangler", () => {
--ttl Time for which the entries should be visible [number]
--expiration Time since the UNIX epoch after which the entry expires [number]
--metadata Arbitrary JSON that is associated with a key [string]
--path Read value from the file at a given path [string]"
--path Read value from the file at a given path [string]
--local Interact with local storage [boolean]
--persist-to Directory for local persistance [string]"
`);
expect(std.err).toMatchInlineSnapshot(`
"X [ERROR] Not enough non-option arguments: got 0, need at least 1
Expand Down Expand Up @@ -608,7 +610,9 @@ describe("wrangler", () => {
--ttl Time for which the entries should be visible [number]
--expiration Time since the UNIX epoch after which the entry expires [number]
--metadata Arbitrary JSON that is associated with a key [string]
--path Read value from the file at a given path [string]"
--path Read value from the file at a given path [string]
--local Interact with local storage [boolean]
--persist-to Directory for local persistance [string]"
`);
expect(std.err).toMatchInlineSnapshot(`
"X [ERROR] Exactly one of the arguments binding and namespace-id is required
Expand Down Expand Up @@ -648,7 +652,9 @@ describe("wrangler", () => {
--ttl Time for which the entries should be visible [number]
--expiration Time since the UNIX epoch after which the entry expires [number]
--metadata Arbitrary JSON that is associated with a key [string]
--path Read value from the file at a given path [string]"
--path Read value from the file at a given path [string]
--local Interact with local storage [boolean]
--persist-to Directory for local persistance [string]"
`);
expect(std.err).toMatchInlineSnapshot(`
"X [ERROR] Arguments binding and namespace-id are mutually exclusive
Expand Down Expand Up @@ -688,7 +694,9 @@ describe("wrangler", () => {
--ttl Time for which the entries should be visible [number]
--expiration Time since the UNIX epoch after which the entry expires [number]
--metadata Arbitrary JSON that is associated with a key [string]
--path Read value from the file at a given path [string]"
--path Read value from the file at a given path [string]
--local Interact with local storage [boolean]
--persist-to Directory for local persistance [string]"
`);
expect(std.err).toMatchInlineSnapshot(`
"X [ERROR] Exactly one of the arguments value and path is required
Expand Down Expand Up @@ -728,7 +736,9 @@ describe("wrangler", () => {
--ttl Time for which the entries should be visible [number]
--expiration Time since the UNIX epoch after which the entry expires [number]
--metadata Arbitrary JSON that is associated with a key [string]
--path Read value from the file at a given path [string]"
--path Read value from the file at a given path [string]
--local Interact with local storage [boolean]
--persist-to Directory for local persistance [string]"
`);
expect(std.err).toMatchInlineSnapshot(`
"X [ERROR] Arguments value and path are mutually exclusive
Expand Down Expand Up @@ -1102,7 +1112,9 @@ describe("wrangler", () => {
--binding The name of the namespace to get from [string]
--namespace-id The id of the namespace to get from [string]
--preview Interact with a preview namespace [boolean] [default: false]
--text Decode the returned value as a utf8 string [boolean] [default: false]"
--text Decode the returned value as a utf8 string [boolean] [default: false]
--local Interact with local storage [boolean]
--persist-to Directory for local persistance [string]"
`);
expect(std.err).toMatchInlineSnapshot(`
"X [ERROR] Not enough non-option arguments: got 0, need at least 1
Expand Down Expand Up @@ -1137,7 +1149,9 @@ describe("wrangler", () => {
--binding The name of the namespace to get from [string]
--namespace-id The id of the namespace to get from [string]
--preview Interact with a preview namespace [boolean] [default: false]
--text Decode the returned value as a utf8 string [boolean] [default: false]"
--text Decode the returned value as a utf8 string [boolean] [default: false]
--local Interact with local storage [boolean]
--persist-to Directory for local persistance [string]"
`);
expect(std.err).toMatchInlineSnapshot(`
"X [ERROR] Exactly one of the arguments binding and namespace-id is required
Expand Down Expand Up @@ -1173,7 +1187,9 @@ describe("wrangler", () => {
--binding The name of the namespace to get from [string]
--namespace-id The id of the namespace to get from [string]
--preview Interact with a preview namespace [boolean] [default: false]
--text Decode the returned value as a utf8 string [boolean] [default: false]"
--text Decode the returned value as a utf8 string [boolean] [default: false]
--local Interact with local storage [boolean]
--persist-to Directory for local persistance [string]"
`);
expect(std.err).toMatchInlineSnapshot(`
"X [ERROR] Arguments binding and namespace-id are mutually exclusive
Expand Down Expand Up @@ -1689,6 +1705,264 @@ describe("wrangler", () => {
});
});
});
describe("local", () => {
it("should put local kv storage", async () => {
await runWrangler(
`kv:key get val --namespace-id some-namespace-id --local --text`
);
expect(std.out).toMatchInlineSnapshot(`"Value not found"`);

await runWrangler(
`kv:key put val value --namespace-id some-namespace-id --local`
);
expect(std.out).toMatchInlineSnapshot(`
"Value not found
Writing the value \\"value\\" to key \\"val\\" on namespace some-namespace-id."
`);

await runWrangler(
`kv:key get val --namespace-id some-namespace-id --local --text`
);
expect(std.out).toMatchInlineSnapshot(`
"Value not found
Writing the value \\"value\\" to key \\"val\\" on namespace some-namespace-id.
value"
`);
});

it("should list local kv storage", async () => {
await runWrangler(`kv:key list --namespace-id some-namespace-id --local`);
expect(std.out).toMatchInlineSnapshot(`
"{
\\"keys\\": [],
\\"list_complete\\": true
}"
`);

await runWrangler(
`kv:key put val value --namespace-id some-namespace-id --local`
);

await runWrangler(`kv:key list --namespace-id some-namespace-id --local`);
expect(std.out).toMatchInlineSnapshot(`
"{
\\"keys\\": [],
\\"list_complete\\": true
}
Writing the value \\"value\\" to key \\"val\\" on namespace some-namespace-id.
{
\\"keys\\": [
{
\\"name\\": \\"val\\"
}
],
\\"list_complete\\": true
}"
`);
});

it("should delete local kv storage", async () => {
await runWrangler(
`kv:key put val value --namespace-id some-namespace-id --local`
);
await runWrangler(
`kv:key get val --namespace-id some-namespace-id --local --text`
);
expect(std.out).toMatchInlineSnapshot(`
"Writing the value \\"value\\" to key \\"val\\" on namespace some-namespace-id.
value"
`);
await runWrangler(
`kv:key delete val --namespace-id some-namespace-id --local`
);
expect(std.out).toMatchInlineSnapshot(`
"Writing the value \\"value\\" to key \\"val\\" on namespace some-namespace-id.
value
Deleting the key \\"val\\" on namespace some-namespace-id."
`);

await runWrangler(
`kv:key get val --namespace-id some-namespace-id --local --text`
);
expect(std.out).toMatchInlineSnapshot(`
"Writing the value \\"value\\" to key \\"val\\" on namespace some-namespace-id.
value
Deleting the key \\"val\\" on namespace some-namespace-id.
Value not found"
`);
});

it("should put local bulk kv storage", async () => {
await runWrangler(`kv:key list --namespace-id bulk-namespace-id --local`);
expect(std.out).toMatchInlineSnapshot(`
"{
\\"keys\\": [],
\\"list_complete\\": true
}"
`);

const keyValues = [
{
key: "hello",
value: "world",
},
{
key: "test",
value: "value",
},
];
writeFileSync("./keys.json", JSON.stringify(keyValues));
await runWrangler(
`kv:bulk put keys.json --namespace-id bulk-namespace-id --local`
);
expect(std.out).toMatchInlineSnapshot(`
"{
\\"keys\\": [],
\\"list_complete\\": true
}
Success!"
`);

await runWrangler(
`kv:key get test --namespace-id bulk-namespace-id --local --text`
);
expect(std.out).toMatchInlineSnapshot(`
"{
\\"keys\\": [],
\\"list_complete\\": true
}
Success!
value"
`);

await runWrangler(`kv:key list --namespace-id bulk-namespace-id --local`);
expect(std.out).toMatchInlineSnapshot(`
"{
\\"keys\\": [],
\\"list_complete\\": true
}
Success!
value
{
\\"keys\\": [
{
\\"name\\": \\"hello\\"
},
{
\\"name\\": \\"test\\"
}
],
\\"list_complete\\": true
}"
`);
});

it("should delete local bulk kv storage", async () => {
const keyValues = [
{
key: "hello",
value: "world",
},
{
key: "test",
value: "value",
},
];
writeFileSync("./keys.json", JSON.stringify(keyValues));
await runWrangler(
`kv:bulk put keys.json --namespace-id bulk-namespace-id --local`
);
await runWrangler(`kv:key list --namespace-id bulk-namespace-id --local`);
expect(std.out).toMatchInlineSnapshot(`
"Success!
{
\\"keys\\": [
{
\\"name\\": \\"hello\\"
},
{
\\"name\\": \\"test\\"
}
],
\\"list_complete\\": true
}"
`);
const keys = ["hello", "test"];
writeFileSync("./keys.json", JSON.stringify(keys));
await runWrangler(
`kv:bulk delete keys.json --namespace-id bulk-namespace-id --local --force`
);
expect(std.out).toMatchInlineSnapshot(`
"Success!
{
\\"keys\\": [
{
\\"name\\": \\"hello\\"
},
{
\\"name\\": \\"test\\"
}
],
\\"list_complete\\": true
}
Success!"
`);

await runWrangler(`kv:key list --namespace-id bulk-namespace-id --local`);
expect(std.out).toMatchInlineSnapshot(`
"Success!
{
\\"keys\\": [
{
\\"name\\": \\"hello\\"
},
{
\\"name\\": \\"test\\"
}
],
\\"list_complete\\": true
}
Success!
{
\\"keys\\": [],
\\"list_complete\\": true
}"
`);
});

it("should follow persist-to for local kv storage", async () => {
await runWrangler(
`kv:key put val value --namespace-id some-namespace-id --local`
);

await runWrangler(
`kv:key put val persistValue --namespace-id some-namespace-id --local --persist-to ./persistdir`
);
expect(std.out).toMatchInlineSnapshot(`
"Writing the value \\"value\\" to key \\"val\\" on namespace some-namespace-id.
Writing the value \\"persistValue\\" to key \\"val\\" on namespace some-namespace-id."
`);

await runWrangler(
`kv:key get val --namespace-id some-namespace-id --local --text`
);
expect(std.out).toMatchInlineSnapshot(`
"Writing the value \\"value\\" to key \\"val\\" on namespace some-namespace-id.
Writing the value \\"persistValue\\" to key \\"val\\" on namespace some-namespace-id.
value"
`);

await runWrangler(
`kv:key get val --namespace-id some-namespace-id --local --text --persist-to ./persistdir`
);
expect(std.out).toMatchInlineSnapshot(`
"Writing the value \\"value\\" to key \\"val\\" on namespace some-namespace-id.
Writing the value \\"persistValue\\" to key \\"val\\" on namespace some-namespace-id.
value
persistValue"
`);
});
});
});

function writeWranglerConfig() {
Expand Down
21 changes: 21 additions & 0 deletions packages/wrangler/src/kv/helpers.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,15 @@
import { URLSearchParams } from "node:url";
import path from "path";
import {
KVGateway,
NoOpLog,
createFileStorage,
sanitisePath,
defaultTimers,
} from "miniflare";
import { FormData } from "undici";
import { fetchListResult, fetchResult, fetchKVGetValue } from "../cfetch";
import { getLocalPersistencePath } from "../dev/get-local-persistence-path";
import { logger } from "../logger";
import type { Config } from "../config";

Expand Down Expand Up @@ -431,3 +440,15 @@ export function isValidKVNamespaceBinding(
typeof binding === "string" && /^[a-zA-Z_][a-zA-Z0-9_]*$/.test(binding)
);
}

export function localGateway(
persistTo: string | undefined,
configPath: string | undefined,
namespaceId: string
): KVGateway {
const persist = getLocalPersistencePath(persistTo, configPath);
const sanitisedNamespace = sanitisePath(namespaceId);
const persistPath = path.join(persist, "v3/kv", sanitisedNamespace);
const storage = createFileStorage(persistPath);
return new KVGateway(new NoOpLog(), storage, defaultTimers);
}
Loading

0 comments on commit ff8603b

Please sign in to comment.