Looking for a cleaner way to get a count of items matching my query

[dbudwin]

I'm new to electrodb, and I'm using it to interact with an existing DynamoDB database. We're not using a single table, but have about 150 tables that make up our schema.

I have this query which does work, but I feel like it could be improved:

public async controlsNotInPlace({ ownerGroup }: WithOwnerGroupParams): Promise<number> {
  try {
    const result: QueryResponse<typeof ControlEntity> = await ControlEntity.query
      .controlsByOwnerGroup({ ownerGroup })
      .where(
        ({ inPlace }, { eq, notExists }) => `
          ${notExists(inPlace)} OR ${eq(inPlace, false)}
        `,
      )
      .go({ data: "raw" });

    console.log(">>> result <<<");
    // eslint-disable-next-line @typescript-eslint/dot-notation
    console.log(result.data["Count"]);

    // eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/dot-notation
    return result.data["Count"];
  } catch (error) {
    // omitted for brevity
  }
}

My ControlEntity entity looks like:

export const ControlEntity = new Entity(
  {
    model: {
      entity: Table.Control,
      version: "1",
      service: rivialPlatformService,
    },
    attributes: {
      id: {
        type: "string",
        required: true,
      },
      ownerGroup: {
        type: "string",
        required: true,
      },
      name: {
        type: "string",
        required: true,
      },
      createdAt: {
        type: "string",
        required: true,
        validate: (value): boolean => dateRegex.test(value),
      },
      updatedAt: {
        type: "string",
        required: true,
        validate: (value): boolean => dateRegex.test(value),
      },
      controlControlSetId: {
        type: "string",
      },
      customFieldData: {
        type: "string", // JSON string
      },
      inPlace: {
        type: "boolean",
      },
      isDisabled: {
        type: "boolean",
      },
      notes: {
        type: "list",
        items: {
          type: "map",
          properties: {
            content: { type: "string" },
            createdAt: { type: "string" },
            id: { type: "string" },
            owner: { type: "string" },
          },
        },
      },
      statementNumber: {
        type: "string",
      },
      templateID: {
        type: "string",
      },
    },
    indexes: {
      primary: {
        pk: {
          field: "pk",
          composite: ["id"],
        },
        sk: {
          field: "sk",
          composite: [],
        },
      },
      controlsByOwnerGroup: {
        index: "controlsByOwnerGroup",
        pk: {
          field: "ownerGroup",
          composite: ["ownerGroup"],
        },
      },
      gsiControlSetControls: {
        index: "gsi-ControlSetControls",
        pk: {
          field: "controlControlSetId",
          composite: ["controlControlSetId"],
        },
      },
      listControlsByControlSet: {
        index: "listControlsByControlSet",
        pk: {
          field: "pk_owner",
          composite: ["ownerGroup"],
        },
        sk: {
          field: "sk_control",
          composite: ["controlControlSetId"],
        },
      },
      // ...commonIndexes,
    },
  },
  { client, table: getTableName(Table.Control), logger },
);

The logger outputs what I expect, 5 total items where only 2 of them match the above query:

{
    "type": "results",
    "method": "query",
    "config": {
        "includeKeys": false,
        "originalErr": false,
        "raw": true,
        "params": {},
        "page": {},
        "lastEvaluatedKeyRaw": false,
        "pager": "named",
        "unprocessed": "item",
        "response": "default",
        "cursor": null,
        "data": "raw",
        "compare": "keys",
        "complete": false,
        "ignoreOwnership": false,
        "_providedIgnoreOwnership": false,
        "_isPagination": true,
        "_isCollectionQuery": false,
        "pages": 1,
        "listeners": [],
        "preserveBatchOrder": false,
        "attributes": [],
        "terminalOperation": "go",
        "formatCursor": {},
        "hydrate": false,
        "_includeOnResponseItem": {}
    },
    "success": true,
    "results": {
        "$metadata": {
            "httpStatusCode": 200,
            "requestId": "6BJN4R5FG2HISJE93KQ1HOQO9NVV4KQNSO5AEMVJF66Q9ASUAAJG",
            "attempts": 1,
            "totalRetryDelay": 0
        },
        "Count": 2,
        "Items": [
            {
                "__typename": "Control",
                "statementNumber": "",
                "ownerGroup": "2f8c6413-cd1e-4578-bcd6-68faa46caf30",
                "customFieldData": {},
                "isDisabled": false,
                "updatedAt": "2024-08-28T14:38:25.083Z",
                "createdAt": "2024-08-28T14:38:16.919Z",
                "id": "fcc9e358-2f1a-f994-86a2-c1d1f0f32b20",
                "name": "Test 1",
                "controlControlSetId": "aad30d0c-9641-4f2f-bbb5-12277420c3ac",
                "inPlace": false
            },
            {
                "__typename": "Control",
                "statementNumber": "",
                "ownerGroup": "2f8c6413-cd1e-4578-bcd6-68faa46caf30",
                "customFieldData": {},
                "isDisabled": false,
                "updatedAt": "2024-08-28T14:38:45.188Z",
                "createdAt": "2024-08-28T14:38:45.188Z",
                "id": "31348396-914f-43a5-982d-f3699791463d",
                "name": "Test 5",
                "controlControlSetId": "aad30d0c-9641-4f2f-bbb5-12277420c3ac",
                "inPlace": false
            }
        ],
        "ScannedCount": 5
    }
}

The other console.log statements from above just print out this:

>>> result <<<
2
{ data: [], cursor: null }

My basic questions boil down to:

1. Is using raw okay here? Feels a little icky to me.
2. Can I get better type hinting so I don't need the eslint ignore comments?
3. Is there a simpler way to get a count of items that would be returned from a query, similar to dynamoose's query.count() function?
4. Is there a simpler way to write the logic in the where caluse?

[tywalch]

Hey @dbudwin 👋

Firstly, I want to mention the execution option { ignoreOwnership: true }, which will remove filters added by ElectroDB to help isolate items in some types of queries. ElectroDB internally manages two fields similar to __typename, and this option removes checks for these fields (which your items naturally won't have).

Now to your questions:

1. Is using raw okay here? Feels a little icky to me.

Raw is an escape hatch for things like this that don't have a proper implementation yet. I started a branch locally to change the response typing on { data: "raw" } but hit a few sticking points tonight.

2. Can I get better type hinting so I don't need the eslint ignore comments?
3. Is there a simpler way to get a count of items that would be returned from a query, similar to dynamoose's query.count() function?

My suggestion would be some type guards like this until a more first class feature becomes available.:

function hasCount(data: unknown): data is { Count: number } {
    return !!data && typeof data === "object" && "Count" in data && typeof data.Count === "number";
}

function toCount(data: unknown): number {
    if (hasCount(data)) {
        return data.Count;
    }
    return 0;
}

async function controlsNotInPlace({ ownerGroup }: WithOwnerGroupParams): Promise<number> {
    const result  = await ControlEntity.query
        .controlsByOwnerGroup({ ownerGroup })
        .where(
            ({ inPlace }, { eq, notExists }) => `
                ${notExists(inPlace)} OR ${eq(inPlace, false)}
            `,
        )
        .go({ 
            data: "raw",
            params: {
                Select: 'COUNT' // <---- note I am adding this to the parameters sent to the request
            }
        });

    return toCount(result.data);
}

4. Is there a simpler way to write the logic in the where caluse?
   Your filters look optimal to me 👍

Hope this Helps! Let me know if you'd like to dig a bit deeper into any of these concepts!

[dbudwin]

@tywalch Thank you, that does work for me! I still don't fully understand the need for ignoreOwnership, I didn't end up needing it in this case, but this is likely due to my inexperience in general with how DynamoDB works (I have a SQL background). I do think providing a built-in count function in ElectroDB would a fantastic feature! I can definitely help test it if it ever comes about with my real-world data too.