feat(voyager): add optimization passes to the queue

dictionary.txt

@@ -564,6 +564,7 @@ iconify
 identifer
 identifing
 idents
+idxs
 iflag
 ilog
 imap

lib/pg-queue/migrations/20240316012044_queue.sql

@@ -1,23 +1,32 @@
-CREATE TABLE queue (
+CREATE TYPE status AS ENUM(
+    'ready',
+    'done',
+    'optimize',
+    'failed'
+);
+
+CREATE TABLE queue(
     id BIGSERIAL,
-    status status NOT NULL DEFAULT 'ready',
+    status STATUS NOT NULL DEFAULT 'ready',
     item JSONB NOT NULL,
     -- Can't have foreign key relations to hypertables, so recreate the constraints as best as possible
-    parent BIGINT DEFAULT NULL CHECK (parent IS NULL OR parent > 0),
-    -- Error message in case of permanent failure. If set, status MUST be 'failed'.
-    message TEXT CHECK (((message IS NULL) AND (status != 'failed'::status)) OR ((message IS NOT NULL) AND (status = 'failed'::status))),
-    created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+    parents BIGINT[] DEFAULT '{}' CHECK (0 < ALL (parents)),
+    message TEXT CHECK ((message IS NULL AND status != 'failed'::status) OR (message IS NOT NULL AND status = 'failed'::status)),
+    created_at timestamptz NOT NULL DEFAULT now(),
     PRIMARY KEY (id, created_at)
 );
 
-CREATE INDEX index_queue_id ON queue (id);
+CREATE INDEX index_queue_id ON queue(id);
 
-CREATE INDEX index_queue_status_id ON queue (status, id);
+CREATE INDEX index_queue_status_id ON queue(status, id);
 
-SELECT create_hypertable('queue', 'created_at');
+SELECT
+    create_hypertable('queue', 'created_at');
 
-SELECT add_retention_policy('queue', INTERVAL '60 days');
+SELECT
+    add_retention_policy('queue', INTERVAL '60 days');
 
 ALTER TABLE queue SET (timescaledb.compress, timescaledb.compress_orderby = 'created_at DESC', timescaledb.compress_segmentby = 'id');
 
-SELECT add_compression_policy('queue', compress_after => INTERVAL '14 days');
+SELECT
+    add_compression_policy('queue', compress_after => INTERVAL '14 days');

lib/pg-queue/migrations/20240316025506_get_list.sql

@@ -1,28 +1,28 @@
-CREATE OR REPLACE FUNCTION get_list(
-  root_id bigint,
-  max_depth int
-) RETURNS TABLE(id bigint, parent bigint, item jsonb, depth integer)
-  AS $$
-    WITH RECURSIVE cte (id_, parent_, item_, depth_) AS
-    (
-      SELECT
-        id, parent, item, 1 as depth
-      FROM
-        queue
-      WHERE
-        queue.id = root_id
-        AND
-          queue.parent IS NOT NULL
-      UNION
-        SELECT 
-          c.parent_, t.parent, t.item, c.depth_::integer + 1
-        FROM
-          cte c
-        JOIN queue t
-        ON c.parent_ = t.id
-        WHERE
-          depth_ < max_depth
-    )
-    SELECT * FROM cte;
-  $$
-  LANGUAGE SQL;
+-- CREATE OR REPLACE FUNCTION get_list(
+--   root_id bigint,
+--   max_depth int
+-- ) RETURNS TABLE(id bigint, parent bigint, item jsonb, depth integer)
+--   AS $$
+--     WITH RECURSIVE cte (id_, parent_, item_, depth_) AS
+--     (
+--       SELECT
+--         id, parent, item, 1 as depth
+--       FROM
+--         queue
+--       WHERE
+--         queue.id = root_id
+--         AND
+--           queue.parent IS NOT NULL
+--       UNION
+--         SELECT 
+--           c.parent_, t.parent, t.item, c.depth_::integer + 1
+--         FROM
+--           cte c
+--         JOIN queue t
+--         ON c.parent_ = t.id
+--         WHERE
+--           depth_ < max_depth
+--     )
+--     SELECT * FROM cte;
+--   $$
+--   LANGUAGE SQL;