Improve documentation templates

.github/workflows/run-tests.yml

@@ -75,3 +75,8 @@ jobs:
         uses: ammaraskar/sphinx-action@master
         with:
           docs-folder: "docs/"
+      - name: Check that docs are up to date
+        run: |
+          cat Dockerfile ci/Dockerfile.docs.template > ci/Dockerfile.docs && \
+          docker build --file=ci/Dockerfile.docs --tag test-docs . && \
+          docker run test-docs

.gitignore

@@ -41,3 +41,4 @@ lib*.pc
 /env/
 /GIT-VERSION-FILE
 /version
+ci/Dockerfile.docs

Makefile

@@ -2,6 +2,7 @@
 # Licensed under the PostgreSQL License.
 
 TOP := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+PGCOPYDB ?= $(TOP)src/bin/pgcopydb/pgcopydb
 
 all: bin ;
 
@@ -21,7 +22,7 @@ maintainer-clean: clean
 docs:
 	$(MAKE) -C docs clean man html
 
-update-docs: install
+update-docs: bin
 	bash ./docs/update-help-messages.sh
 
 test: build
@@ -66,7 +67,7 @@ debsh-qa: deb-qa
 	docker run --rm -it pgcopydb_debian_qa bash
 
 .PHONY: all
-.PHONY: bin clean install docs maintainer-clean
+.PHONY: bin clean install docs maintainer-clean update-docs
 .PHONY: test tests tests/ci tests/*
 .PHONY: deb debsh deb-qa debsh-qa
 .PHONY: GIT-VERSION-FILE

ci/Dockerfile.docs.template

@@ -0,0 +1,24 @@
+# This file requires a copy of the main Dockerfile in the beginning.
+#
+# Run `cat Dockerfile ci/Dockerfile.docs.template > ci/Dockerfile.docs` in the
+# top directory to generate a working Dockerfile.
+#
+# Notice that the "build" image that is not defined in here is used as a base
+# image. Since Docker does not support multi-stage builds with dependencies
+# across different Dockerfiles for now, we need to prepend a copy of the
+# original Dockerfile in the beginning of this file.
+
+FROM --platform=${TARGETPLATFORM} build
+RUN dpkg --add-architecture ${TARGETARCH:-arm64} && apt update \
+  && apt install -qqy --no-install-suggests --no-install-recommends \
+	git \
+	&& apt clean \
+	&& rm -rf /var/lib/apt/lists/*
+
+WORKDIR /usr/src/pgcopydb
+
+ENTRYPOINT make -s -j$(nproc) update-docs && \
+		   git add docs && \
+		   git diff --staged --exit-code || \
+		   (echo "Docs are not up to date, please run 'make update-docs'" && false) && \
+		   echo "Docs are up to date"

docs/include/help.rst

@@ -0,0 +1,80 @@
+::
+
+     pgcopydb
+       clone     Clone an entire database from source to target
+       fork      Clone an entire database from source to target
+       follow    Replay changes from the source database to the target database
+       snapshot  Create and export a snapshot on the source database
+     + compare   Compare source and target databases
+     + copy      Implement the data section of the database copy
+     + dump      Dump database objects from a Postgres instance
+     + restore   Restore database objects into a Postgres instance
+     + list      List database objects from a Postgres instance
+     + stream    Stream changes from the source database
+       ping      Attempt to connect to the source and target instances
+       help      Print help message
+       version   Print pgcopydb version
+
+     pgcopydb compare
+       schema  Compare source and target schema
+       data    Compare source and target data
+
+     pgcopydb copy
+       db           Copy an entire database from source to target
+       roles        Copy the roles from the source instance to the target instance
+       extensions   Copy the extensions from the source instance to the target instance
+       schema       Copy the database schema from source to target
+       data         Copy the data section from source to target
+       table-data   Copy the data from all tables in database from source to target
+       blobs        Copy the blob data from the source database to the target
+       sequences    Copy the current value from all sequences in database from source to target
+       indexes      Create all the indexes found in the source database in the target
+       constraints  Create all the constraints found in the source database in the target
+
+     pgcopydb dump
+       schema     Dump source database schema as custom files in work directory
+       pre-data   Dump source database pre-data schema as custom files in work directory
+       post-data  Dump source database post-data schema as custom files in work directory
+       roles      Dump source database roles as custome file in work directory
+
+     pgcopydb restore
+       schema      Restore a database schema from custom files to target database
+       pre-data    Restore a database pre-data schema from custom file to target database
+       post-data   Restore a database post-data schema from custom file to target database
+       roles       Restore database roles from SQL file to target database
+       parse-list  Parse pg_restore --list output from custom file
+
+     pgcopydb list
+       databases    List databases
+       extensions   List all the source extensions to copy
+       collations   List all the source collations to copy
+       tables       List all the source tables to copy data from
+       table-parts  List a source table copy partitions
+       sequences    List all the source sequences to copy data from
+       indexes      List all the indexes to create again after copying the data
+       depends      List all the dependencies to filter-out
+       schema       List the schema to migrate, formatted in JSON
+       progress     List the progress
+
+     pgcopydb stream
+       setup      Setup source and target systems for logical decoding
+       cleanup    Cleanup source and target systems for logical decoding
+       prefetch   Stream JSON changes from the source database and transform them to SQL
+       catchup    Apply prefetched changes from SQL files to the target database
+       replay     Replay changes from the source to the target database, live
+     + sentinel   Maintain a sentinel table on the source database
+       receive    Stream changes from the source database
+       transform  Transform changes from the source database into SQL commands
+       apply      Apply changes from the source database into the target database
+
+     pgcopydb stream sentinel
+       setup  Setup the sentinel table
+       get    Get the sentinel table values on the source database
+     + set    Maintain a sentinel table on the source database
+
+     pgcopydb stream sentinel set
+       startpos  Set the sentinel start position LSN on the source database
+       endpos    Set the sentinel end position LSN on the source database
+       apply     Set the sentinel apply mode on the source database
+       prefetch  Set the sentinel prefetch mode on the source database
+

docs/include/stream-cleanup.rst

@@ -1,6 +1,6 @@
 ::
 
-   pgcopydb stream cleanup: cleanup source and target systems for logical decoding
+   pgcopydb stream cleanup: Cleanup source and target systems for logical decoding
    usage: pgcopydb stream cleanup 
 
      --source         Postgres URI to the source database

docs/include/stream.rst

@@ -5,7 +5,7 @@
    Available commands:
      pgcopydb stream
        setup      Setup source and target systems for logical decoding
-       cleanup    cleanup source and target systems for logical decoding
+       cleanup    Cleanup source and target systems for logical decoding
        prefetch   Stream JSON changes from the source database and transform them to SQL
        catchup    Apply prefetched changes from SQL files to the target database
        replay     Replay changes from the source to the target database, live

docs/ref/pgcopydb.rst

@@ -34,86 +34,7 @@ pgcopydb help
 
 The ``pgcopydb help`` command lists all the supported sub-commands:
 
-::
-
-   $ pgcopydb help
-    pgcopydb
-      clone     Clone an entire database from source to target
-      fork      Clone an entire database from source to target
-      follow    Replay changes from the source database to the target database
-      snapshot  Create and export a snapshot on the source database
-    + compare   Compare source and target databases
-    + copy      Implement the data section of the database copy
-    + dump      Dump database objects from a Postgres instance
-    + restore   Restore database objects into a Postgres instance
-    + list      List database objects from a Postgres instance
-    + stream    Stream changes from the source database
-      ping      Attempt to connect to the source and target instances
-      help      Print help message
-      version   Print pgcopydb version
-
-    pgcopydb compare
-      schema  Compare source and target schema
-      data    Compare source and target data
-
-    pgcopydb copy
-      db           Copy an entire database from source to target
-      roles        Copy the roles from the source instance to the target instance
-      extensions   Copy the extensions from the source instance to the target instance
-      schema       Copy the database schema from source to target
-      data         Copy the data section from source to target
-      table-data   Copy the data from all tables in database from source to target
-      blobs        Copy the blob data from the source database to the target
-      sequences    Copy the current value from all sequences in database from source to target
-      indexes      Create all the indexes found in the source database in the target
-      constraints  Create all the constraints found in the source database in the target
-
-    pgcopydb dump
-      schema     Dump source database schema as custom files in work directory
-      pre-data   Dump source database pre-data schema as custom files in work directory
-      post-data  Dump source database post-data schema as custom files in work directory
-      roles      Dump source database roles as custome file in work directory
-
-    pgcopydb restore
-      schema      Restore a database schema from custom files to target database
-      pre-data    Restore a database pre-data schema from custom file to target database
-      post-data   Restore a database post-data schema from custom file to target database
-      roles       Restore database roles from SQL file to target database
-      parse-list  Parse pg_restore --list output from custom file
-
-    pgcopydb list
-      databases    List databases
-      extensions   List all the source extensions to copy
-      collations   List all the source collations to copy
-      tables       List all the source tables to copy data from
-      table-parts  List a source table copy partitions
-      sequences    List all the source sequences to copy data from
-      indexes      List all the indexes to create again after copying the data
-      depends      List all the dependencies to filter-out
-      schema       List the schema to migrate, formatted in JSON
-      progress     List the progress
-
-    pgcopydb stream
-      setup      Setup source and target systems for logical decoding
-      cleanup    Cleanup source and target systems for logical decoding
-      prefetch   Stream JSON changes from the source database and transform them to SQL
-      catchup    Apply prefetched changes from SQL files to the target database
-      replay     Replay changes from the source to the target database, live
-    + sentinel   Maintain a sentinel table on the source database
-      receive    Stream changes from the source database
-      transform  Transform changes from the source database into SQL commands
-      apply      Apply changes from the source database into the target database
-
-    pgcopydb stream sentinel
-      setup   Setup the sentinel table
-      get     Get the sentinel table values on the source database
-    + set     Maintain a sentinel table on the source database
-
-    pgcopydb stream sentinel set
-      startpos  Set the sentinel start position LSN on the source database
-      endpos    Set the sentinel end position LSN on the source database
-      apply     Set the sentinel apply mode on the source database
-      prefetch  Set the sentinel prefetch mode on the source database
+.. include:: ../include/help.rst
 
 pgcopydb version
 ----------------

docs/update-help-messages.sh

@@ -9,6 +9,13 @@ set -o pipefail
 SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
 INCLUDE_DIR="$SCRIPT_DIR/include"
 
+# set PGCOPYDB if not already set
+if [ -z "${PGCOPYDB:-}" ]; then
+    PGCOPYDB="${SCRIPT_DIR}/../src/bin/pgcopydb/pgcopydb"
+fi
+
+echo "Updating documentation templates using binary ${PGCOPYDB}"
+
 # Given a single command, print the help text, and wrap the output in rst style
 # code block format in a file under include directory
 function print_help_to_file() {
@@ -18,17 +25,26 @@ function print_help_to_file() {
     local cmd
     cmd=$(echo "$*" | sed -E "s/^ +//")
 
-    # Replace spaces in the file name with dashes We print the output of
-    # `pgcopydb --help` to `pgcopydb.rst`
-    # `pgcopydb compare --help` to `compare.rst`
-    # `pgcopydb compare data --help` to `compare-data.rst`
+    # Replace spaces in the command name with dashes to generate the file name.
+    #
+    # We print the output of
+    #   `pgcopydb --help` to `pgcopydb.rst`
+    #   `pgcopydb compare --help` to `compare.rst`
+    #   `pgcopydb compare data --help` to `compare-data.rst` etc.
+    #
+    #  One exception is the `pgcopydb help` command. We print the output of
+    #  `pgcopydb help` to `help.rst` instead of supplying a `--help` argument to
+    #  the command.
     local file_path
     local help_cmd
     if [ "${cmd}" = "" ]; then
-        help_cmd="pgcopydb --help 2>&1"
+        help_cmd="${PGCOPYDB} --help 2>&1"
         file_path="${INCLUDE_DIR}/pgcopydb.rst"
+    elif [ "${cmd}" = "help" ]; then
+        help_cmd="${PGCOPYDB} help 2>&1"
+        file_path="${INCLUDE_DIR}/help.rst"
     else
-        help_cmd="pgcopydb ${cmd} --help 2>&1"
+        help_cmd="${PGCOPYDB} ${cmd} --help 2>&1"
         file_path="${INCLUDE_DIR}/${cmd// /-}.rst"
     fi
 
@@ -51,6 +67,8 @@ function print_help_to_file() {
 # Parse the output of `pgcopydb help` and call print_help_to_file for each command
 function parse_help_output() {
 
+    local cmd=""
+    local subcmd=""
     # Loop over all the lines of the help text, parse commands and subcommands,
     # and call print_help_to_file for each command.
     #
@@ -115,7 +133,7 @@ function parse_help_output() {
         # print the help message for the subcommand to file
         print_help_to_file "${cmd} ${subcmd}"
 
-    done < <(pgcopydb help 2>&1)
+    done < <(${PGCOPYDB} help 2>&1)
 }
 
 # Delete all the existing help files and recreate them
@@ -124,5 +142,4 @@ parse_help_output
 
 # Remove the help messages for the commands that are not covered in docs
 rm -rf "${INCLUDE_DIR}/fork.rst" \
-       "${INCLUDE_DIR}/help.rst" \
        "${INCLUDE_DIR}/version.rst"

src/bin/pgcopydb/cli_stream.c

@@ -66,7 +66,7 @@ static CommandLine stream_setup_command =
 static CommandLine stream_cleanup_command =
 	make_command(
 		"cleanup",
-		"cleanup source and target systems for logical decoding",
+		"Cleanup source and target systems for logical decoding",
 		"",
 		"  --source         Postgres URI to the source database\n"
 		"  --target         Postgres URI to the target database\n"