✨ Maintenance/enhance consistency checks

scripts/maintenance/check_consistency_data.py

@@ -275,11 +275,12 @@ async def main_async(
     # ---------------------- COMPARISON ---------------------------------------------------------------------
     db_file_uuids = {db_file_uuid for db_file_uuid, _, _ in db_file_entries}
     s3_file_uuids = {s3_file_uuid for s3_file_uuid, _, _ in s3_file_entries}
-    common_files = db_file_uuids.intersection(s3_file_uuids)
+    common_files_uuids = db_file_uuids.intersection(s3_file_uuids)
     s3_missing_files_uuids = db_file_uuids.difference(s3_file_entries)
     db_missing_files_uuids = s3_file_uuids.difference(db_file_uuids)
     typer.secho(
-        f"{len(common_files)} files are the same in both system", fg=typer.colors.BLUE
+        f"{len(common_files_uuids)} files are the same in both system",
+        fg=typer.colors.BLUE,
     )
     typer.secho(
         f"{len(s3_missing_files_uuids)} files are missing in S3", fg=typer.colors.RED
@@ -289,6 +290,7 @@ async def main_async(
     )
 
     # ------------------ WRITING REPORT --------------------------------------------
+    consistent_files_path = Path.cwd() / "consistent_files.csv"
     s3_missing_files_path = Path.cwd() / "s3_missing_files.csv"
     db_missing_files_path = Path.cwd() / "db_missing_files.csv"
     db_file_map: Dict[str, Tuple[int, datetime]] = {
@@ -314,12 +316,13 @@ def order_by_owner(
 
     def write_to_file(path: Path, files_by_owner):
         with path.open("wt") as fp:
-            fp.write(f"owner,name,email,file,size,last_modified\n")
+            fp.write("owner,name,email,file,size,last_modified\n")
             for (owner, name, email), files in files_by_owner.items():
                 for file in files:
                     size, modified = db_file_map.get(file, ("?", "?"))
                     fp.write(f"{owner},{name},{email},{file}, {size}, {modified}\n")
 
+    write_to_file(consistent_files_path, order_by_owner(common_files_uuids))
     write_to_file(s3_missing_files_path, order_by_owner(s3_missing_files_uuids))
     write_to_file(db_missing_files_path, order_by_owner(db_missing_files_uuids))
 

scripts/maintenance/consistency/dump-restore.md

@@ -0,0 +1,31 @@
+# dump the database
+
+This command will dump a database to the file ```dump.sql```
+
+```bash
+pg_dump --host POSTGRESHOST --username POSTGRES_USER --format c --blobs --verbose --file dump.sql POSTGRESDB
+```
+
+# restore the database locally
+
+First a postgres instance must be setup, and the following will allow saving its contents into a docker-volume name POSTGRES_DATA_VOLUME
+
+```bash
+export POSTGRES_DATA_VOLUME=aws-production-simcore_postgres_data; docker-compose up
+```
+
+This allows connecting to the local postgres instance
+```bash
+psql -h 127.0.0.1 -p 5432 -U test
+```
+
+
+```sql
+CREATE DATABASE simcoredb;
+CREATE USER postgres_osparc WITH PASSWORD 'test';
+GRANT ALL PRIVILEGES ON DATABASE simcoredb to postgres_osparc;
+```
+
+```bash
+pg_restore --host 127.0.0.1 --port 5432 --username postgres_osparc -d simcoredb dump.sql
+```