Graphql poc

.pre-commit-config.yaml

@@ -27,6 +27,7 @@ repos:
 
     -   id: pretty-format-json
         args: [--autofix]
+        exclude: ^src/web_interface/static/package-lock.json
 
 -   repo: https://github.com/jumanjihouse/pre-commit-hooks
     rev: 2.1.5

docsrc/conf.py

@@ -77,7 +77,10 @@
     'flask_restx',
     'flask_security',
     'flask_sqlalchemy',
+    'gql',
+    'graphql',
     'itsdangerous',
+    'json5',
     'lief',
     'magic',
     'markupsafe',

src/config.py

@@ -108,6 +108,13 @@ class Authentication(BaseModel):
 
     radare2_url: str
 
+    hasura: Frontend.Hasura
+
+    class Hasura(BaseModel):
+        model_config = ConfigDict(extra='forbid')
+        admin_secret: str
+        port: int = 33_333
+
 
 class Backend(Common):
     model_config = ConfigDict(extra='forbid')

src/config/fact-core-config.toml

@@ -169,3 +169,7 @@ radare2-url = "http://localhost:8000"
 enabled = false
 user-database = "sqlite:////media/data/fact_auth_data/fact_users.db"
 password-salt = "5up3r5tr0n6_p455w0rd_5417"
+
+
+[frontend.hasura]
+admin-secret = "4dM1n_S3cR3T_changemeplz"

src/conftest.py

@@ -144,6 +144,8 @@ def frontend_config(request, common_config) -> config.Frontend:
             'user_database': 'sqlite:////media/data/fact_auth_data/fact_users.db',
             'password_salt': '5up3r5tr0n6_p455w0rd_5417',
         },
+        # we need the actual secret to set up the test configuration
+        'hasura': {'admin_secret': config.frontend.hasura.admin_secret},
     }
 
     test_config.update(common_config.model_dump())

src/install.py

@@ -73,6 +73,9 @@ def _setup_argparser():
     install_options.add_argument(
         '-R', '--no_radare', action='store_true', default=False, help='do not install radare view container'
     )
+    install_options.add_argument(
+        '-H', '--no-hasura', action='store_true', default=False, help='do not set up hasura for GraphQL'
+    )
     install_options.add_argument(
         '-U',
         '--statistic_cronjob',
@@ -192,10 +195,10 @@ def install():
 def install_fact_components(args, distribution, none_chosen, skip_docker):
     if (args.common or args.frontend or args.backend or none_chosen) and not args.no_common:
         common(distribution)
-    if args.frontend or none_chosen:
-        frontend(skip_docker, not args.no_radare, args.nginx, distribution)
     if args.db or none_chosen:
         db()
+    if args.frontend or none_chosen:
+        frontend(skip_docker, not args.no_radare, args.nginx, distribution, args.no_hasura)
     if args.backend or none_chosen:
         backend(skip_docker, distribution)
 

src/install/db.py

@@ -33,9 +33,15 @@ def install_postgres(version: int = 14):
         if process.returncode != 0:
             raise InstallationError(f'Failed to set up PostgreSQL: {process.stderr}')
 
-    # increase the maximum number of concurrent connections (and restart for the change to take effect)
+
+def configure_postgres(version: int = 14):
     config_path = f'/etc/postgresql/{version}/main/postgresql.conf'
+    # increase the maximum number of concurrent connections
     run(f'sudo sed -i -E "s/max_connections = [0-9]+/max_connections = 999/g" {config_path}', shell=True, check=True)
+    hba_config_path = f'/etc/postgresql/{version}/main/pg_hba.conf'
+    # change UNIX domain socket auth mode from peer to user/pw
+    run(f'sudo sed -i -E "s/(local +all +all +)peer/\\1scram-sha-256/g" {hba_config_path}', shell=True, check=True)
+    # restart for the changes to take effect
     run('sudo service postgresql restart', shell=True, check=True)
 
 
@@ -53,6 +59,7 @@ def main():
     else:
         logging.info('Setting up PostgreSQL database')
         install_postgres()
+    configure_postgres()
 
     # initializing DB
     logging.info('Initializing PostgreSQL database')

src/install/frontend.py

@@ -16,6 +16,7 @@
     read_package_list_from_file,
     run_cmd_with_logging,
 )
+from storage.graphql.util import get_env
 
 DEFAULT_CERT = '.\n.\n.\n.\n.\nexample.com\n.\n\n\n'
 INSTALL_DIR = Path(__file__).parent
@@ -102,7 +103,8 @@ def _configure_nginx():
             # copy is better on redhat to respect selinux context
             '(cd ../config && sudo install -m 644 $PWD/nginx.conf /etc/nginx/nginx.conf)',
             '(sudo mkdir /etc/nginx/error || true)',
-            '(cd ../web_interface/templates/ && sudo ln -s $PWD/maintenance.html /etc/nginx/error/maintenance.html) || true',  # noqa: E501
+            '(cd ../web_interface/templates/ '
+            '&& sudo ln -s $PWD/maintenance.html /etc/nginx/error/maintenance.html) || true',
         ],
         error='configuring nginx',
     )
@@ -141,7 +143,13 @@ def _copy_mime_icons():
         run_cmd_with_logging(f'cp -rL {ICON_THEME_INSTALL_PATH / source} {MIME_ICON_DIR / target}')
 
 
-def main(skip_docker, radare, nginx, distribution):
+def _init_hasura():
+    with OperateInDirectory(INSTALL_DIR.parent / 'storage' / 'graphql' / 'hasura'):
+        run_cmd_with_logging('docker compose up -d', env=get_env())
+        run_cmd_with_logging('python3 init_hasura.py')
+
+
+def main(skip_docker, radare, nginx, distribution, skip_hasura):
     if distribution != 'fedora':
         pkgs = read_package_list_from_file(INSTALL_DIR / 'apt-pkgs-frontend.txt')
         apt_install_packages(*pkgs)
@@ -170,6 +178,9 @@ def main(skip_docker, radare, nginx, distribution):
     if not skip_docker:
         _install_docker_images(radare)
 
+    if not skip_hasura:
+        _init_hasura()
+
     if not MIME_ICON_DIR.is_dir():
         MIME_ICON_DIR.mkdir()
         _copy_mime_icons()

src/install/requirements_backend.txt

@@ -1,5 +1,4 @@
 cryptography==42.0.4
-docker~=7.1.0
 MarkupSafe==2.1.1
 networkx==2.6.3
 Pillow==10.3.0

src/install/requirements_common.txt

@@ -3,6 +3,7 @@ testresources==2.0.1
 
 # General python dependencies
 appdirs==1.4.4
+docker~=7.1.0
 flaky==3.7.0
 lief==0.12.3
 psutil==5.9.4

src/install/requirements_frontend.txt

@@ -7,6 +7,7 @@ flask-wtf~=1.2.1
 flask~=3.0.3
 flask-restx~=1.3.0
 flask-sqlalchemy~=3.1.1
+gql~=3.5.0
 itsdangerous~=2.2.0
 matplotlib~=3.7.5
 more-itertools~=10.2.0

src/storage/db_connection.py

@@ -17,6 +17,9 @@ def __init__(
         self.base = Base
 
         address = config.common.postgres.server
+        if address in ('localhost', '127.0.0.1', '::1'):
+            # local postgres => connect through UNIX domain socket (faster than TCP)
+            address = '/var/run/postgresql'
         port = config.common.postgres.port
         user = getattr(config.common.postgres, user)
         password = getattr(config.common.postgres, password)

src/storage/graphql/hasura/docker-compose.base.yml

@@ -0,0 +1,5 @@
+  services:
+    postgres-local:
+      volumes:
+        - /var/run/postgresql:/var/run/postgresql
+    postgres-remote:

src/storage/graphql/hasura/docker-compose.yaml

@@ -0,0 +1,50 @@
+services:
+  # the postgres instance is only for Hasura to store its metadata and should not be available from outside
+  postgres:
+    image: postgres:15
+    restart: always
+    volumes:
+      - db_data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_PASSWORD: postgrespassword
+  graphql-engine:
+    image: hasura/graphql-engine:v2.38.0
+    ports:
+      - "${HASURA_PORT}:8080"
+    restart: always
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    environment:
+      HASURA_GRAPHQL_METADATA_DATABASE_URL: postgres://postgres:postgrespassword@postgres:5432/postgres
+      PG_DATABASE_URL: postgres://postgres:postgrespassword@postgres:5432/postgres
+      HASURA_GRAPHQL_ENABLE_CONSOLE: "true"
+      HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log
+      HASURA_GRAPHQL_CONSOLE_ASSETS_DIR: /srv/console-assets
+      HASURA_GRAPHQL_METADATA_DEFAULTS: '{"backend_configs":{"dataconnector":{"athena":{"uri":"http://data-connector-agent:8081/api/v1/athena"},"mariadb":{"uri":"http://data-connector-agent:8081/api/v1/mariadb"},"mysql8":{"uri":"http://data-connector-agent:8081/api/v1/mysql"},"oracle":{"uri":"http://data-connector-agent:8081/api/v1/oracle"},"snowflake":{"uri":"http://data-connector-agent:8081/api/v1/snowflake"}}}}'
+      # should be set during init
+      FACT_DB_URL: "${FACT_DB_URL}"
+      HASURA_GRAPHQL_ADMIN_SECRET: "${HASURA_ADMIN_SECRET}"
+      HASURA_GRAPHQL_UNAUTHORIZED_ROLE: "ro_user"
+    depends_on:
+      data-connector-agent:
+        condition: service_healthy
+    extends:
+      # this is kind of a hack to make mounting a volume optional, see https://github.com/docker/compose/issues/3979
+      file: docker-compose.base.yml
+      service: "postgres-${DB_LOCALITY:-local}"
+  data-connector-agent:
+    image: hasura/graphql-data-connector:v2.38.0
+    restart: always
+    ports:
+      - "8081:8081"
+    environment:
+      QUARKUS_LOG_LEVEL: ERROR
+      QUARKUS_OPENTELEMETRY_ENABLED: "false"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8081/api/v1/athena/health"]
+      interval: 5s
+      timeout: 10s
+      retries: 5
+      start_period: 5s
+volumes:
+  db_data: