Add PostgreSQL support (#17)

README.md

@@ -258,15 +258,46 @@ impl PriceFeed for Kraken {
     //snip
 }
 ```
+# Persistence
+
+
+
+Sibyls supports two backends for data persistence:
+
+1. [Sled](https://sled.rs/) - A modern, high-performance embedded database that offers a simple and efficient way to store and manage data locally.
+
+2. [PostgreSQL](https://www.postgresql.org/) - A powerful, open-source relational database system that provides robust features and scalability for more complex data storage needs.
+
+These options allow Sibyls to be flexible and adaptable, catering to the diverse needs of its users.
+
+
+### Sled
+
+Sled is an embedded database that stores data on the local file system. 
+The Sibyls database backend stores data in `events/{AssetPair}` (eg. `events/BTCUSD`). 
+Sled is the default database, no additional configurations are needed to use it. 
+The user can also enable it explicitly by using the command line argument: `--database_backend sled`.
+
+### PostgreSQL
+
+PostgreSQL is a full futured RDBMS, it can be used in enterprise settings. 
+To use PostgreSQL as the database, the user will need to use the command line argument `--database_backend pg` and `--database_url postgres://user:password@database_host/database_name`.
+The `DATABASE_URL` environment variable can also be used to set the database URL.
+
+### Dual Database Backend
+
+The dual database backend is useful for transitioning from Sled to PostgreSQL.
+If data is stored in PostgreSQL, it reads from the PostgreSQL backend. 
+If not, data is read from the Sled backend and added to the PostgreSQL backend.
+To enable it, the user must include the `--database_backend dual` parameter 
+and set the database URL as it is done for PostgreSQL.
 
 ## Run Sibyls
 
 If you are running Sibyls, or want to run Sibyls and need help, please email us. You can find our contact info at lava.xyz. 
 
 # TODO
 The following todos are in decreasing priority.
-## Persistence
-Currently, the Sybils database is on instance only. It should also support using an external database such as Postgres.
 ## Key Handling
 Additional functionality can be added to make working with the key easier. 
 ### Encryption at Rest

sibyls/Cargo.toml

@@ -36,6 +36,7 @@ sled = "0.34"
 thiserror = "1.0.31"
 time = { version = "0.3.9", features = ["formatting", "serde-human-readable"] }
 tokio = { version = "1.18.2", features = ["full"] }
+diesel = { version = "2.2.0", features = ["postgres", "serde_json", "time", "r2d2"] }
 
 [dev-dependencies]
 dlc = "~0.4.0"

sibyls/diesel.toml

@@ -0,0 +1,9 @@
+# For documentation on how to configure this file,
+# see https://diesel.rs/guides/configuring-diesel-cli
+
+[print_schema]
+file = "src/schema.rs"
+custom_type_derives = ["diesel::query_builder::QueryId", "Clone"]
+
+[migrations_directory]
+dir = "migrations"

sibyls/migrations/00000000000000_diesel_initial_setup/down.sql

@@ -0,0 +1,6 @@
+-- This file was automatically created by Diesel to setup helper functions
+-- and other internal bookkeeping. This file is safe to edit, any future
+-- changes will be added to existing projects as new migrations.
+
+DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass);
+DROP FUNCTION IF EXISTS diesel_set_updated_at();

sibyls/migrations/00000000000000_diesel_initial_setup/up.sql

@@ -0,0 +1,36 @@
+-- This file was automatically created by Diesel to setup helper functions
+-- and other internal bookkeeping. This file is safe to edit, any future
+-- changes will be added to existing projects as new migrations.
+
+
+
+
+-- Sets up a trigger for the given table to automatically set a column called
+-- `updated_at` whenever the row is modified (unless `updated_at` was included
+-- in the modified columns)
+--
+-- # Example
+--
+-- ```sql
+-- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW());
+--
+-- SELECT diesel_manage_updated_at('users');
+-- ```
+CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$
+BEGIN
+    EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s
+                    FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl);
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$
+BEGIN
+    IF (
+        NEW IS DISTINCT FROM OLD AND
+        NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at
+    ) THEN
+        NEW.updated_at := current_timestamp;
+    END IF;
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;

sibyls/migrations/2024-07-01-232220_create_events/down.sql

@@ -0,0 +1 @@
+drop table events

sibyls/migrations/2024-07-01-232220_create_events/up.sql

@@ -0,0 +1,9 @@
+CREATE TABLE events (
+    maturation TIMESTAMPTZ NOT NULL,
+    asset_pair VARCHAR NOT NULL,
+    announcement TEXT NOT NULL,
+    outstanding_sk_nonces TEXT,
+    attestation TEXT,
+    price BIGINT,
+    PRIMARY KEY (maturation, asset_pair)
+)

sibyls/src/common.rs

@@ -1,8 +1,13 @@
+use crate::error::SibylsError;
+use clap::ValueEnum;
 use dlc_messages::oracle_msgs::EventDescriptor::{DigitDecompositionEvent, EnumEvent};
-use dlc_messages::oracle_msgs::{DigitDecompositionEventDescriptor, EventDescriptor};
+use dlc_messages::oracle_msgs::{
+    DigitDecompositionEventDescriptor, EventDescriptor, OracleAnnouncement, OracleAttestation,
+};
 use serde::{Deserialize, Serialize};
 use std::fmt::{self, Debug, Display, Formatter};
-use time::{serde::format_description, Duration, Time};
+use std::str::FromStr;
+use time::{serde::format_description, Duration, OffsetDateTime, Time};
 
 use crate::oracle::pricefeeds::FeedId;
 
@@ -12,6 +17,27 @@ pub enum AssetPair {
     BTCUSDT,
 }
 
+impl FromStr for AssetPair {
+    type Err = SibylsError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        if s == "BTCUSD" {
+            Ok(AssetPair::BTCUSD)
+        } else if s == "BTCUSDT" {
+            Ok(AssetPair::BTCUSDT)
+        } else {
+            Err(SibylsError::UnknownAssetPairError(s.to_string()))
+        }
+    }
+}
+
+#[derive(Debug, Clone, ValueEnum)]
+pub enum DatabaseBackend {
+    Sled,
+    Pg,
+    Dual,
+}
+
 #[derive(Clone, Debug, Deserialize)]
 pub struct SerializableEventDescriptor {
     pub base: u16,
@@ -68,6 +94,43 @@ impl Display for AssetPair {
     }
 }
 
+#[derive(PartialEq, Debug, Clone)]
+pub struct OracleEvent {
+    pub asset_pair: AssetPair,
+    pub maturation: OffsetDateTime,
+    pub(crate) outstanding_sk_nonces: Option<Vec<[u8; 32]>>,
+    pub announcement: OracleAnnouncement,
+    pub attestation: Option<OracleAttestation>,
+    pub outcome: Option<u64>,
+}
+
+pub const PAGE_SIZE: u32 = 100;
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum SortOrder {
+    Insertion,
+    ReverseInsertion,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(default, rename_all = "camelCase")]
+pub struct Filters {
+    pub sort_by: SortOrder,
+    pub page: u32,
+    pub asset_pair: AssetPair,
+}
+
+impl Default for Filters {
+    fn default() -> Self {
+        Filters {
+            sort_by: SortOrder::ReverseInsertion,
+            page: 0,
+            asset_pair: AssetPair::BTCUSD,
+        }
+    }
+}
+
 format_description!(standard_time, Time, "[hour]:[minute]");
 
 mod standard_duration {