From a501469a8893fc6226dcd4556ce1a905034629fe Mon Sep 17 00:00:00 2001 From: NathanFlurry Date: Tue, 17 Sep 2024 04:42:42 +0000 Subject: [PATCH] docs: add backend internal docs (#400) --- :w | 280 ------------------ docs/backend/comparison/nakama.mdx | 70 +++++ docs/backend/comparison/playfab.mdx | 93 ++++++ docs/backend/comparison/pragma.mdx | 27 ++ docs/backend/comparison/supabase.mdx | 99 +++++++ docs/backend/conventions.mdx | 19 ++ docs/backend/design/deno.mdx | 31 ++ docs/backend/design/drizzle.mdx | 55 ++++ docs/backend/design/misc.mdx | 12 + docs/backend/design/postgres.mdx | 72 +++++ .../design/registries-dependencies.mdx | 10 + docs/backend/design/typescript.mdx | 73 +++++ docs/backend/introduction.mdx | 7 + 13 files changed, 568 insertions(+), 280 deletions(-) delete mode 100644 :w create mode 100644 docs/backend/comparison/nakama.mdx create mode 100644 docs/backend/comparison/playfab.mdx create mode 100644 docs/backend/comparison/pragma.mdx create mode 100644 docs/backend/comparison/supabase.mdx create mode 100644 docs/backend/conventions.mdx create mode 100644 docs/backend/design/deno.mdx create mode 100644 docs/backend/design/drizzle.mdx create mode 100644 docs/backend/design/misc.mdx create mode 100644 docs/backend/design/postgres.mdx create mode 100644 docs/backend/design/registries-dependencies.mdx create mode 100644 docs/backend/design/typescript.mdx create mode 100644 docs/backend/introduction.mdx diff --git a/:w b/:w deleted file mode 100644 index 261fe092..00000000 --- a/:w +++ /dev/null @@ -1,280 +0,0 @@ -use anyhow::*; -use futures_util::{StreamExt, TryStreamExt}; -use rivet_api::{apis, models}; -use serde::Deserialize; -use std::{ - collections::HashMap, - path::{Path, PathBuf}, - sync::Arc, -}; -use tokio::fs; - -use crate::{ - backend, config, - game::TEMPEnvironment, - paths, - toolchain_ctx::ToolchainCtx, - util::{net::upload, task, term}, -}; - -pub struct DeployOpts { - pub env: TEMPEnvironment, - - /// The location of the project. - pub project_path: String, - - /// Skip the migration step. - pub skip_migrate: bool, -} - -pub async fn deploy(ctx: &ToolchainCtx, task: task::TaskCtx, opts: DeployOpts) -> Result<()> { - task.log("[Deploying Backend]"); - - let backend = backend::get_or_create_backend(ctx, opts.env.id).await?; - let game_id_str = ctx.game_id.to_string(); - let env_id_str = opts.env.id.to_string(); - let project_path = PathBuf::from(opts.project_path.clone()); - - // Build - task.log(format!("[Building Project] {}", project_path.display())); - let (cmd_env, config_path) = config::settings::try_read(&paths::data_dir()?, |settings| { - let mut env = settings.backend.command_environment.clone(); - env.extend(settings.backend.deploy.command_environment.clone()); - Ok((env, settings.backend.deploy.config_path.clone())) - }) - .await?; - let cmd = backend::run_backend_command_from_task( - task.clone(), - backend::BackendCommandOpts { - command: "build", - opts: serde_json::json!({ - "watch": false, - "runtime": "cloudflare_workers_platforms", - "outputFormat": "bundled", - "dbDriver": "neon_serverless", - "migrate": false, - "project": config_path, - }), - env: cmd_env, - }, - ) - .await?; - ensure!(cmd == 0, "Failed to build backend project"); - - backend::database::provision_database(task.clone(), ctx, opts.env.id).await?; - - let db_url = config::meta::try_read_project(&paths::data_dir()?, |config| { - let env_config = config - .environments - .get(&opts.env.id) - .context("could not find environment")?; - Ok(env_config.backend.db_url.clone()) - }) - .await?; - - if !opts.skip_migrate { - task.log("[Migrating Database]"); - - // Migrate - let mut migrate_env = HashMap::new(); - migrate_env.insert( - "DATABASE_URL".to_string(), - db_url.context("no db url for env")?, - ); - - let migrate_cmd = backend::run_backend_command_from_task( - task.clone(), - backend::BackendCommandOpts { - command: "dbMigrateApply", - opts: serde_json::json!({ - "project": config_path, - }), - env: migrate_env, - }, - ) - .await?; - ensure!(migrate_cmd == 0, "Failed to migrate databases"); - } - - // Read files for upload - let gen_manifest = read_generated_manifest(&project_path).await?; - let bundle_path = project_path.join(gen_manifest.bundle); - let wasm_path = gen_manifest.wasm.map(|x| project_path.join(x)); - let mut files = vec![upload::prepare_upload_file( - &bundle_path, - "bundle.js", - fs::metadata(&bundle_path).await?, - )?]; - if let Some(wasm) = wasm_path.as_ref() { - files.push(upload::prepare_upload_file( - wasm, - "query-engine.wasm", - fs::metadata(wasm).await?, - )?); - } - let total_len = files - .iter() - .fold(0, |acc, x| acc + x.prepared.content_length); - - task.log(format!( - "[Uploading Environment] {name} ({count} files, {size} total)", - name = &opts.env.name, - count = files.len(), - size = upload::format_file_size(total_len as u64)?, - )); - - task.log(format!("[Fetching Environment Variables]")); - // let variables = apis::ee_backend_api::ee_backend_get_variables( - // &ctx.openapi_config_cloud, - // &game_id_str, - // &env_id_str, - // ) - // .await? - // .variables; - let mut update_variables = HashMap::::new(); - // if !variables.contains_key("RIVET_BACKEND_PUBLIC_ENDPOINT") { - update_variables.insert( - "RIVET_BACKEND_PUBLIC_ENDPOINT".to_string(), - models::EeBackendUpdateVariable { - text: Some(backend.endpoint.clone()), - ..Default::default() - }, - ); - // } - // if !variables.contains_key("RIVET_API_ENDPOINT") { - update_variables.insert( - "RIVET_API_ENDPOINT".to_string(), - models::EeBackendUpdateVariable { - text: Some(ctx.api_endpoint.clone()), - ..Default::default() - }, - ); - // } - // if !variables.contains_key("RIVET_GAME_ID") { - update_variables.insert( - "RIVET_GAME_ID".to_string(), - models::EeBackendUpdateVariable { - text: Some(game_id_str.clone()), - ..Default::default() - }, - ); - // } - // if !variables.contains_key("RIVET_ENVIRONMENT_ID") { - update_variables.insert( - "RIVET_ENVIRONMENT_ID".to_string(), - models::EeBackendUpdateVariable { - text: Some(env_id_str.clone()), - ..Default::default() - }, - ); - // } - // if !variables.contains_key("RIVET_SERVICE_TOKEN") { - task.log(format!("[Creating Service Token]")); - let service_token = - apis::games_environments_tokens_api::games_environments_tokens_create_service_token( - &ctx.openapi_config_cloud, - &game_id_str, - &opts.env.id.to_string(), - ) - .await?; - update_variables.insert( - "RIVET_SERVICE_TOKEN".to_string(), - models::EeBackendUpdateVariable { - secret: Some(service_token.token), - ..Default::default() - }, - ); - // } - if !update_variables.is_empty() { - task.log(format!( - "[Updating Variables] {}", - update_variables - .keys() - .cloned() - .collect::>() - .join(", ") - )); - } - - let prepare_res = apis::ee_backend_api::ee_backend_prepare_deploy( - &ctx.openapi_config_cloud, - &game_id_str, - &env_id_str, - models::EeBackendPrepareDeployRequest { - files: files.iter().map(|f| f.prepared.clone()).collect(), - }, - ) - .await - .context("failed to prepare deploy")?; - - // Upload files - let reqwest_client = Arc::new(reqwest::Client::new()); - let pb = term::EitherProgressBar::Multi(term::multi_progress_bar(task.clone())); - - futures_util::stream::iter(prepare_res.presigned_requests) - .map(Ok) - .try_for_each_concurrent(8, |presigned_req| { - let task = task.clone(); - let pb = pb.clone(); - let files = files.clone(); - let reqwest_client = reqwest_client.clone(); - - async move { - // Find the matching prepared file - let file = files - .iter() - .find(|f| f.prepared.path == presigned_req.path) - .context("missing prepared file")?; - - upload::upload_file( - task.clone(), - &reqwest_client, - &presigned_req, - &file.absolute_path, - file.prepared.content_type.as_ref(), - pb, - ) - .await?; - - Result::<()>::Ok(()) - } - }) - .await?; - - task.log(format!("[Deploying Environment] {}", opts.env.name)); - - let deploy_res = apis::ee_backend_api::ee_backend_deploy( - &ctx.openapi_config_cloud, - &game_id_str, - &env_id_str, - models::EeBackendDeployRequest { - upload_id: prepare_res.upload_id, - variables: Some(update_variables), - }, - ) - .await?; - - task.log(format!( - "[Done] Backend API available at {}", - deploy_res.url - )); - - Ok(()) -} - -#[derive(Deserialize)] -struct GenManifest { - bundle: String, - wasm: Option, -} - -async fn read_generated_manifest(project_path: &Path) -> Result { - let manifest_str = fs::read_to_string( - project_path - .join(".rivet") - .join("backend") - .join("output_manifest.json"), - ) - .await?; - Ok(serde_json::from_str::(&manifest_str)?) -} diff --git a/docs/backend/comparison/nakama.mdx b/docs/backend/comparison/nakama.mdx new file mode 100644 index 00000000..89948f51 --- /dev/null +++ b/docs/backend/comparison/nakama.mdx @@ -0,0 +1,70 @@ +--- +title: Nakama +--- + + + Please read Open Game Backend's [roadmap](/docs/roadmap) for known limitations. + + +[Nakama](https://heroiclabs.com/nakama/) was founded in 2014 and is still an independent company. + +## What Nakama does well + +- Open source (Apache 2.0) +- LiveOps features provided by Satori (closed-source) +- Written in Go +- Focus on real-time multiplayer games +- Provides scripting in TypeScript, Go, and Lua + +## How Supabase and Open Game Backend are similar + +- Open-source (Apache 2.0) +- Self-hostable on any cloud provider + +## How Nakama and Open Game Backend differ + +### Prefab backend vs modular backend + +**Nakama** + +Nakama provides a predefined set of features out of the box. While you can fork Nakama to extend it with your own features, it has a high learning curve & requires you to run your own servers. + +Nakama encourages developers to [use scripting to create RPCs and hooks](https://heroiclabs.com/docs/nakama/server-framework/introduction/index.html) to extend the server. While scripting support is extensive, it's still constrainted to Nakama's limitations. + +This means Nakama has first-class services written in Go and asks you to write second-class code written in a scripting language without access to a mature database. + +**Open Game Backend** + +Open Game Backend is designed from the ground up to be modular at its core. It provides a set of official modules to get you started quickly, but the primary use case is to extend the server with your own modules. All modules are first-class citizens have access to a mature database. + +### Scripting persistence + +**Nakama** + +Nakama provides persistence through its [storage engine](https://heroiclabs.com/docs/nakama/concepts/storage/) built on top of CockroachDB. The storage engine is a document store similar that stores JSON objects along with access controls. It also provides a well-designed mechanism for indexing data that prevents slow queries. The engine supports atomic & conditional writes, but does not support full transactions. + +**Open Game Backend** + +Open Game Backend modules are provided the full flexibility of PostgreSQL, including transactions, complex queries, and indexing. Unlike the Nakama storage engine, PostgreSQL enforces a strict schema on your data which makes it easier to assert the correctness of your data. PostgreSQL provides a [large collection of extensions](https://neon.tech/docs/extensions/pg-extensions) for managing many complex data types. PostgreSQL also integrates natively with many existing tools, such as SQL clients, BI tools, and data warehousing tools which are important when your game grows. + +### Cost + +**Nakama** + +Nakama provides a managed cloud [starting at {"$"}600 per month](https://heroiclabs.com/pricing/). However, running a high-availability cluter starts at $1,000 per month, which is required for any production deployment. Nakama's traditional server architecture requires beefy machines and doesn't scale to meet demand easily. Nakama can be self-hosted on your own server if you're willing to manage it yourself. + +**Open Game Backend** + +Open Game Backend can be hosted on serverless JS platforms (Cloudflare Workers, Deno Deploy) which can cost as little as penies per month and blitz scale to meet demand as needed. Open Game Backend can also be self-hosted on traditional web servers. + +### Go vs TypeScript + +**Nakama** + +Nakama is written in Go, while Open Game Backend is written in TypeScript. Both projects have different goals which lead to this choice. + +Go is a mature language for building high-performance servers. Nakama has a focus on running multiplayer servers within the server, so Go's performance is important. However, it's not as easy to learn as TypeScript. + +**Open Game Backend** + +TypeScript was chosen intentionally for Open Game Backend to make it dead simple to write & modify modules. TypeScript is not as fast as Go, but the performance gains are negligible since most overhead is spent waiting for the database. diff --git a/docs/backend/comparison/playfab.mdx b/docs/backend/comparison/playfab.mdx new file mode 100644 index 00000000..7ca113a5 --- /dev/null +++ b/docs/backend/comparison/playfab.mdx @@ -0,0 +1,93 @@ +--- +title: Microsoft PlayFab +--- + + + Please read Open Game Backend's [roadmap](/docs/roadmap) for known limitations. + + +[PlayFab](https://playfab.com/) was founded in 2014 and acquired by Microsoft in 2018. It aims to provide a comprehensive set of features for your games. + +## What PlayFab does well + +- Mature +- Anyone can sign up +- Free tier available +- Tightly integrated with console platforms +- Tightly integrated with Azure + +## How PlayFab and Open Game Backend differ + +### Closed source + +**PlayFab** + +PlayFab is closed source, which means you can't see how it works if something breaks. If PlayFab ever goes down, you're at the mercy of Microsoft to fix it. If PlayFab is ever shut down (like Game Sparks), you're out of luck. + +**Open Game Backend** + +Open Game Backend is open-source under a permissive Apache 2.0 license. This means you can see how it works, and if something breaks, you can fix it yourself. You can host it yourself or pay someone else to host it for you. + +### Rigidity & second-class scripting + +**PlayFab** + +PlayFab does not allow you to modify the functionality of existing features. This means that if you want to change how something works, you will have to build it from scratch. + +PlayFab provides [CloudScript](https://learn.microsoft.com/en-us/gaming/playfab/features/automation/cloudscript/) for scripting & [Entities](https://learn.microsoft.com/en-us/gaming/playfab/features/entities/) for custom data storage. However, these have limited flexibility and are not easy to write. + +In order to have more flexibility, Microsoft recommends building a second backend using other Azure services, which is not ideal. + +**Open Game Backend** + +Open Game Backend is built from the ground up with scripting as a first-class citizen. If you want to change how something works, run `opengb fork ` and customize as needed. + +### Scripting persistence + +**PlayFab** + +PlayFab provides persistence through [Entities](https://learn.microsoft.com/en-us/gaming/playfab/features/entities/). Entities are are simple key-value storage mechanism with limited access control and no support for transactions, complex queries, or indexing + +**Open Game Backend** + +Open Game Backend modules are provided the full flexibility of PostgreSQL, including transactions, complex queries, and indexing. + +### Forces you to use Azure & its ecosystem + +**PlayFab** + +If you're not already an Azure customer, you may not want to be forced in to the Azure ecosystem. + +**Open Game Backend** + +Open Game Backend is built to be cloud-agnostic on common technologies. You can host it on any cloud provider, or on your own hardware. + +### Unpredictable usage-based pricing + +**PlayFab** + +PlayFab's pricing is based on usage, which can be unpredictable. + +**Open Game Backend** + +Open Game Backend has a predictable pricing model based on the number of players. + +### Fragmentation + +**PlayFab** + +After being acquired by Microsoft, efforts to integrate PlayFab with Azure have fallen short and resulted in fragemented features, documentation, and SDKs upport. [Read more.](https://medium.com/@imperium42/the-silent-death-of-playfab-29614f5b9f15) + +**Open Game Backend** + +Open Game Backend is intented to provide a strong, core engine that other modules can built on top of, which is much more sustainable than maintaining numerous APIs under poor management. Modules can come and go, but the OpenGB engine will always be well maintained. + +### Documentation + +**PlayFab** + +Partially because of the fragmentation, PlayFab's documentation is incredibly hard to navigate. It's difficult to find what you're looking for, and when you do, it's often outdated. + +**Open Game Backend** + +The OpenGB Engine provides a way to automatically generate documentation for modules, scripts, and errorrs. This means that documentation is built in to the module configs and is always up to date & autogenerated. diff --git a/docs/backend/comparison/pragma.mdx b/docs/backend/comparison/pragma.mdx new file mode 100644 index 00000000..b7438bee --- /dev/null +++ b/docs/backend/comparison/pragma.mdx @@ -0,0 +1,27 @@ +--- +title: AccelByte +--- + + + Please read Open Game Backend's [roadmap](/docs/roadmap) for known limitations. + + +[Pragma](https://pragma.gg/) was founded in 202. Both platforms have similar limitations, so we'll compare them in the same document. + +## What Pragma does well + +- Source-disclosed engine + +## How Pragma and Open Game Backend compare + +### Source-discord vs open-source + +### Prefab backend vs modular backend + +Pragma provides a predefined set of features out of the box. While you can modify Pragma to extend it with your own features, you must have a license to the engine + +Nakama encourages developers to [use scripting](https://heroiclabs.com/docs/nakama/server-framework/introduction/index.html) to extend the server. While the scripting is extensive, it's not intended for all use cases. + +This means Nakama has first-class services written in Go and asks you to write second-class code written in a scripting language without access to a mature database. + +Open Game Backend is designed from the ground up to be modular at its core. It provides a set of official modules to get you started quickly, but the primary use case is to extend the server with your own modules. All modules are first-class citizens have access to a mature database. diff --git a/docs/backend/comparison/supabase.mdx b/docs/backend/comparison/supabase.mdx new file mode 100644 index 00000000..6cb2c9ec --- /dev/null +++ b/docs/backend/comparison/supabase.mdx @@ -0,0 +1,99 @@ +--- +title: Supabase +--- + + + Please read Open Game Backend's [roadmap](/docs/roadmap) for known limitations. + + +[Supabase](https://supabase.com/) was founded in 2020. While not specifically a game backend, it is a rising choice for game developers due to its ease of use and flexibility. + +## What Supabase does well + +- Open-source (Apache 2.0) +- Tightly integrated with PostgreSQL +- Realtime +- Easy to use UI & API +- Quick to get started + +## How Supabase and Open Game Backend are similar + +- Open-source (Apache 2.0) +- Built on top of PostgreSQL +- Built on Deno + +## How Supabase and Open Game Backend differ + +### Secure by default + +Games are notorious for the amount of DDoS & botting attacks launched against them. Security needs to be considered early on when developing your backend. + +**Supabase** + +Supabase Database is designed to let developers iterate quickly, but leaves security as an afterthought. + +A few key points: + +- Your database is exposed publicly by default without well configured [RLS policies](https://supabase.com/docs/guides/database/postgres/row-level-security). Many developers [miss this key point](https://saifur-rahman.com/supabase-private-schema.html) or have [trouble using RLS correctly](https://maxlynch.com/2023/11/04/tips-for-row-level-security-rls-in-postgres-and-supabase/). +- Rate limiting is effectively non-existent. Currently, you have to [reverse engineer their API and set up a reverse proxy in front of Supabase in order to implement rate limiting](https://www.reddit.com/r/Supabase/comments/17jxcdc/supabase_security_concerns/). This adds latency, complexity, and a heavy time commitment. +- Captchas are [only available for auth endpoints](https://supabase.com/docs/guides/platform/going-into-prod#abuse-prevention) and cannot be enabled for other sensitive scripts without manually writing this yourself in an Edge Function. +- Many developers with mature products on Supabase end up needing to completely rewrite their database logic on the backend to explicitly handle security concerns. Edge Functions are built for this use case, but they're still maturing. ([1](https://github.com/supabase/edge-runtime/issues/212) [2](https://github.com/supabase/supabase/issues/12293#issuecomment-2028699985) [3](https://news.ycombinator.com/item?id=36005966)) + +**Open Game Backend** + +Open Game Backend requires everything to be secure by default: + +- All logic is forced to be a server-side script. Never trust the client. +- Database & scripts are private by default, explicitly specify what's public. +- Granular rate limiting (per account and/or per IP) is provided out of the box. +- Captchas are available for all use cases. +- Strict schemas are enforced on all requests. + +### Modularity + +**Supabase** + +Supabase is built to be a general purpose backend framework. Backend code is not meant to be published as reusable packages or shared across multiple games within a studio. + +**Open Game Backend** + +Open Game Backend is a combination of two components: + +- **OpenGB Engine** Abstracts away backend complexity to make writing [modules](/docs/modules) simple & fast +- **OpenGB Module Registry** Pre-made [modules](/docs/modules) that can be used to get up and running instantly + +OpenGB's modularity lets you use pre-existing [modules](/docs/modules) to get started faster, use modules from the community, and share modules across multiple games. This allows it to be easily customized and extended to fit the needs of a game. + +### Supabase Edge Functions vs OpenGB Scripts + +**Supabase** + +Supabase [Edge Functions](https://supabase.com/docs/guides/functions/quickstart) are similar in some ways to Open Game Backend's scripts. Both are built on Deno, backed by PostgreSQL, and requests are ephemeral + isolated. + +Supabase Edge Functions are intended to be used for simple serverless functions. Most use cases today are for handling a patchwork of functionality that isn't provided by Supabase by default (e.g. webhooks, push notifications, Discord API calls). Edge Functions [still behave like an experimental pre-release feature](https://github.com/supabase/supabase/issues/12293#issuecomment-2028699985). + +**Open Game Backend** + +Open Game Backend scripts are at the core of the project, so they're built to be easy to use, flexible, and reliable. They're used to power core functionality such as tokens, authentication, and rate limiting. + +### Rigid Core Features vs Flexible Modular Features + +**Supabase** + +Supabase provides core functionality that cannot be modified, such as: [authentication](https://supabase.com/docs/guides/auth), [storage](https://supabase.com/docs/guides/storage), and [realtime](https://supabase.com/docs/guides/realtime). + +**Open Game Backend** + +Open Game Backend instead provides these same features as [modules](/docs/modules) that can be modified & extended as needed. For example: authentication and storage modules. These can be customized easily by running the `opengb fork` command. + +For example, this is specifically important for games where authentication with platforms like Steam, consoles, and instant gaming platforms don't fit well with Supabase's core authentication features. + +### Dashboard + +**Supabase** + +Supabase has a well built dashboard for managing your database. + +**Open Game Backend** + +While Open Game Backend does not have a dashboard for interacting with the database (yet), tools like [Beekeeper Studio](https://www.beekeeperstudio.io/) can be used with OpenGB Postgres databases out of the box. diff --git a/docs/backend/conventions.mdx b/docs/backend/conventions.mdx new file mode 100644 index 00000000..7604746f --- /dev/null +++ b/docs/backend/conventions.mdx @@ -0,0 +1,19 @@ +--- +title: "Conventions" +--- + + + This page is a work in progress. + + +## Only use classes for developer-facing APIs + +We don't expect the user to use pure interfaces & functions; consuming well +designed OOP interfaces can be cleaner. + +Examples of using classes: `Context` and `Runtime` are classes designed to have +a clean interface for the user. + +Examples of using interfaces: `Trace` is a simple data container that can be +easily serialized & deserialized. There are standalone functions that can +operate on a trace. diff --git a/docs/backend/design/deno.mdx b/docs/backend/design/deno.mdx new file mode 100644 index 00000000..afc9be6a --- /dev/null +++ b/docs/backend/design/deno.mdx @@ -0,0 +1,31 @@ +--- +title: Why Deno? +--- + +For simplicity, we'll assume that we're comparing Deno to Node.js. To read more on why Open Game Backend chose TypeScript as a language, see [here](/docs/engine/design/typescript). + +## Portability + +Unlike Node.js which uses CommonJS modules with a heavy standard library, Deno provides an absolutely minimal standard library and relies instead on ECMAScript modules which are very portable. ECMAScript modules are the standard way of managing JavaScript modules across multiple JS runtimes & browsers. + +This means that code written for Deno is usually very portable. Node modules can still be used easily via something like [ESM](https://esm.sh/), but they way they're used is more portable. + +### Portable code + +This is important because it means that code written for Open Game Backend modules are very portable, instead of being built around a specific runtime (such as Node.js). + +Because of this, adding support for a runtime like [Bun](https://bun.sh/) would be very easy. + +### Serverless Runtimes + +Open Game Backend aims to be as portable as possible, and this means that it should be able to run on any serverless runtime, such as Deno Deploy, Cloudflare Workers, Vercel, and Netlify. + +## Tooling + +Open Game Backend was initiated by a bunch of Rust nerds who value the quality tooling (e.g. Cargo, Crates, Clippy, Rustfmt, RA, etc.) provided around the language. + +Deno aims to provide the same quality of tooling (tests, documentation, formatting, linting, package managers, etc) but for TypeScript, which provides faster iteration speeds and is accessible to a wider audience than Rust. If you've used Rust before, you'll notice a lot of similarities to the Rust ecosystem in the tooling provided by Deno. + +## Easy installer + +Deno makes it very easy to install the Open Game Backend using the `deno install` command. diff --git a/docs/backend/design/drizzle.mdx b/docs/backend/design/drizzle.mdx new file mode 100644 index 00000000..b5eeb415 --- /dev/null +++ b/docs/backend/design/drizzle.mdx @@ -0,0 +1,55 @@ +--- +title: Why Drizzle? +--- + +## Drizzle vs vanilla SQL + +Open Game Backend is designed to use an ORM to be more friendly for game developers & provide type safety out to the database of the box. The only thing better than docs you can build in a tool is an obvious interface & great tab completion. Well-designed ORMs provide this. + +Vanilla SQL requires a lot of repetition between redefining the model in TypeScript & selecting columns to query. It's very easy to write hard to find bugs using vanilla SQL because of this repetition. + +## Drizzle vs Prisma + +One core goal of Open Game Backend is to make using SQL databases easy for game devs who have no experience with SQL. OpenGB was originally built on Prisma instead of Drizzle, but was later refactored on to Drizzle. + +**Friendliness towards non-SQL users** + +Drizzle is explicitly designed for developers who know SQL well, which makes it slightly more difficult to pick up. + +Prisma is built primarily to make using SQL databases easy for developers who don't know SQL, while still providing a powerful API for developers who do know SQL. The API is easy to understand, schemas are easy to write, migrations are buit to be fool-proof, and the entire API is intellisense friendly so you rarely have to read the docs. + +While Drizzle does have a [Queries API](https://orm.drizzle.team/docs/rqb) intended to work similar to Prisma, designing schemas is still done in an SQL-like manner. + +We partially improved this by providing objects that have everything you need in one place (`Query` and `Database`). This allows devs to type `Query.` and get a full list of everything they can do or `Database.` to get a list of all tables. This prevents the need to hunt through Drizzle docs to find the function you need. + +There's a lot of room for improvement here, and we'll likely write a thin layer on top of Drizzle to help streamline insert & update statements. + +**WASM vs pure TypeScript** + +Prisma depends ona WASM module in order to serialize & deserialize database queries. This is intended to make the client faster, but it makes Prisma (a) harder to run in serverless with larger bundles and (b) requires finnessing to reuse the same WASM engine across multiple generated libraries. + +Drizzle's TypeScript-native interface is much more lightweight. + +**Dependency on Node (and therefore Docker)** + +Prisma has a lot of bugs when running under Drizzle. + +To work around this, we used to ship a copy of _both_ Deno & Node in order to run Prisma commands. + +**Heavy vs lightweight** + +Prisma takes a _long_ time to run commands & generate SDKs. This made running `opengb dev` take a long time. It also frequently failed for unknown reasons, likely because it had so many network dependencies it had to download and race conditions when running Prisma in parallel. + +**Unnecessary `SELECT` statement after `create` & other unoptimized commands** + +See [this issue](https://github.com/prisma/prisma/issues/4246). + +This has a pretty nasty performance impact when inserting rows frequently. Most use cases won't notice the impact, but it's important to be aware of for performance-sensitive modules. + +**No headless migrations CLI** + +Prisma requires a TTY to run development migrations. It took a lot of work to get the Prisma CLI to play nice with the Open Game Backend `dev` command. It would be nice to have a raw API we can call to run migrations within our own code. + +**Slow builds** + +Prisma takes a long time to build new clients. Open Game Backend caches heavily and parallelizes builds to make this less of an issue, but it's still a pain point. diff --git a/docs/backend/design/misc.mdx b/docs/backend/design/misc.mdx new file mode 100644 index 00000000..1e0c987a --- /dev/null +++ b/docs/backend/design/misc.mdx @@ -0,0 +1,12 @@ +--- +title: "Misc" +--- + +### Don't use `deno compile` + +OpenGB relies on access to the `deno` binary to run dev servers. + +Binaries generated with `deno compile` don't give you access to the `deno` binary. + +Therefore, all instances of OpenGB must be ran using `deno install` or `deno run`. + diff --git a/docs/backend/design/postgres.mdx b/docs/backend/design/postgres.mdx new file mode 100644 index 00000000..b54c9e82 --- /dev/null +++ b/docs/backend/design/postgres.mdx @@ -0,0 +1,72 @@ +--- +title: Why Postgres? +--- + +## SQL vs Document Stores + +Many tools use document store databases (like MongoDB) to make it easier to get started with. + +Document stores make it _seem_ like you don't need to worry about these things: + +- Schemas & migrations +- Querying +- Relationships +- Performance (sort of, I mean [it's web scale](https://www.youtube.com/watch?v=b2F-DItXtZs) after all) + +However, every production ready backend needs to worry about these things for both SQL & document stores. The difference is, SQL forces you to do this up front when you write a schema. Document stores let you shoot yourself in the foot later on when you realize you need to do these things. + +Open Game Backend instead opts to force developers to worry about these things up front, but leverage [Prisma](https://prisma.io) to abstract away the complexities of migrations, querying, joining, indexing, etc. + +Working with databases in Open Game Backend feels a lot like working in a game engine (which is the goal), but you still have full access to the power of SQL if you need it. + +## Postgres vs MySQL + +MySQL is old and decrepid. Stop using it if you care about your data and your sanity. + +## Postgres vs MySQL + Vitess + +Vitess is a well proven way to scale MySQL, but you're still dealing with the same pain points of MySQL. PlanetScale is built on top of Vitess. + +Solutions like Citus Data are built on top of Postgres and provide a similar sharding solution. + +## Postgres vs CockroachDB + +There's a lot to love about CockroachDB. + +Cockroach can be more scalable & performant & fault tolerant than Postgres since its architecture was inspired by Google Spanner. + +However, Cockroach is licensed under BSL. BSL is not an an OSI-approved license and limits the things you can do with Open Game Backend. For example, companies cannot provide hosting services for Open Game Backend since you need to also serve the Cockroach backend. This is similar to how WordPress hosting services helped grow thep popularity of WP. + +Additionally, Cockroach does not benefit from the same extension ecosystem that Postgres has. + +Since Cockroach is designed to be _mostly_ compatible with Postgres, it's possible to eventually allow choosing between Postgres & Cockroach. Because Open Game Backend is built on top of Prisma, we can swap the Primsa provider to `cockroachdb` and support multiple backends out of the box. We can then use our extensive testing to certify which modules work on Cockroach without modification. + +## Postgres vs KV (DynamoDB/Cassandra) + +Many game backends are built on top of KV databases because of their scalability and fault tolerance. This works well since many game backends don't rely on heavily relational data. + +However, these databases come at the cost of consistency, schemas, rigid querying, and transactions. + +One of the core goals of Open Game Backend is to have guardrails to help prevent common mistakes when building databases & backends. + +Some big pain points that develoeprs using KV databases run in to that Postgres solves: + +- **Schemas & constraints** KV databases don't enforce schemas. Postgres provides constraints (e.g. unique usernames, foreign keys, integer ranges) to ensure data integrity. +- **Transactions** Transactions in KV databases are either very limited or nonexistent. When dealing with important data like currency or inventory, not using transactions can ruin your game's economy and lose developers a lot of money. Postgres supports full transactions out of the box which makes handling important data much easier. +- **Querying** KV databases are great for simple key-value lookups, but as soon as you need to do anything more complex, you're out of luck. DynamoDB supports secondary indexes and Cassandra has materialized views, but both have tradeoffs. Postgres indexes are much more flexible and powerful. + +One big benefit of KV stores is it's hard to write a slow query by design (e.g. Cassandra requires an explicit `ALLOW FILTERING` to run a query that doesn't use the PK or secondary index). Postgres allows you to run queries without an associated index, which makes it really easy to accidentally kill your database. This is something that hopefully Open Game Backend will help prevent in the future. + +## Postgres vs ClickHouse + +More and more workloads require OLAP capabilities. It's common to turn to a combination of an OLTP database (usually SQL) and an OLAP database (usually a KV store or columnar store like ClickHouse). + +We're going to focus specifically on the use case that ClickHouse targets. ClickHouse provides a flexible OLAP database that can run complex queries on large datasets very quickly. + +Postgres's extensive extension ecosystem allows you to add OLAP capabilities to Postgres at comparable speeds to ClickHouse. [Read more here.](https://pigsty.io/blog/pg/pg-eat-db-world/) + +## Extensions + +Postgres has a rich ecosystem of extensions that can add powerful features to your database. See a full list [here](https://neon.tech/docs/extensions/pg-extensions). + +Databases are usually the most common point of failure, so adding more types of databases for specialized data types adds more points of failure. Postgres extensions allow you to add specialized data types and features to Postgres without adding more points of failure & a common query interface. diff --git a/docs/backend/design/registries-dependencies.mdx b/docs/backend/design/registries-dependencies.mdx new file mode 100644 index 00000000..513b94cf --- /dev/null +++ b/docs/backend/design/registries-dependencies.mdx @@ -0,0 +1,10 @@ +--- +title: Registries & Dependencies +--- + +## Why do we pull source code instead of bundling modules? + +The OpenGB dev experience encourages developers to view & fork the source code of other modules. By downloading the original source files, developers can jump to the source code of their dependencies from their IDE. + +Downloading a bundle would require developers to manually find the repository & ref of their dependencies on GitHub. + diff --git a/docs/backend/design/typescript.mdx b/docs/backend/design/typescript.mdx new file mode 100644 index 00000000..d75a04f0 --- /dev/null +++ b/docs/backend/design/typescript.mdx @@ -0,0 +1,73 @@ +--- +title: Why TypeScript? +--- + +TypeScript is the language used for Open Game Backend scripts. Most of the Open Game Backend core is also written in TypeScript, but the performance-sensitive parts are/will be written in Rust and compiled to WASM. + +## Goals + +- Iteration speed +- Ease of access +- Ecosystem + +## Built like a game engine: scripts vs core engine + +Game engines are often designed with with separate languages for the scripting layer (C# in Unity, GDScript in Godot, Blueprints in Unreal, Lua in Roblox) and the core engine (C++ in Unity & Godot & Unreal & Roblox). In the case of Open Game Backend, the scripting layer is TypeScript and the core engine optimizes the performance sensitive parts using Rust + WASM. + +Open Game Backend is designed with the same methodology as game engines for the same reason: + +- **Iteration speed** Developers aren't using Open Game Backend to spend time writing code. They're here to get the job done as fast as possible and sleep easy knowing it's written on a solid foundation. High-level logic is defined in scripts and the heavy lifting is done in the core engine. + +## Ease of acess + +There are not many developers who have not touched TypeScript at some point in their career. It's a language that's easy to pick up and get started with no matter what engine you're using. + +## Ecosystem + +TypeScript/JavaScript/NPM has arguably the most comprehensive ecosystem of packages available to developers that makes it very easy to plug in to Open Game Backend. + +## Serverless deployments on V8 isolates + +Game backends need to be both cost-effective and scalable to handle the unpredictable traffic patterns of games. Serverless deployments are the easiest way to achieve this. + +Platforms like AWS Lamda, Google Cloud Functions, and Azure Functions often give the idea of "serverless backends" bad reputation for things like coldstart times and cost. These platforms are built to run Docker images, which require a lot of overhead to start up and run. + +On the other hand, serverless runtimes like Deno Deploy, Cloudflare Workers, Vercel, and Netlify are built on top of V8 isolates. These runtimes are designed to run JavaScript/TypeScript code in very lightweight environments. Read more [here](https://blog.cloudflare.com/cloud-computing-without-containers/). + +V8 serverless environments come at the cost of requiring your backend to be written in JavaScript & WASM, which was a factor in choosing TypeScript for Open Game Backend. + +## TypeScript vs Rust + +As much as we love Rust and believe it's the best language for writing high-performance code, it has two main problems that prohibited us from using it for Open Game Backend: + +- **Steep learning curve** Many developers have a hard time learning Rust. We considered providing proc macros & helper types to make writing Rust easier, but it would still be a barrier to entry for many developers. +- **Slow iteration speed** The Rust compiler is notoriously slow. We want Open Game Backend to get out of the way of building your game, so iteration speed is a top priority. + +## TypeScript vs Go + +Go is great for high-performance backends and has a blazing fast compiler, which makes it an interesting candidate for backend scripting. It was also designed by Google to be easy to learn for junior developers. + +However, Go has a few problems that prohibited us from choosing it: + +- **Pointers** Most game developers coming from Unity, Godot, and HTML are not familiar with the concept of poitners. Go requires understanding pointers and structs in order to use it. +- **Unique syntax** TypeScript's syntax is very similar to C# & Java. Go's syntax is unique and requires learning a new language. +- **Difficult to use data structures** Workign with arrays & maps in Go are unnecesarily difficult compared to TypeScript for the use case of Open Game Backend. + +## TypeScript vs Java/C#/Python + +There are many other widely-used langauges, but none of them provided no tangible advantage over TypeScript. Most importantly, TypeScript has the widest developer audience. + +## TypeScript vs esoteric langauge + +All three engineers that developed Open Game Backend have all developed their own programming langauges at some point. Believe me, we joked about creating a `.ogb` language for Open Game Backend. + +## Future possibility: WASM Open Game Backend modules + +Open Game Backend is designed to be interopable as a langauge-agnostic protocol instead of a library. For example, all scripts are interfaced with using JSON schemas. The script calling API is designed to work across language boundaries. + +This was done for two reasons: + +1. Ability to optimize by writing performance-sensitive parts in Rust and compiling to WASM +2. Ability to write scripts in any language that can compile to WASM in the future + +At the moment, the project is focused on supporting & providng the best experience for only TypeScript. diff --git a/docs/backend/introduction.mdx b/docs/backend/introduction.mdx new file mode 100644 index 00000000..842d8f76 --- /dev/null +++ b/docs/backend/introduction.mdx @@ -0,0 +1,7 @@ +--- +title: Introduction +--- + + + Documentation coming very soon! +