From 3d620c496120d4284e3f6c610cfefdc547edd743 Mon Sep 17 00:00:00 2001 From: rimutaka Date: Tue, 31 May 2022 22:29:06 +0000 Subject: [PATCH] Added GraphQL support #50 * added pub mod GraphQL * added juniper fork as a dependency * added GQL support to all public structures used in Report --- stackmuncher_lib/Cargo.toml | 5 +- stackmuncher_lib/src/contributor.rs | 22 ++- stackmuncher_lib/src/graphql.rs | 165 ++++++++++++++++++ stackmuncher_lib/src/lib.rs | 1 + .../src/report/commit_time_histo.rs | 8 +- stackmuncher_lib/src/report/kwc.rs | 5 +- stackmuncher_lib/src/report/mod.rs | 2 +- stackmuncher_lib/src/report/overview.rs | 8 +- stackmuncher_lib/src/report/report.rs | 8 +- stackmuncher_lib/src/report/tech.rs | 8 +- 10 files changed, 215 insertions(+), 17 deletions(-) create mode 100644 stackmuncher_lib/src/graphql.rs diff --git a/stackmuncher_lib/Cargo.toml b/stackmuncher_lib/Cargo.toml index 1c38c26..2be6a23 100644 --- a/stackmuncher_lib/Cargo.toml +++ b/stackmuncher_lib/Cargo.toml @@ -16,7 +16,7 @@ chrono = "0.4" tracing = { version = "0.1", features = ["log"] } encoding_rs_io = "0.1" encoding_rs = "0.8" -uuid = { version = "0.8", features = ["v4"] } +uuid = { version = "1.1.0", features = ["v4"] } tokio = { version = "1", features = ["full"] } sha-1 = "0.10" sha2 = "0.10" @@ -24,6 +24,9 @@ bs58 = "0.4" path-absolutize = "3.0" flate2 = "1.0" rust-embed = { version = "6", features = ["compression"] } +# Temporarily running off a fork until https://github.com/graphql-rust/juniper/issues/1071 is resolved +# juniper = { git = "https://github.com/graphql-rust/juniper.git" } +juniper = { git = "https://github.com/rimutaka/juniper.git", branch = "impl-hashset-as-vec" } [dev-dependencies] tracing-subscriber = "0.3" diff --git a/stackmuncher_lib/src/contributor.rs b/stackmuncher_lib/src/contributor.rs index a293760..0803c56 100644 --- a/stackmuncher_lib/src/contributor.rs +++ b/stackmuncher_lib/src/contributor.rs @@ -1,10 +1,19 @@ use super::git::GitLogEntry; +use crate::graphql::RustScalarValue; +use juniper::GraphQLObject; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; +/// This type would normally be a `(String, String)` tuple, but GraphQL requires a custom implementation for that. +/// On the other hand there is a default impl for `[T]`. +/// +/// The serialized output looks the same for both: `["rimutaka","max@onebro.me"]`. +pub type NameEmailPairType = [String; 2]; + /// A GIT author or committer. E.g. `Author: rimutaka ` from `git log`. /// It contains extended info like what was committed, when, contact details. -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, GraphQLObject)] +#[graphql(scalar = RustScalarValue)] pub struct Contributor { /// Email is the preferred ID, but it can be just the name if the email is missing, e.g. `max@onebro.me` for `Author: rimutaka ` /// @@ -13,7 +22,7 @@ pub struct Contributor { pub git_id: String, /// A list of possible identities as name/email pairs for extracting contact details and de-duplication. /// E.g. `Author: rimutaka would be `rimutaka`/`max@onebro.me`. - pub name_email_pairs: HashSet<(String, String)>, + pub name_email_pairs: HashSet, /// The full SHA1 of the very last commit by this contributor. This bit should be retained for matching repositories on STM server. pub last_commit_sha1: String, /// The timestamp as EPOCH of the very last commit by this contributor. @@ -32,7 +41,8 @@ pub struct Contributor { pub commits: Vec, } -#[derive(Serialize, Deserialize, Debug, Clone, Eq)] +#[derive(Serialize, Deserialize, Debug, Clone, Eq, GraphQLObject)] +#[graphql(scalar = RustScalarValue)] pub struct ContributorFile { /// The file name extracted from GIT, including the relative path, e.g. `myproject/src/main.rs` pub name: String, @@ -86,7 +96,7 @@ impl Contributor { // this is a known contributor - merge with the existing one contributor .name_email_pairs - .insert((commit.author_name_email.0, commit.author_name_email.1)); + .insert([commit.author_name_email.0, commit.author_name_email.1]); // only the latest version of the file is of interest for file in commit.files { @@ -102,8 +112,8 @@ impl Contributor { // it's a new contributor - add as-is // add the identities as name/email pairs - let mut name_email_pairs: HashSet<(String, String)> = HashSet::new(); - name_email_pairs.insert((commit.author_name_email.0, commit.author_name_email.1)); + let mut name_email_pairs: HashSet = HashSet::new(); + name_email_pairs.insert([commit.author_name_email.0, commit.author_name_email.1]); // collect the list of touched files with the commit SHA1 let mut touched_files: HashMap = HashMap::new(); diff --git a/stackmuncher_lib/src/graphql.rs b/stackmuncher_lib/src/graphql.rs new file mode 100644 index 0000000..64c586f --- /dev/null +++ b/stackmuncher_lib/src/graphql.rs @@ -0,0 +1,165 @@ +//! This module is needed to support the cloud-side of the project. +//! It enables GraphQL support for core structures used on the client and server sides. + +use juniper::{ + graphql_scalar, + parser::{ParseError, ScalarToken, Token}, + serde::{de, Deserialize, Deserializer, Serialize}, + InputValue, ParseScalarResult, ScalarValue, Value, +}; +use std::{convert::TryInto as _, fmt}; + +/// An extension to the standard GraphQL set of types to include Rust scalar values. +/// Only the types used in this project are added to the list. +/// ### About GraphQL scalars +/// * https://graphql.org/learn/schema/#scalar-types +/// * https://www.graphql-tools.com/docs/scalars#custom-scalars +/// ### About extending the GraphQL scalars in Juniper +/// * https://graphql-rust.github.io/juniper/master/types/scalars.html#custom-scalars +/// * https://github.com/graphql-rust/juniper/issues/862 +#[derive(Clone, Debug, PartialEq, ScalarValue, Serialize)] +#[serde(untagged)] +pub enum RustScalarValue { + /// A GraphQL scalar for i32 + #[value(as_float, as_int)] + Int(i32), + /// A custom scalar for u64. The value is serialized into JSON number and should not be more than 53 bits to fit into JS Number type: + /// * Number.MAX_SAFE_INTEGER = 2^53 - 1 = 9_007_199_254_740_991 + /// * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number + /// JSON spec does not constrain integer values unless specified in the schema. 53 bits is sufficient for our purposes. + U64(u64), + /// A custom scalar for i64 used in EPOCH timestamps. Theoretically, the value should never be negative because all STM dates are post 1970. + /// The value is serialized into JSON number and should not be more than 53 bits to fit into JS Number type: + /// * Number.MIN_SAFE_INTEGER = -(2^53 - 1) = -9,007,199,254,740,991 + I64(i64), + /// A GraphQL scalar for f64 + #[value(as_float)] + Float(f64), + /// A GraphQL scalar for String + #[value(as_str, as_string, into_string)] + String(String), + /// A GraphQL scalar for bool + #[value(as_bool)] + Boolean(bool), +} + +impl<'de> Deserialize<'de> for RustScalarValue { + fn deserialize>(de: D) -> Result { + struct Visitor; + + impl<'de> de::Visitor<'de> for Visitor { + type Value = RustScalarValue; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a valid input value") + } + + fn visit_bool(self, b: bool) -> Result { + Ok(RustScalarValue::Boolean(b)) + } + + fn visit_i32(self, n: i32) -> Result { + Ok(RustScalarValue::Int(n)) + } + + fn visit_u64(self, b: u64) -> Result { + if b <= u64::from(i32::MAX as u32) { + self.visit_i32(b.try_into().unwrap()) + } else { + Ok(RustScalarValue::U64(b)) + } + } + + fn visit_u32(self, n: u32) -> Result { + if n <= i32::MAX as u32 { + self.visit_i32(n.try_into().unwrap()) + } else { + self.visit_u64(n.into()) + } + } + + fn visit_i64(self, n: i64) -> Result { + if n <= i64::MAX as i64 { + self.visit_i64(n.try_into().unwrap()) + } else { + // Browser's `JSON.stringify()` serializes all numbers + // having no fractional part as integers (no decimal point), + // so we must parse large integers as floating point, + // otherwise we would error on transferring large floating + // point numbers. + // TODO: Use `FloatToInt` conversion once stabilized: + // https://github.com/rust-lang/rust/issues/67057 + Ok(RustScalarValue::Float(n as f64)) + } + } + + fn visit_f64(self, f: f64) -> Result { + Ok(RustScalarValue::Float(f)) + } + + fn visit_str(self, s: &str) -> Result { + self.visit_string(s.into()) + } + + fn visit_string(self, s: String) -> Result { + Ok(RustScalarValue::String(s)) + } + } + + de.deserialize_any(Visitor) + } +} + +#[graphql_scalar(with = u64_scalar, scalar = RustScalarValue)] +type U64 = u64; + +mod u64_scalar { + use super::*; + + pub(super) fn to_output(v: &U64) -> Value { + Value::scalar(*v) + } + + pub(super) fn from_input(v: &InputValue) -> Result { + v.as_scalar_value::() + .copied() + .ok_or_else(|| format!("Expected `RustScalarValue::U64`, found: {}", v)) + } + + pub(super) fn parse_token(value: ScalarToken<'_>) -> ParseScalarResult<'_, RustScalarValue> { + if let ScalarToken::Int(v) = value { + v.parse() + .map_err(|_| ParseError::UnexpectedToken(Token::Scalar(value))) + .map(|s: u64| s.into()) + } else { + Err(ParseError::UnexpectedToken(Token::Scalar(value))) + } + } +} + +#[graphql_scalar(with = i64_scalar, scalar = RustScalarValue)] +type I64 = i64; + +mod i64_scalar { + use super::*; + + pub(super) fn to_output(v: &I64) -> Value { + Value::scalar(*v) + } + + pub(super) fn from_input(v: &InputValue) -> Result { + v.as_scalar_value::() + .copied() + .ok_or_else(|| format!("Expected `RustScalarValue::I64`, found: {}", v)) + } + + pub(super) fn parse_token(value: ScalarToken<'_>) -> ParseScalarResult<'_, RustScalarValue> { + if let ScalarToken::Int(v) = value { + v.parse() + .map_err(|_| ParseError::UnexpectedToken(Token::Scalar(value))) + .map(|s: i64| s.into()) + } else { + Err(ParseError::UnexpectedToken(Token::Scalar(value))) + } + } +} diff --git a/stackmuncher_lib/src/lib.rs b/stackmuncher_lib/src/lib.rs index 117cfc7..d627459 100644 --- a/stackmuncher_lib/src/lib.rs +++ b/stackmuncher_lib/src/lib.rs @@ -11,6 +11,7 @@ pub mod config; pub mod contributor; pub mod file_type; pub mod git; +pub mod graphql; mod ignore_paths; pub mod muncher; pub mod processors; diff --git a/stackmuncher_lib/src/report/commit_time_histo.rs b/stackmuncher_lib/src/report/commit_time_histo.rs index aeefe63..fde1d6f 100644 --- a/stackmuncher_lib/src/report/commit_time_histo.rs +++ b/stackmuncher_lib/src/report/commit_time_histo.rs @@ -1,5 +1,7 @@ use super::Report; +use crate::graphql::RustScalarValue; use chrono::{self, Duration, TimeZone, Timelike, Utc}; +use juniper::GraphQLObject; use serde::{Deserialize, Serialize}; use tracing::warn; @@ -8,7 +10,8 @@ pub const RECENT_PERIOD_LENGTH_IN_DAYS: i64 = 365; /// Number of commits or percentage of commits per UTC hour. /// The structure is skipped in JSON if all values are zero and is initialized to all zeros to have fewer Option unwraps. -#[derive(Serialize, Deserialize, Clone, Debug)] +#[derive(Serialize, Deserialize, Clone, Debug, GraphQLObject)] +#[graphql(scalar = RustScalarValue)] pub struct CommitTimeHistoHours { #[serde(skip_serializing_if = "CommitTimeHistoHours::is_zero", default = "u64::default")] pub h00: u64, @@ -61,7 +64,8 @@ pub struct CommitTimeHistoHours { } /// Contains members and methods related to commit time histogram -#[derive(Serialize, Deserialize, Clone, Debug)] +#[derive(Serialize, Deserialize, Clone, Debug, GraphQLObject)] +#[graphql(scalar = RustScalarValue)] pub struct CommitTimeHisto { /// The sum of all commits included in `histogram_recent`. This value is used as the 100% of all recent commits. /// The value is populated once after all commits have been added. diff --git a/stackmuncher_lib/src/report/kwc.rs b/stackmuncher_lib/src/report/kwc.rs index db71197..a45c41e 100644 --- a/stackmuncher_lib/src/report/kwc.rs +++ b/stackmuncher_lib/src/report/kwc.rs @@ -1,8 +1,11 @@ +use crate::graphql::RustScalarValue; +use juniper::GraphQLObject; use serde::{Deserialize, Serialize}; use std::collections::HashSet; use tracing::{error, warn}; -#[derive(Debug, Serialize, Deserialize, Eq, Clone)] +#[derive(Debug, Serialize, Deserialize, Eq, Clone, GraphQLObject)] +#[graphql(scalar = RustScalarValue)] pub struct KeywordCounter { /// keyword pub k: String, diff --git a/stackmuncher_lib/src/report/mod.rs b/stackmuncher_lib/src/report/mod.rs index 3059f47..a13a861 100644 --- a/stackmuncher_lib/src/report/mod.rs +++ b/stackmuncher_lib/src/report/mod.rs @@ -1,8 +1,8 @@ +pub mod commit_time_histo; pub mod kwc; pub mod overview; pub mod report; pub mod tech; -pub mod commit_time_histo; pub use overview::{ProjectReportOverview, TechOverview}; pub use report::Report; diff --git a/stackmuncher_lib/src/report/overview.rs b/stackmuncher_lib/src/report/overview.rs index e157353..ee32b57 100644 --- a/stackmuncher_lib/src/report/overview.rs +++ b/stackmuncher_lib/src/report/overview.rs @@ -1,12 +1,15 @@ use super::tech::Tech; +use crate::graphql::RustScalarValue; use chrono::{DateTime, Datelike, Timelike, Utc}; +use juniper::GraphQLObject; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use tracing::warn; /// A very concise overview of a single Tech record /// to show the share of the technology in the project -#[derive(Serialize, Deserialize, Clone, Debug, Eq)] +#[derive(Serialize, Deserialize, Clone, Debug, Eq, GraphQLObject)] +#[graphql(scalar = RustScalarValue)] pub struct TechOverview { /// The same as Tech.language pub language: String, @@ -36,7 +39,8 @@ impl PartialEq for TechOverview { /// An overview of an individual project report included in the combined report /// to avoid loading the full project report every time the combined report is looked at. -#[derive(Serialize, Deserialize, Clone, Debug, Eq)] +#[derive(Serialize, Deserialize, Clone, Debug, Eq, GraphQLObject)] +#[graphql(scalar = RustScalarValue)] pub struct ProjectReportOverview { /// A human-readable project name. It should not be used as an ID. #[serde(default = "String::new")] diff --git a/stackmuncher_lib/src/report/report.rs b/stackmuncher_lib/src/report/report.rs index 6dcea90..3d97cd8 100644 --- a/stackmuncher_lib/src/report/report.rs +++ b/stackmuncher_lib/src/report/report.rs @@ -2,11 +2,13 @@ use super::commit_time_histo::CommitTimeHisto; use super::kwc::{KeywordCounter, KeywordCounterSet}; use super::tech::{Tech, TechHistory}; use super::ProjectReportOverview; +use crate::graphql::RustScalarValue; use crate::utils::sha256::hash_str_to_sha256_as_base58; use crate::{contributor::Contributor, git::GitLogEntry, utils}; use chrono::{DateTime, Utc}; use flate2::write::GzEncoder; use flate2::Compression; +use juniper::GraphQLObject; use path_absolutize::{self, Absolutize}; use serde::{Deserialize, Serialize}; use serde_json; @@ -18,7 +20,8 @@ use tracing::{debug, error, info, warn}; /// Contains the number of elements per list to help with DB queries. /// The numbers are calculated once before saving the Report in the DB. -#[derive(Serialize, Deserialize, Clone, Debug)] +#[derive(Serialize, Deserialize, Clone, Debug, GraphQLObject)] +#[graphql(scalar = RustScalarValue)] pub struct ListCounts { tech: u64, contributor_git_ids: u64, @@ -33,7 +36,8 @@ pub struct ListCounts { keywords: u64, } -#[derive(Serialize, Deserialize, Clone, Debug)] +#[derive(Serialize, Deserialize, Clone, Debug, GraphQLObject)] +#[graphql(scalar = RustScalarValue)] #[serde(rename = "tech")] pub struct Report { /// The exact timestamp of the report generation in ISO3389 format. diff --git a/stackmuncher_lib/src/report/tech.rs b/stackmuncher_lib/src/report/tech.rs index 348c5ec..dad208d 100644 --- a/stackmuncher_lib/src/report/tech.rs +++ b/stackmuncher_lib/src/report/tech.rs @@ -1,11 +1,14 @@ use super::kwc::{KeywordCounter, KeywordCounterSet}; +use crate::graphql::RustScalarValue; +use juniper::GraphQLObject; use regex::Regex; use serde::{Deserialize, Serialize}; use std::collections::HashSet; use tracing::{debug, trace, warn}; /// Contains time-range data for its parent Tech. -#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Clone)] +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Clone, GraphQLObject)] +#[graphql(scalar = RustScalarValue)] #[serde(rename = "tech")] pub struct TechHistory { /// Number of months between the first and the last commit. @@ -24,7 +27,8 @@ pub struct TechHistory { /// Any additions to this struct should be considered for clean up before submission to stackmuncher.com /// to avoid sending out any info that doesn't need to be sent. /// See https://github.com/stackmuncher/stm_app/issues/12 -#[derive(Serialize, Deserialize, Debug, Eq, Clone)] +#[derive(Serialize, Deserialize, Debug, Eq, Clone, GraphQLObject)] +#[graphql(scalar = RustScalarValue)] #[serde(rename = "tech")] pub struct Tech { /// The name of the file for individual file reports. Not present in combined tech reports.