Skip to content

Commit

Permalink
Added GraphQL support #50
Browse files Browse the repository at this point in the history
 * added pub mod GraphQL
 * added juniper fork as a dependency
 * added GQL support to all public structures used in Report
  • Loading branch information
rimutaka committed May 31, 2022
1 parent ea178d5 commit 3d620c4
Show file tree
Hide file tree
Showing 10 changed files with 215 additions and 17 deletions.
5 changes: 4 additions & 1 deletion stackmuncher_lib/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,17 @@ chrono = "0.4"
tracing = { version = "0.1", features = ["log"] }
encoding_rs_io = "0.1"
encoding_rs = "0.8"
uuid = { version = "0.8", features = ["v4"] }
uuid = { version = "1.1.0", features = ["v4"] }
tokio = { version = "1", features = ["full"] }
sha-1 = "0.10"
sha2 = "0.10"
bs58 = "0.4"
path-absolutize = "3.0"
flate2 = "1.0"
rust-embed = { version = "6", features = ["compression"] }
# Temporarily running off a fork until https://github.com/graphql-rust/juniper/issues/1071 is resolved
# juniper = { git = "https://github.com/graphql-rust/juniper.git" }
juniper = { git = "https://github.com/rimutaka/juniper.git", branch = "impl-hashset-as-vec" }

[dev-dependencies]
tracing-subscriber = "0.3"
22 changes: 16 additions & 6 deletions stackmuncher_lib/src/contributor.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,19 @@
use super::git::GitLogEntry;
use crate::graphql::RustScalarValue;
use juniper::GraphQLObject;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};

/// This type would normally be a `(String, String)` tuple, but GraphQL requires a custom implementation for that.
/// On the other hand there is a default impl for `[T]`.
///
/// The serialized output looks the same for both: `["rimutaka","max@onebro.me"]`.
pub type NameEmailPairType = [String; 2];

/// A GIT author or committer. E.g. `Author: rimutaka <max@onebro.me>` from `git log`.
/// It contains extended info like what was committed, when, contact details.
#[derive(Serialize, Deserialize, Debug, Clone)]
#[derive(Serialize, Deserialize, Debug, Clone, GraphQLObject)]
#[graphql(scalar = RustScalarValue)]
pub struct Contributor {
/// Email is the preferred ID, but it can be just the name if the email is missing, e.g. `max@onebro.me` for `Author: rimutaka <max@onebro.me>`
///
Expand All @@ -13,7 +22,7 @@ pub struct Contributor {
pub git_id: String,
/// A list of possible identities as name/email pairs for extracting contact details and de-duplication.
/// E.g. `Author: rimutaka <max@onebro.me> would be `rimutaka`/`max@onebro.me`.
pub name_email_pairs: HashSet<(String, String)>,
pub name_email_pairs: HashSet<NameEmailPairType>,
/// The full SHA1 of the very last commit by this contributor. This bit should be retained for matching repositories on STM server.
pub last_commit_sha1: String,
/// The timestamp as EPOCH of the very last commit by this contributor.
Expand All @@ -32,7 +41,8 @@ pub struct Contributor {
pub commits: Vec<u64>,
}

#[derive(Serialize, Deserialize, Debug, Clone, Eq)]
#[derive(Serialize, Deserialize, Debug, Clone, Eq, GraphQLObject)]
#[graphql(scalar = RustScalarValue)]
pub struct ContributorFile {
/// The file name extracted from GIT, including the relative path, e.g. `myproject/src/main.rs`
pub name: String,
Expand Down Expand Up @@ -86,7 +96,7 @@ impl Contributor {
// this is a known contributor - merge with the existing one
contributor
.name_email_pairs
.insert((commit.author_name_email.0, commit.author_name_email.1));
.insert([commit.author_name_email.0, commit.author_name_email.1]);

// only the latest version of the file is of interest
for file in commit.files {
Expand All @@ -102,8 +112,8 @@ impl Contributor {
// it's a new contributor - add as-is

// add the identities as name/email pairs
let mut name_email_pairs: HashSet<(String, String)> = HashSet::new();
name_email_pairs.insert((commit.author_name_email.0, commit.author_name_email.1));
let mut name_email_pairs: HashSet<NameEmailPairType> = HashSet::new();
name_email_pairs.insert([commit.author_name_email.0, commit.author_name_email.1]);

// collect the list of touched files with the commit SHA1
let mut touched_files: HashMap<String, (String, String, i64)> = HashMap::new();
Expand Down
165 changes: 165 additions & 0 deletions stackmuncher_lib/src/graphql.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
//! This module is needed to support the cloud-side of the project.
//! It enables GraphQL support for core structures used on the client and server sides.
use juniper::{
graphql_scalar,
parser::{ParseError, ScalarToken, Token},
serde::{de, Deserialize, Deserializer, Serialize},
InputValue, ParseScalarResult, ScalarValue, Value,
};
use std::{convert::TryInto as _, fmt};

/// An extension to the standard GraphQL set of types to include Rust scalar values.
/// Only the types used in this project are added to the list.
/// ### About GraphQL scalars
/// * https://graphql.org/learn/schema/#scalar-types
/// * https://www.graphql-tools.com/docs/scalars#custom-scalars
/// ### About extending the GraphQL scalars in Juniper
/// * https://graphql-rust.github.io/juniper/master/types/scalars.html#custom-scalars
/// * https://github.com/graphql-rust/juniper/issues/862
#[derive(Clone, Debug, PartialEq, ScalarValue, Serialize)]
#[serde(untagged)]
pub enum RustScalarValue {
/// A GraphQL scalar for i32
#[value(as_float, as_int)]
Int(i32),
/// A custom scalar for u64. The value is serialized into JSON number and should not be more than 53 bits to fit into JS Number type:
/// * Number.MAX_SAFE_INTEGER = 2^53 - 1 = 9_007_199_254_740_991
/// * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number
/// JSON spec does not constrain integer values unless specified in the schema. 53 bits is sufficient for our purposes.
U64(u64),
/// A custom scalar for i64 used in EPOCH timestamps. Theoretically, the value should never be negative because all STM dates are post 1970.
/// The value is serialized into JSON number and should not be more than 53 bits to fit into JS Number type:
/// * Number.MIN_SAFE_INTEGER = -(2^53 - 1) = -9,007,199,254,740,991
I64(i64),
/// A GraphQL scalar for f64
#[value(as_float)]
Float(f64),
/// A GraphQL scalar for String
#[value(as_str, as_string, into_string)]
String(String),
/// A GraphQL scalar for bool
#[value(as_bool)]
Boolean(bool),
}

impl<'de> Deserialize<'de> for RustScalarValue {
fn deserialize<D: Deserializer<'de>>(de: D) -> Result<Self, D::Error> {
struct Visitor;

impl<'de> de::Visitor<'de> for Visitor {
type Value = RustScalarValue;

fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("a valid input value")
}

fn visit_bool<E: de::Error>(self, b: bool) -> Result<Self::Value, E> {
Ok(RustScalarValue::Boolean(b))
}

fn visit_i32<E: de::Error>(self, n: i32) -> Result<Self::Value, E> {
Ok(RustScalarValue::Int(n))
}

fn visit_u64<E: de::Error>(self, b: u64) -> Result<Self::Value, E> {
if b <= u64::from(i32::MAX as u32) {
self.visit_i32(b.try_into().unwrap())
} else {
Ok(RustScalarValue::U64(b))
}
}

fn visit_u32<E: de::Error>(self, n: u32) -> Result<Self::Value, E> {
if n <= i32::MAX as u32 {
self.visit_i32(n.try_into().unwrap())
} else {
self.visit_u64(n.into())
}
}

fn visit_i64<E: de::Error>(self, n: i64) -> Result<Self::Value, E> {
if n <= i64::MAX as i64 {
self.visit_i64(n.try_into().unwrap())
} else {
// Browser's `JSON.stringify()` serializes all numbers
// having no fractional part as integers (no decimal point),
// so we must parse large integers as floating point,
// otherwise we would error on transferring large floating
// point numbers.
// TODO: Use `FloatToInt` conversion once stabilized:
// https://github.com/rust-lang/rust/issues/67057
Ok(RustScalarValue::Float(n as f64))
}
}

fn visit_f64<E: de::Error>(self, f: f64) -> Result<Self::Value, E> {
Ok(RustScalarValue::Float(f))
}

fn visit_str<E: de::Error>(self, s: &str) -> Result<Self::Value, E> {
self.visit_string(s.into())
}

fn visit_string<E: de::Error>(self, s: String) -> Result<Self::Value, E> {
Ok(RustScalarValue::String(s))
}
}

de.deserialize_any(Visitor)
}
}

#[graphql_scalar(with = u64_scalar, scalar = RustScalarValue)]
type U64 = u64;

mod u64_scalar {
use super::*;

pub(super) fn to_output(v: &U64) -> Value<RustScalarValue> {
Value::scalar(*v)
}

pub(super) fn from_input(v: &InputValue<RustScalarValue>) -> Result<U64, String> {
v.as_scalar_value::<u64>()
.copied()
.ok_or_else(|| format!("Expected `RustScalarValue::U64`, found: {}", v))
}

pub(super) fn parse_token(value: ScalarToken<'_>) -> ParseScalarResult<'_, RustScalarValue> {
if let ScalarToken::Int(v) = value {
v.parse()
.map_err(|_| ParseError::UnexpectedToken(Token::Scalar(value)))
.map(|s: u64| s.into())
} else {
Err(ParseError::UnexpectedToken(Token::Scalar(value)))
}
}
}

#[graphql_scalar(with = i64_scalar, scalar = RustScalarValue)]
type I64 = i64;

mod i64_scalar {
use super::*;

pub(super) fn to_output(v: &I64) -> Value<RustScalarValue> {
Value::scalar(*v)
}

pub(super) fn from_input(v: &InputValue<RustScalarValue>) -> Result<I64, String> {
v.as_scalar_value::<i64>()
.copied()
.ok_or_else(|| format!("Expected `RustScalarValue::I64`, found: {}", v))
}

pub(super) fn parse_token(value: ScalarToken<'_>) -> ParseScalarResult<'_, RustScalarValue> {
if let ScalarToken::Int(v) = value {
v.parse()
.map_err(|_| ParseError::UnexpectedToken(Token::Scalar(value)))
.map(|s: i64| s.into())
} else {
Err(ParseError::UnexpectedToken(Token::Scalar(value)))
}
}
}
1 change: 1 addition & 0 deletions stackmuncher_lib/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ pub mod config;
pub mod contributor;
pub mod file_type;
pub mod git;
pub mod graphql;
mod ignore_paths;
pub mod muncher;
pub mod processors;
Expand Down
8 changes: 6 additions & 2 deletions stackmuncher_lib/src/report/commit_time_histo.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
use super::Report;
use crate::graphql::RustScalarValue;
use chrono::{self, Duration, TimeZone, Timelike, Utc};
use juniper::GraphQLObject;
use serde::{Deserialize, Serialize};
use tracing::warn;

Expand All @@ -8,7 +10,8 @@ pub const RECENT_PERIOD_LENGTH_IN_DAYS: i64 = 365;

/// Number of commits or percentage of commits per UTC hour.
/// The structure is skipped in JSON if all values are zero and is initialized to all zeros to have fewer Option<T> unwraps.
#[derive(Serialize, Deserialize, Clone, Debug)]
#[derive(Serialize, Deserialize, Clone, Debug, GraphQLObject)]
#[graphql(scalar = RustScalarValue)]
pub struct CommitTimeHistoHours {
#[serde(skip_serializing_if = "CommitTimeHistoHours::is_zero", default = "u64::default")]
pub h00: u64,
Expand Down Expand Up @@ -61,7 +64,8 @@ pub struct CommitTimeHistoHours {
}

/// Contains members and methods related to commit time histogram
#[derive(Serialize, Deserialize, Clone, Debug)]
#[derive(Serialize, Deserialize, Clone, Debug, GraphQLObject)]
#[graphql(scalar = RustScalarValue)]
pub struct CommitTimeHisto {
/// The sum of all commits included in `histogram_recent`. This value is used as the 100% of all recent commits.
/// The value is populated once after all commits have been added.
Expand Down
5 changes: 4 additions & 1 deletion stackmuncher_lib/src/report/kwc.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
use crate::graphql::RustScalarValue;
use juniper::GraphQLObject;
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
use tracing::{error, warn};

#[derive(Debug, Serialize, Deserialize, Eq, Clone)]
#[derive(Debug, Serialize, Deserialize, Eq, Clone, GraphQLObject)]
#[graphql(scalar = RustScalarValue)]
pub struct KeywordCounter {
/// keyword
pub k: String,
Expand Down
2 changes: 1 addition & 1 deletion stackmuncher_lib/src/report/mod.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
pub mod commit_time_histo;
pub mod kwc;
pub mod overview;
pub mod report;
pub mod tech;
pub mod commit_time_histo;

pub use overview::{ProjectReportOverview, TechOverview};
pub use report::Report;
Expand Down
8 changes: 6 additions & 2 deletions stackmuncher_lib/src/report/overview.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
use super::tech::Tech;
use crate::graphql::RustScalarValue;
use chrono::{DateTime, Datelike, Timelike, Utc};
use juniper::GraphQLObject;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use tracing::warn;

/// A very concise overview of a single Tech record
/// to show the share of the technology in the project
#[derive(Serialize, Deserialize, Clone, Debug, Eq)]
#[derive(Serialize, Deserialize, Clone, Debug, Eq, GraphQLObject)]
#[graphql(scalar = RustScalarValue)]
pub struct TechOverview {
/// The same as Tech.language
pub language: String,
Expand Down Expand Up @@ -36,7 +39,8 @@ impl PartialEq for TechOverview {

/// An overview of an individual project report included in the combined report
/// to avoid loading the full project report every time the combined report is looked at.
#[derive(Serialize, Deserialize, Clone, Debug, Eq)]
#[derive(Serialize, Deserialize, Clone, Debug, Eq, GraphQLObject)]
#[graphql(scalar = RustScalarValue)]
pub struct ProjectReportOverview {
/// A human-readable project name. It should not be used as an ID.
#[serde(default = "String::new")]
Expand Down
8 changes: 6 additions & 2 deletions stackmuncher_lib/src/report/report.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,13 @@ use super::commit_time_histo::CommitTimeHisto;
use super::kwc::{KeywordCounter, KeywordCounterSet};
use super::tech::{Tech, TechHistory};
use super::ProjectReportOverview;
use crate::graphql::RustScalarValue;
use crate::utils::sha256::hash_str_to_sha256_as_base58;
use crate::{contributor::Contributor, git::GitLogEntry, utils};
use chrono::{DateTime, Utc};
use flate2::write::GzEncoder;
use flate2::Compression;
use juniper::GraphQLObject;
use path_absolutize::{self, Absolutize};
use serde::{Deserialize, Serialize};
use serde_json;
Expand All @@ -18,7 +20,8 @@ use tracing::{debug, error, info, warn};

/// Contains the number of elements per list to help with DB queries.
/// The numbers are calculated once before saving the Report in the DB.
#[derive(Serialize, Deserialize, Clone, Debug)]
#[derive(Serialize, Deserialize, Clone, Debug, GraphQLObject)]
#[graphql(scalar = RustScalarValue)]
pub struct ListCounts {
tech: u64,
contributor_git_ids: u64,
Expand All @@ -33,7 +36,8 @@ pub struct ListCounts {
keywords: u64,
}

#[derive(Serialize, Deserialize, Clone, Debug)]
#[derive(Serialize, Deserialize, Clone, Debug, GraphQLObject)]
#[graphql(scalar = RustScalarValue)]
#[serde(rename = "tech")]
pub struct Report {
/// The exact timestamp of the report generation in ISO3389 format.
Expand Down
8 changes: 6 additions & 2 deletions stackmuncher_lib/src/report/tech.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
use super::kwc::{KeywordCounter, KeywordCounterSet};
use crate::graphql::RustScalarValue;
use juniper::GraphQLObject;
use regex::Regex;
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
use tracing::{debug, trace, warn};

/// Contains time-range data for its parent Tech.
#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Clone)]
#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Clone, GraphQLObject)]
#[graphql(scalar = RustScalarValue)]
#[serde(rename = "tech")]
pub struct TechHistory {
/// Number of months between the first and the last commit.
Expand All @@ -24,7 +27,8 @@ pub struct TechHistory {
/// Any additions to this struct should be considered for clean up before submission to stackmuncher.com
/// to avoid sending out any info that doesn't need to be sent.
/// See https://github.com/stackmuncher/stm_app/issues/12
#[derive(Serialize, Deserialize, Debug, Eq, Clone)]
#[derive(Serialize, Deserialize, Debug, Eq, Clone, GraphQLObject)]
#[graphql(scalar = RustScalarValue)]
#[serde(rename = "tech")]
pub struct Tech {
/// The name of the file for individual file reports. Not present in combined tech reports.
Expand Down

0 comments on commit 3d620c4

Please sign in to comment.