Skip to content

Commit

Permalink
Improve "BETWEEN" clause formatting
Browse files Browse the repository at this point in the history 
Fixes #17
  • Loading branch information
@shssoichiro
shssoichiro committed Jan 13, 2023
1 parent 3db4f02 commit 69f2d19
Show file tree
Hide file tree
Showing 2 changed files with 65 additions and 44 deletions.
5 changes: 3 additions & 2 deletions src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -627,7 +627,8 @@ mod tests {
#[test]
fn it_formats_and_or_operators() {
let strings = [
("foo BETWEEN bar AND baz", "foo BETWEEN bar\nAND baz"),
("foo BETWEEN bar AND baz", "foo BETWEEN bar AND baz"),
("foo BETWEEN\nbar\nAND baz", "foo BETWEEN bar AND baz"),
("foo AND bar", "foo\nAND bar"),
("foo OR bar", "foo\nOR bar"),
];
Expand Down Expand Up @@ -832,7 +833,7 @@ mod tests {
#[test]
fn it_formats_insert_without_into() {
let input =
"INSERT Customers (ID, MoneyBalance, Address, City) VALUES (12,-123.4, 'Skagen 2111','Stv');";
"INSERT Customers (ID, MoneyBalance, Address, City) VALUES (12,-123.4, 'Skagen 2111','Stv');";
let options = FormatOptions::default();
let expected = indoc!(
"
Expand Down
104 changes: 62 additions & 42 deletions src/tokenizer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,17 @@ use std::borrow::Cow;
use unicode_categories::UnicodeCategories;

pub(crate) fn tokenize(mut input: &str) -> Vec<Token<'_>> {
let mut tokens = Vec::new();
let mut token = None;
let mut tokens: Vec<Token> = Vec::new();

// Keep processing the string until it is empty
while let Ok(result) = get_next_token(input, token.as_ref()) {
token = Some(result.1.clone());
while let Ok(result) = get_next_token(
input,
tokens.last().cloned(),
tokens
.iter()
.rfind(|token| token.kind == TokenKind::Reserved)
.cloned(),
) {
input = result.0;

tokens.push(result.1);
Expand Down Expand Up @@ -76,7 +81,8 @@ impl<'a> PlaceholderKind<'a> {

fn get_next_token<'a>(
input: &'a str,
previous_token: Option<&Token<'a>>,
previous_token: Option<Token<'a>>,
last_reserved_token: Option<Token<'a>>,
) -> IResult<&'a str, Token<'a>> {
get_whitespace_token(input)
.or_else(|_| get_comment_token(input))
Expand All @@ -85,7 +91,7 @@ fn get_next_token<'a>(
.or_else(|_| get_close_paren_token(input))
.or_else(|_| get_placeholder_token(input))
.or_else(|_| get_number_token(input))
.or_else(|_| get_reserved_word_token(input, previous_token))
.or_else(|_| get_reserved_word_token(input, previous_token, last_reserved_token))
.or_else(|_| get_word_token(input))
.or_else(|_| get_operator_token(input))
}
Expand Down Expand Up @@ -394,7 +400,8 @@ fn scientific_notation(input: &str) -> IResult<&str, &str> {

fn get_reserved_word_token<'a>(
input: &'a str,
previous_token: Option<&Token<'a>>,
previous_token: Option<Token<'a>>,
last_reserved_token: Option<Token<'a>>,
) -> IResult<&'a str, Token<'a>> {
// A reserved word cannot be preceded by a "."
// this makes it so in "my_table.from", "from" is not considered a reserved word
Expand All @@ -406,7 +413,7 @@ fn get_reserved_word_token<'a>(

alt((
get_top_level_reserved_token,
get_newline_reserved_token,
get_newline_reserved_token(last_reserved_token),
get_top_level_reserved_token_no_indent,
get_plain_reserved_token,
))(input)
Expand Down Expand Up @@ -469,40 +476,53 @@ fn get_top_level_reserved_token(input: &str) -> IResult<&str, Token<'_>> {
}
}

fn get_newline_reserved_token(input: &str) -> IResult<&str, Token<'_>> {
let uc_input = get_uc_words(input, 3);
let result: IResult<&str, &str> = alt((
terminated(tag("AND"), end_of_word),
terminated(tag("CROSS APPLY"), end_of_word),
terminated(tag("CROSS JOIN"), end_of_word),
terminated(tag("ELSE"), end_of_word),
terminated(tag("INNER JOIN"), end_of_word),
terminated(tag("JOIN"), end_of_word),
terminated(tag("LEFT JOIN"), end_of_word),
terminated(tag("LEFT OUTER JOIN"), end_of_word),
terminated(tag("OR"), end_of_word),
terminated(tag("OUTER APPLY"), end_of_word),
terminated(tag("OUTER JOIN"), end_of_word),
terminated(tag("RIGHT JOIN"), end_of_word),
terminated(tag("RIGHT OUTER JOIN"), end_of_word),
terminated(tag("WHEN"), end_of_word),
terminated(tag("XOR"), end_of_word),
))(&uc_input);
if let Ok((_, token)) = result {
let final_word = token.split(' ').last().unwrap();
let input_end_pos =
input.to_ascii_uppercase().find(&final_word).unwrap() + final_word.len();
let (token, input) = input.split_at(input_end_pos);
Ok((
input,
Token {
kind: TokenKind::ReservedNewline,
value: token,
key: None,
},
))
} else {
Err(Err::Error(Error::new(input, ErrorKind::Alt)))
fn get_newline_reserved_token<'a>(
last_reserved_token: Option<Token<'a>>,
) -> impl FnMut(&'a str) -> IResult<&'a str, Token<'a>> {
move |input: &'a str| {
let uc_input = get_uc_words(input, 3);
let result: IResult<&str, &str> = alt((
terminated(tag("AND"), end_of_word),
terminated(tag("CROSS APPLY"), end_of_word),
terminated(tag("CROSS JOIN"), end_of_word),
terminated(tag("ELSE"), end_of_word),
terminated(tag("INNER JOIN"), end_of_word),
terminated(tag("JOIN"), end_of_word),
terminated(tag("LEFT JOIN"), end_of_word),
terminated(tag("LEFT OUTER JOIN"), end_of_word),
terminated(tag("OR"), end_of_word),
terminated(tag("OUTER APPLY"), end_of_word),
terminated(tag("OUTER JOIN"), end_of_word),
terminated(tag("RIGHT JOIN"), end_of_word),
terminated(tag("RIGHT OUTER JOIN"), end_of_word),
terminated(tag("WHEN"), end_of_word),
terminated(tag("XOR"), end_of_word),
))(&uc_input);
if let Ok((_, token)) = result {
let final_word = token.split(' ').last().unwrap();
let input_end_pos =
input.to_ascii_uppercase().find(&final_word).unwrap() + final_word.len();
let (token, input) = input.split_at(input_end_pos);
let kind = if token == "AND"
&& last_reserved_token.is_some()
&& last_reserved_token.as_ref().unwrap().value == "BETWEEN"
{
// If the "AND" is part of a "BETWEEN" clause, we want to handle it as one clause by not adding a new line.
TokenKind::Reserved
} else {
TokenKind::ReservedNewline
};
Ok((
input,
Token {
kind,
value: token,
key: None,
},
))
} else {
Err(Err::Error(Error::new(input, ErrorKind::Alt)))
}
}
}

Expand Down

0 comments on commit 69f2d19

Please sign in to comment.