Skip to content

Commit

Permalink
Switched from black to ruff.
Browse files Browse the repository at this point in the history
  • Loading branch information
coady committed Dec 22, 2023
1 parent b174c06 commit 47371bd
Show file tree
Hide file tree
Showing 17 changed files with 236 additions and 226 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ jobs:
- uses: actions/setup-python@v5
with:
python-version: 3.x
- run: pip install black ruff mypy
- run: pip install ruff mypy
- run: make lint

docs:
Expand Down
2 changes: 0 additions & 2 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,3 @@ jobs:
- run: make html
- run: PYTHONPATH=$PWD python -m mkdocs gh-deploy --force
- uses: pypa/gh-action-pypi-publish@release/v1
with:
verbose: true
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ check:
python -m pytest -s --cov

lint:
black --check .
ruff .
ruff format --check .
mypy -p graphique

html: docs/schema.md
Expand Down
1 change: 0 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
[![image](https://github.com/coady/graphique/workflows/build/badge.svg)](https://github.com/coady/graphique/actions)
[![image](https://codecov.io/gh/coady/graphique/branch/main/graph/badge.svg)](https://codecov.io/gh/coady/graphique/)
[![image](https://github.com/coady/graphique/workflows/codeql/badge.svg)](https://github.com/coady/graphique/security/code-scanning)
[![image](https://img.shields.io/badge/code%20style-black-000000.svg)](https://pypi.org/project/black/)
[![image](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
[![image](https://mypy-lang.org/static/mypy_badge.svg)](https://mypy-lang.org/)

Expand Down
16 changes: 8 additions & 8 deletions docs/examples.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -37,15 +37,15 @@
"outputs": [],
"source": [
"execute(\n",
" '''{\n",
" \"\"\"{\n",
" length\n",
" schema {\n",
" names\n",
" types\n",
" partitioning\n",
" index\n",
" }\n",
"}'''\n",
"}\"\"\"\n",
")"
]
},
Expand Down Expand Up @@ -103,7 +103,7 @@
"outputs": [],
"source": [
"execute(\n",
" '''{\n",
" \"\"\"{\n",
" filter(state: {eq: \"CA\"}) {\n",
" group(by: \"county\", aggregate: {countDistinct: {name: \"city\", alias: \"cities\"}}) {\n",
" sort(by: \"-cities\", length: 5) {\n",
Expand All @@ -120,7 +120,7 @@
" }\n",
" }\n",
" }\n",
"}'''\n",
"}\"\"\"\n",
")"
]
},
Expand All @@ -141,7 +141,7 @@
"outputs": [],
"source": [
"execute(\n",
" '''{\n",
" \"\"\"{\n",
" scan(filter: {eq: [{name: \"county\"}, {name: \"city\"}]}) {\n",
" columns {\n",
" state {\n",
Expand All @@ -152,7 +152,7 @@
" }\n",
" }\n",
" }\n",
"}'''\n",
"}\"\"\"\n",
")"
]
},
Expand All @@ -177,7 +177,7 @@
"outputs": [],
"source": [
"execute(\n",
" '''{\n",
" \"\"\"{\n",
" scan(columns: {alias: \"match\", eq: [{name: \"county\"}, {name: \"city\"}]}) {\n",
" group(by: \"state\", aggregate: {any: {name: \"match\"}}) {\n",
" scan(filter: {inv: {name: \"match\"}}) {\n",
Expand All @@ -189,7 +189,7 @@
" }\n",
" }\n",
" }\n",
"}'''\n",
"}\"\"\"\n",
")"
]
}
Expand Down
6 changes: 5 additions & 1 deletion graphique/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
Arrow forbids subclassing, so the classes are for logical grouping.
Their methods are called as functions.
"""

import bisect
import contextlib
import functools
Expand Down Expand Up @@ -106,7 +107,10 @@ def register(func: Callable, kind: str = 'scalar') -> pc.Function:

@register
def digitize(
ctx, array: pa.float64(), bins: pa.list_(pa.float64()), right: pa.bool_() # type: ignore
ctx,
array: pa.float64(), # type: ignore
bins: pa.list_(pa.float64()), # type: ignore
right: pa.bool_(), # type: ignore
) -> pa.int64(): # type: ignore
"""Return the indices of the bins to which each value in input array belongs."""
return pa.array(np.digitize(array, bins.values, right.as_py()))
Expand Down
1 change: 1 addition & 0 deletions graphique/inputs.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""
GraphQL input types.
"""

import functools
import inspect
import operator
Expand Down
11 changes: 7 additions & 4 deletions graphique/interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
Doesn't require knowledge of the schema.
"""

# mypy: disable-error-code=valid-type
import collections
import inspect
Expand Down Expand Up @@ -234,7 +235,11 @@ def slice(
aggregate="aggregation functions applied to other columns",
)
def group(
self, info: Info, by: list[str] = [], counts: str = '', aggregate: HashAggregates = {} # type: ignore
self,
info: Info,
by: list[str] = [],
counts: str = '',
aggregate: HashAggregates = {}, # type: ignore
) -> Self:
"""Return table grouped by columns.
Expand Down Expand Up @@ -497,9 +502,7 @@ def oneshot(cls, info: Info, scanner: ds.Scanner) -> Union[ds.Scanner, pa.Table]
return scanner

@doc_field(filter="selected rows", columns="projected columns")
def scan(
self, info: Info, filter: Expression = {}, columns: list[Projection] = [] # type: ignore
) -> Self:
def scan(self, info: Info, filter: Expression = {}, columns: list[Projection] = []) -> Self: # type: ignore
"""Select rows and project columns without memory usage."""
expr = filter.to_arrow()
if expr is not None and not columns and isinstance(self.table, ds.Dataset):
Expand Down
1 change: 1 addition & 0 deletions graphique/middleware.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""
ASGI GraphQL utilities.
"""

import warnings
from collections.abc import Iterable, Mapping
from datetime import timedelta
Expand Down
4 changes: 2 additions & 2 deletions graphique/models.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""
GraphQL output types and resolvers.
"""

import collections
import functools
import inspect
Expand Down Expand Up @@ -174,8 +175,7 @@ def fill_null(self, value: T) -> list[T]:

@Column.register(str)
@strawberry.type(name='ingColumn', description="column of strings")
class StringColumn(OrdinalColumn[T]):
...
class StringColumn(OrdinalColumn[T]): ...


@strawberry.type
Expand Down
1 change: 1 addition & 0 deletions graphique/scalars.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""
GraphQL scalars.
"""

from datetime import date, datetime, time, timedelta
from decimal import Decimal
import pyarrow as pa
Expand Down
1 change: 1 addition & 0 deletions graphique/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
* datasets, scanners, and tables
* filtering and projection
"""

import json
from pathlib import Path
import pyarrow.dataset as ds
Expand Down
1 change: 1 addition & 0 deletions graphique/shell.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
sufficient once partitioned, but there is a `fragments` option to optimize for memory or show
progress on the second pass.
"""

import argparse
import shutil
from pathlib import Path
Expand Down
9 changes: 5 additions & 4 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,13 @@ version = {attr = "graphique.__version__"}
[tool.setuptools.package-data]
graphique = ["py.typed"]

[tool.black]
[tool.ruff]
line-length = 100
skip-string-normalization = true
extend-include = ["*.ipynb"]

[tool.ruff]
ignore = ["E501"]
[tool.ruff.format]
preview = true
quote-style = "preserve"

[[tool.mypy.overrides]]
module = ["pyarrow.*", "strawberry.*", "starlette.*"]
Expand Down
70 changes: 35 additions & 35 deletions tests/test_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,18 +40,18 @@ def test_search(dsclient):
data = dsclient.execute('{ filter(zipcode: {}) { row { zipcode } } }')
assert data == {'filter': {'row': {'zipcode': 501}}}
data = dsclient.execute(
'''{ filter(zipcode: {gt: 90000}) { filter(state: {eq: "CA"}) {
length } } }'''
"""{ filter(zipcode: {gt: 90000}) { filter(state: {eq: "CA"}) {
length } } }"""
)
assert data == {'filter': {'filter': {'length': 2647}}}
data = dsclient.execute(
'''{ filter(zipcode: {gt: 90000}) { filter(state: {eq: "CA"}) {
length row { zipcode } } } }'''
"""{ filter(zipcode: {gt: 90000}) { filter(state: {eq: "CA"}) {
length row { zipcode } } } }"""
)
assert data == {'filter': {'filter': {'length': 2647, 'row': {'zipcode': 90001}}}}
data = dsclient.execute(
'''{ filter(zipcode: {lt: 90000}) { filter(state: {eq: "CA"}) {
group(by: "county") { length } } } }'''
"""{ filter(zipcode: {lt: 90000}) { filter(state: {eq: "CA"}) {
group(by: "county") { length } } } }"""
)
assert data == {'filter': {'filter': {'group': {'length': 0}}}}

Expand All @@ -75,12 +75,12 @@ def test_slice(dsclient):

def test_group(dsclient):
data = dsclient.execute(
'''{ group(by: ["state"], aggregate: {min: {name: "county"}}) { row { state county } } }'''
"""{ group(by: ["state"], aggregate: {min: {name: "county"}}) { row { state county } } }"""
)
assert data == {'group': {'row': {'state': 'NY', 'county': 'Albany'}}}
data = dsclient.execute(
'''{ group(by: ["state"], counts: "c") { slice(length: 1) {
column(name: "c") { ... on LongColumn { values } } } } }'''
"""{ group(by: ["state"], counts: "c") { slice(length: 1) {
column(name: "c") { ... on LongColumn { values } } } } }"""
)
assert data == {'group': {'slice': {'column': {'values': [2205]}}}}
data = dsclient.execute(
Expand All @@ -92,34 +92,34 @@ def test_group(dsclient):
)
assert data['group']['row']['county']
data = dsclient.execute(
'''{ group(by: ["state"], aggregate: {mean: {name: "zipcode"}}) { slice(length: 1) {
column(name: "zipcode") { ... on FloatColumn { values } } } } }'''
"""{ group(by: ["state"], aggregate: {mean: {name: "zipcode"}}) { slice(length: 1) {
column(name: "zipcode") { ... on FloatColumn { values } } } } }"""
)
assert data == {'group': {'slice': {'column': {'values': [pytest.approx(12614.62721)]}}}}
data = dsclient.execute(
'''{ group(by: ["state"], aggregate: {list: {name: "zipcode"}}) { aggregate(mean: {name: "zipcode"}) {
slice(length: 1) { column(name: "zipcode") { ... on FloatColumn { values } } } } } }'''
"""{ group(by: ["state"], aggregate: {list: {name: "zipcode"}}) { aggregate(mean: {name: "zipcode"}) {
slice(length: 1) { column(name: "zipcode") { ... on FloatColumn { values } } } } } }"""
)
assert data == {
'group': {'aggregate': {'slice': {'column': {'values': [pytest.approx(12614.62721)]}}}}
}
data = dsclient.execute(
'''{ group(aggregate: {min: {alias: "st", name: "state"}}) {
column(name: "st") { ... on StringColumn { values } } } }'''
"""{ group(aggregate: {min: {alias: "st", name: "state"}}) {
column(name: "st") { ... on StringColumn { values } } } }"""
)
assert data == {'group': {'column': {'values': ['AK']}}}


def test_list(partclient):
data = partclient.execute(
'''{ group(by: "state", aggregate: {distinct: {alias: "counties", name: "county"}}) {
tables { row { state } column(name: "counties") { length } } } } '''
"""{ group(by: "state", aggregate: {distinct: {alias: "counties", name: "county"}}) {
tables { row { state } column(name: "counties") { length } } } } """
)
(table,) = [table for table in data['group']['tables'] if table['row']['state'] == 'PR']
assert table == {'row': {'state': 'PR'}, 'column': {'length': 78}}
data = partclient.execute(
'''{ group(by: "north", aggregate: {distinct: {name: "west"}}) {
tables { row { north } columns { west { length } } } } }'''
"""{ group(by: "north", aggregate: {distinct: {name: "west"}}) {
tables { row { north } columns { west { length } } } } }"""
)
tables = data['group']['tables']
assert {table['row']['north'] for table in tables} == {0, 1}
Expand Down Expand Up @@ -147,13 +147,13 @@ def test_fragments(partclient):
)
assert data == {'group': {'length': 1, 'row': {'state': 'AK'}}}
data = partclient.execute(
'''{ group(by: ["north", "west"], aggregate: {distinct: {name: "city"}, mean: {name: "zipcode"}}) {
length column(name: "city") { type } } }'''
"""{ group(by: ["north", "west"], aggregate: {distinct: {name: "city"}, mean: {name: "zipcode"}}) {
length column(name: "city") { type } } }"""
)
assert data == {'group': {'length': 4, 'column': {'type': 'large_list<item: string>'}}}
data = partclient.execute(
'''{ group(by: "north", aggregate: {countDistinct: {name: "west"}}) {
column(name: "west") { ... on LongColumn { values } } } }'''
"""{ group(by: "north", aggregate: {countDistinct: {name: "west"}}) {
column(name: "west") { ... on LongColumn { values } } } }"""
)
assert data == {'group': {'column': {'values': [2, 2]}}}
data = partclient.execute(
Expand Down Expand Up @@ -204,14 +204,14 @@ def test_scan(dsclient):
with pytest.raises(ValueError, match="name or alias"):
dsclient.execute('{ scan(columns: {}) { length } }')
data = dsclient.execute(
'''{ scan(filter: {eq: [{name: "state"}, {value: "CA"}]})
"""{ scan(filter: {eq: [{name: "state"}, {value: "CA"}]})
{ scan(filter: {eq: [{name: "county"}, {value: "Santa Clara"}]})
{ length row { county } } } }'''
{ length row { county } } } }"""
)
assert data == {'scan': {'scan': {'length': 108, 'row': {'county': 'Santa Clara'}}}}
data = dsclient.execute(
'''{ scan(filter: {or: [{eq: [{name: "state"}, {value: "CA"}]},
{eq: [{name: "county"}, {value: "Santa Clara"}]}]}) { length } }'''
"""{ scan(filter: {or: [{eq: [{name: "state"}, {value: "CA"}]},
{eq: [{name: "county"}, {value: "Santa Clara"}]}]}) { length } }"""
)
assert data == {'scan': {'length': 2647}}

Expand Down Expand Up @@ -256,30 +256,30 @@ def test_federation(fedclient):
assert data['zipDb'] == {'__typename': 'ZipDbTable', 'length': 42724}

data = fedclient.execute(
'''{ zipcodes { scan(columns: {name: "zipcode", cast: "int64"}) {
join(right: "zip_db", keys: "zipcode", rightKeys: "zip") { length schema { names } } } } }'''
"""{ zipcodes { scan(columns: {name: "zipcode", cast: "int64"}) {
join(right: "zip_db", keys: "zipcode", rightKeys: "zip") { length schema { names } } } } }"""
)
table = data['zipcodes']['scan']['join']
assert table['length'] == 41700
assert set(table['schema']['names']) > {'zipcode', 'timezone', 'latitude'}
data = fedclient.execute(
'''{ zipcodes { scan(columns: {alias: "zip", name: "zipcode", cast: "int64"}) {
join(right: "zip_db", keys: "zip", joinType: "right outer") { length schema { names } } } } }'''
"""{ zipcodes { scan(columns: {alias: "zip", name: "zipcode", cast: "int64"}) {
join(right: "zip_db", keys: "zip", joinType: "right outer") { length schema { names } } } } }"""
)
table = data['zipcodes']['scan']['join']
assert table['length'] == 42724
assert set(table['schema']['names']) > {'zip', 'timezone', 'latitude'}

data = fedclient.execute(
'''{ _entities(representations: {__typename: "ZipcodesTable", zipcode: 90001}) {
... on ZipcodesTable { length row { state } schema { names } } } }'''
"""{ _entities(representations: {__typename: "ZipcodesTable", zipcode: 90001}) {
... on ZipcodesTable { length row { state } schema { names } } } }"""
)
assert data == {
'_entities': [{'length': 1, 'row': {'state': 'CA'}, 'schema': {'names': ['state']}}]
}
data = fedclient.execute(
'''{ states { filter(state: {eq: "CA"}) { columns { indices {
takeFrom(field: "zipcodes") { __typename column(name: "state") { length } } } } } } }'''
"""{ states { filter(state: {eq: "CA"}) { columns { indices {
takeFrom(field: "zipcodes") { __typename column(name: "state") { length } } } } } } }"""
)
table = data['states']['filter']['columns']['indices']['takeFrom']
assert table == {'__typename': 'ZipcodesTable', 'column': {'length': 2647}}
Expand Down
Loading

0 comments on commit 47371bd

Please sign in to comment.