Skip to content
This repository has been archived by the owner on Oct 5, 2024. It is now read-only.

Commit

Permalink
Merge branch 'master' of github.com:hotosm/underpass
Browse files Browse the repository at this point in the history 
Sync with upstream
  • Loading branch information
@rsavoye
rsavoye committed Jun 14, 2024
2 parents 920b46b + 224fc2a commit d02341c
Show file tree
Hide file tree
Showing 7 changed files with 42 additions and 39 deletions.
4 changes: 2 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@ It **updates a local copy of the OSM database** in near real-time, and provides

## Demo

We've deployed a rudimentary demo that keeps a database up-to-date for (some country),
We've deployed a basic demo that keeps a database up-to-date for (some country),
rendering buildings and highlighting the ones identified as "un-squared":

[https://underpass.live](https://underpass.live)

<img width="993" alt="Screenshot 2023-11-22 at 10 32 56" src="https://github.com/hotosm/underpass/assets/1226194/73622131-b69f-4716-bf95-9e195efdbba9">
<img width="1160" alt="Screenshot 2024-06-05 at 15 51 57" src="https://github.com/hotosm/underpass/assets/1226194/8a95e518-a12f-45c3-b460-d74d81ca3cfe">

## Getting started

Expand Down
4 changes: 2 additions & 2 deletions m4/ax_boost_timer.m4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,8 @@ AC_DEFUN([AX_BOOST_TIMER],
[AC_LANG_PUSH([C++])
CXXFLAGS_SAVE=$CXXFLAGS
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include <boost/timer.hpp>]],
[[boost::timer timer;]])],
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include <boost/timer/timer.hpp>]],
[[boost::timer::auto_cpu_timer t;]])],
ax_cv_boost_timer=yes, ax_cv_boost_timer=no)
CXXFLAGS=$CXXFLAGS_SAVE
AC_LANG_POP([C++])
Expand Down
18 changes: 9 additions & 9 deletions python/dbapi/api/db.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,21 +58,21 @@ async def run(self, query, singleObject = False, asJson=False):
if not self.pool:
await self.connect()
if self.pool:
result = None
try:
conn = await self.pool.acquire()
result = await conn.fetch(query)
data = None
if asJson:
if singleObject:
return result[0]['result']
return result[0]['result']
data = result[0]['result']
elif singleObject:
data = result[0]
else:
if singleObject:
return result[0]
return result
data = result
await self.pool.release(conn)
return data
except Exception as e:
print("\n******* \n" + query + "\n******* \n")
print(e)
return None
finally:
await self.pool.release(conn)
return None

26 changes: 13 additions & 13 deletions python/dbapi/api/raw.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@

# Order by
class OrderBy(Enum):
createdAt = "created_at"
closedAt = "closed_at"
id = "id"
timestamp = "timestamp"

Expand Down Expand Up @@ -73,7 +73,7 @@ def geoFeaturesQuery(params: RawFeaturesParamsDTO, asJson: bool = False):
tags, \n \
hashtags, \n \
editor, \n \
created_at \n \
closed_at \n \
FROM {table} \n \
LEFT JOIN changesets c ON c.id = {table}.changeset \n \
WHERE{area}{tags}{hashtag}{date} {limit}; \n \
Expand All @@ -84,7 +84,7 @@ def geoFeaturesQuery(params: RawFeaturesParamsDTO, asJson: bool = False):
.format(area=params.area) if params.area else "",
tags=" AND (" + tagsQueryFilter(params.tags, params.table.value) + ") \n" if params.tags else "",
hashtag=" AND " + hashtagQueryFilter(params.hashtag, params.table.value) if params.hashtag else "",
date=" AND created_at >= {dateFrom} AND created_at <= {dateTo}\n"
date=" AND closed_at >= {dateFrom} AND closed_at <= {dateTo}\n"
.format(dateFrom=params.dateFrom, dateTo=params.dateTo)
if params.dateFrom and params.dateTo else "\n",
limit=" LIMIT {limit}".format(limit=RESULTS_PER_PAGE)
Expand Down Expand Up @@ -115,7 +115,7 @@ def listFeaturesQuery(
{table}.timestamp, \n \
tags, \n \
{table}.changeset, \n \
c.created_at \n \
c.closed_at \n \
FROM {table} \n \
LEFT JOIN changesets c ON c.id = {table}.changeset \n \
WHERE{fromDate}{toDate}{hashtag}{area}{tags}{order} \
Expand All @@ -124,8 +124,8 @@ def listFeaturesQuery(
type=osmType.value,
geotype=geoType.value,
table=table.value,
fromDate=" AND created_at >= '{dateFrom}'".format(dateFrom=params.dateFrom) if (params.dateFrom) else "",
toDate=" AND created_at <= '{dateTo}'".format(dateTo=params.dateTo) if (params.dateTo) else "",
fromDate=" AND closed_at >= '{dateFrom}'".format(dateFrom=params.dateFrom) if (params.dateFrom) else "",
toDate=" AND closed_at <= '{dateTo}'".format(dateTo=params.dateTo) if (params.dateTo) else "",
hashtag=" AND " + hashtagQueryFilter(params.hashtag, table.value) if params.hashtag else "",
area=" AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({area})))', 4326) )"
.format(
Expand All @@ -137,7 +137,7 @@ def listFeaturesQuery(
order=params.orderBy.value,
limit=RESULTS_PER_PAGE_LIST,
offset=params.page * RESULTS_PER_PAGE_LIST
) if params.page else ""
) if params.page is not None else " LIMIT {limit} OFFSET {offset}"
).replace("WHERE AND", "WHERE")
if asJson:
return listQueryToJSON(query, params)
Expand All @@ -152,7 +152,7 @@ def listQueryToJSON(query: str, params: ListFeaturesParamsDTO):
predata.timestamp, \n \
tags, \n \
predata.changeset, \n \
predata.created_at as created_at, \n \
predata.closed_at as closed_at, \n \
lat, \n \
lon \n \
from predata \n \
Expand All @@ -163,10 +163,10 @@ def listQueryToJSON(query: str, params: ListFeaturesParamsDTO):
) SELECT jsonb_agg(t_features.feature) as result FROM t_features;" \
.format(
query=query,
date="created_at >= '{dateFrom}' AND created_at <= '{dateTo}'"
date="closed_at >= '{dateFrom}' AND closed_at <= '{dateTo}'"
.format(
dateFrom=params.dateFrom if (params.dateFrom) else "",
dateTo=" AND created_at <= '{dateTo}'".format(dateTo=params.dateTo) if (params.dateTo) else ""
dateTo=" AND closed_at <= '{dateTo}'".format(dateTo=params.dateTo) if (params.dateTo) else ""
) if params.dateFrom and params.dateTo else "",
orderBy=" AND {orderBy} IS NOT NULL ORDER BY {orderBy} DESC"
.format(
Expand Down Expand Up @@ -238,7 +238,7 @@ async def getPolygons(
params.table = Table.polygons
result = await self.db.run(geoFeaturesQuery(params, asJson), asJson=asJson)
if asJson:
return result
return result or {}
return deserializeTags(result)

# Get line features
Expand All @@ -250,7 +250,7 @@ async def getLines(
params.table = Table.lines
result = await self.db.run(geoFeaturesQuery(params, asJson), asJson=asJson)
if asJson:
return result
return result or {}
return deserializeTags(result)


Expand All @@ -263,7 +263,7 @@ async def getNodes(
params.table = Table.nodes
result = await self.db.run(geoFeaturesQuery(params, asJson), asJson=asJson)
if asJson:
return result
return result or {}
return deserializeTags(result)

# Get all (polygon, line, node) features
Expand Down
21 changes: 11 additions & 10 deletions python/dbapi/api/rawValidation.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
from .filters import tagsQueryFilter, hashtagQueryFilter
from .serialization import queryToJSON
from .config import RESULTS_PER_PAGE, RESULTS_PER_PAGE_LIST, DEBUG
from .raw import RawFeaturesParamsDTO, ListFeaturesParamsDTO, rawQueryToJSON, listQueryToJSON
from .raw import RawFeaturesParamsDTO, ListFeaturesParamsDTO, rawQueryToJSON, listQueryToJSON, OrderBy
from .serialization import deserializeTags
import json

Expand Down Expand Up @@ -92,8 +92,8 @@ def countQuery(
) \
select count, total from count_validated_features, count_features".format(
table=params.table.value,
dateFrom=" AND created_at >= '{dateFrom}'".format(dateFrom=params.dateFrom) if (params.dateFrom) else "",
dateTo=" AND created_at <= '{dateTo}'".format(dateTo=params.dateTo) if (params.dateTo) else "",
dateFrom=" AND closed_at >= '{dateFrom}'".format(dateFrom=params.dateFrom) if (params.dateFrom) else "",
dateTo=" AND closed_at <= '{dateTo}'".format(dateTo=params.dateTo) if (params.dateTo) else "",
area=" AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({area})))', 4326) )".format(area=params.area) if params.area else "",
tags=" AND (" + tagsQueryFilter(params.tags, params.table.value) + ") \n" if params.tags else "",
hashtag=" AND " + hashtagQueryFilter(params.hashtag, params.table.value) if params.hashtag else "",
Expand All @@ -115,7 +115,7 @@ def geoFeaturesQuery(params: RawValidationFeaturesParamsDTO, asJson: bool = Fals
status, \n \
hashtags, \n \
editor, \n \
created_at \n \
closed_at \n \
FROM {table} \n \
LEFT JOIN changesets c ON c.id = {table}.changeset \n \
LEFT JOIN validation ON validation.osm_id = {table}.osm_id \
Expand All @@ -127,7 +127,7 @@ def geoFeaturesQuery(params: RawValidationFeaturesParamsDTO, asJson: bool = Fals
.format(area=params.area) if params.area else "",
tags=" AND (" + tagsQueryFilter(params.tags, params.table.value) + ") \n" if params.tags else "",
hashtag=" AND " + hashtagQueryFilter(params.hashtag, params.table.value) if params.hashtag else "",
date=" AND created_at >= {dateFrom} AND created_at <= {dateTo}\n"
date=" AND closed_at >= {dateFrom} AND closed_at <= {dateTo}\n"
.format(dateFrom=params.dateFrom, dateTo=params.dateTo)
if params.dateFrom and params.dateTo else "\n",
status=" AND status = '{status}'".format(status=params.status.value) if (params.status) else "",
Expand All @@ -148,6 +148,7 @@ def listFeaturesQuery(
geoType:GeoType = GeoType[params.table]
osmType:OsmType = OsmType[params.table]
table:Table = Table[params.table]
orderBy:OrderBy = OrderBy[params.orderBy]

query = "( \
SELECT '{type}' as type, \n \
Expand All @@ -158,7 +159,7 @@ def listFeaturesQuery(
{table}.timestamp, \n \
tags, \n \
{table}.changeset, \n \
c.created_at, \n \
c.closed_at, \n \
status \n \
FROM {table} \n \
LEFT JOIN changesets c ON c.id = {table}.changeset \n \
Expand All @@ -169,8 +170,8 @@ def listFeaturesQuery(
type=osmType.value,
geotype=geoType.value,
table=table.value,
fromDate=" AND created_at >= '{dateFrom}'".format(dateFrom=params.dateFrom) if (params.dateFrom) else "",
toDate=" AND created_at <= '{dateTo}'".format(dateTo=params.dateTo) if (params.dateTo) else "",
fromDate=" AND closed_at >= '{dateFrom}'".format(dateFrom=params.dateFrom) if (params.dateFrom) else "",
toDate=" AND closed_at <= '{dateTo}'".format(dateTo=params.dateTo) if (params.dateTo) else "",
hashtag=" AND " + hashtagQueryFilter(params.hashtag, table.value) if params.hashtag else "",
area=" AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({area})))', 4326) )"
.format(
Expand All @@ -180,10 +181,10 @@ def listFeaturesQuery(
status=" AND status = '{status}'".format(status=params.status.value) if (params.status) else "",
order=" AND {order} IS NOT NULL ORDER BY {order} DESC LIMIT {limit} OFFSET {offset}"
.format(
order=params.orderBy.value,
order=orderBy.value,
limit=RESULTS_PER_PAGE_LIST,
offset=params.page * RESULTS_PER_PAGE_LIST
) if params.page else ""
) if params.page is not None else " LIMIT {limit} OFFSET {offset}"
).replace("WHERE AND", "WHERE")
if asJson:
return listQueryToJSON(query, params)
Expand Down
4 changes: 3 additions & 1 deletion python/dbapi/api/stats.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def featureCountQuery(params: StatsParamsDTO, asJson: bool = False):
.format(area=params.area) if params.area else "",
tags=" AND (" + tagsQueryFilter(params.tags, params.table.value) + ") \n" if params.tags else "",
hashtag=" AND " + hashtagQueryFilter(params.hashtag, params.table.value) if params.hashtag else "",
date=" AND created_at >= {dateFrom} AND created_at <= {dateTo}\n"
date=" AND closed_at >= {dateFrom} AND closed_at <= {dateTo}\n"
.format(dateFrom=params.dateFrom, dateTo=params.dateTo)
if params.dateFrom and params.dateTo else "\n"
).replace("WHERE AND", "WHERE")
Expand All @@ -65,6 +65,7 @@ async def getNodesCount(
result = await self.db.run(featureCountQuery(params), singleObject=True)
if asJson:
return json.dumps(dict(result))
return result

async def getLinesCount(
self,
Expand All @@ -75,6 +76,7 @@ async def getLinesCount(
result = await self.db.run(featureCountQuery(params), singleObject=True)
if asJson:
return json.dumps(dict(result))
return result

async def getPolygonsCount(
self,
Expand Down
4 changes: 2 additions & 2 deletions python/restapi/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,14 @@ class BaseRequest(BaseModel):
featureType: str = None

class BaseListRequest(BaseRequest):
orderBy: str = None
orderBy: str = "id"
page: int = None

class BaseRawValidationRequest(BaseRequest):
status: str = None

class RawValidationListRequest(BaseRawValidationRequest):
orderBy: str = None
orderBy: str = "id"
page: int = None

class RawRequest(BaseRequest):
Expand Down

0 comments on commit d02341c

Please sign in to comment.