diff --git a/.gitignore b/.gitignore index d96e5b2..efcede4 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,8 @@ yarn-error.log* coverage # Netlify build folders and files -.netlify/plugins +.netlify functions/*.zip + +.env diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bc917ca..a61bfea 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,16 +1,20 @@ We welcome contributions to the project! + # How to contribute to the project? + All contributions to this project are welcome. To propose changes, we encourage contributors to: 1. Fork this project on GitHub 2. Create a new branch -3. Propose changes by opening a new pull request. -If you're looking for somewhere to start, check out the issues labeled "Good first issue" or Community. +3. Propose changes by opening a new pull request. + If you're looking for somewhere to start, check out the issues labeled "Good first issue" or Community. # Issue and PR templates + We encourage contributors to format pull request titles following the [Conventional Commit Specification](https://www.conventionalcommits.org/en/v1.0.0/). # Folder organization + - gbfs-validator This is the heart of the validator. This folder contains a NodeJs package to validate GBFS Feeds. @@ -21,7 +25,7 @@ Contains JSON schemas - website -Contains the frontend, currently hosted by Netlify on https://gbfs-validator.netlify.app/ +Contains the frontend, currently hosted by Netlify on https://gbfs-validator.mobilitydata.org/ It’s a tiny Vue SPA. - functions @@ -35,10 +39,13 @@ The function is only compatible with Netlify Function (https://www.netlify.com/p Check-systems is a CLI tool to validate the whole “systems.csv” from https://github.com/NABSA/gbfs locally # Code convention + "Sticking to a single consistent and documented coding style for this project is important to ensure that code reviewers dedicate their attention to the functionality of the validation, as opposed to disagreements about the coding style (and avoid bike-shedding https://en.wikipedia.org/wiki/Law_of_triviality )." This project uses the Eslint + Prettier to ensure lint (See .eslintrc.js and .prettierrc) # Adding a new version + For adding a new version: + - Create a new folder under “gbfs-validator/schema” with the version as name (Eg: “vX.Y”). - Add an “index.js” file. This file will define the possible JSON-schema to call for validation and the mandatory ones. See [master/gbfs-validator/schema/v2.2/index.js](https://github.com/fluctuo/gbfs-validator/blob/master/gbfs-validator/schema/v2.2/index.js) for an exemple. - Fill the folder with all JSON-schemas for this version. diff --git a/README.md b/README.md index d99394d..e0e516d 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Questions? Please open an issue or reach out to MobilityData on the GBFS slack c The validator is developed to be used “online” (hosted with a lambda function). -1. Open gbfs-validator.netlify.com/ +1. Open gbfs-validator.mobilitydata.org/ 2. Enter the feed’s auto-discovery URL 3. If needed, select the version. If not specified, the validator will pick the version mentioned in the `gbfs.json` file 4. Select file requirement options (free-floating or docked) @@ -54,13 +54,18 @@ git clone https://github.com/fluctuo/gbfs-validator.git cd gbfs-validator ``` +### Set Environment variables + +Copy `./website/.env.exemple` to `./website/.env` +And set values + ### Run dev environment With Node.js ```shell yarn -yarn start +yarn run dev ``` Open `localhost:8080` on your browser @@ -83,6 +88,7 @@ Open `localhost:8080` on your browser This project follows the [all-contributors](https://allcontributors.org/docs/en/overview) specification, find the [emoji key here](https://allcontributors.org/docs/en/emoji-key). Contributions of any kind welcome! Please check out our [Contribution guidelines](/CONTRIBUTING.md) for details. :warning: for contributions on schemas, please see [Versions README](gbfs-validator/versions/README.md) + ## Acknowledgements This project was originally created by Pierrick Paul at [fluctuo](https://fluctuo.com/) - MobilityData started maintaining the project in September 2021. diff --git a/functions/doc.js b/functions/doc.js new file mode 100644 index 0000000..4bfecbf --- /dev/null +++ b/functions/doc.js @@ -0,0 +1,8 @@ +const openapiSchema = require('./openapi') + +exports.handler = function (event, context, callback) { + callback(null, { + statusCode: 200, + body: JSON.stringify(openapiSchema) + }) +} diff --git a/functions/feed.js b/functions/feed.js new file mode 100644 index 0000000..db3dd99 --- /dev/null +++ b/functions/feed.js @@ -0,0 +1,31 @@ +const GBFS = require('gbfs-validator') + +exports.handler = function (event, context, callback) { + let body + + try { + body = JSON.parse(event.body) + } catch (err) { + callback(err, { + statusCode: 500, + body: JSON.stringify(err) + }) + } + + const gbfs = new GBFS(body.url) + + gbfs + .getFiles() + .then((result) => { + callback(null, { + statusCode: 200, + body: JSON.stringify(result) + }) + }) + .catch((err) => { + callback(null, { + statusCode: 500, + body: JSON.stringify(err.message) + }) + }) +} diff --git a/functions/openapi.js b/functions/openapi.js new file mode 100644 index 0000000..66b194d --- /dev/null +++ b/functions/openapi.js @@ -0,0 +1,269 @@ +const validatorVersion = process.env.COMMIT_REF + ? process.env.COMMIT_REF.substring(0, 7) + : require('./package.json').version + +module.exports = { + openapi: '3.0.3', + info: { + title: 'GBFS validator', + version: `${validatorVersion}` + }, + servers: [ + { + url: 'https://gbfs-validator.netlify.app/.netlify/functions' + } + ], + paths: { + '/validator': { + post: { + summary: 'Validate a GBFS feed', + description: 'Validate the GBFS feed according to passed options', + requestBody: { + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/ValidatorRequest' + } + } + }, + required: true + }, + responses: { + 200: { + description: 'Validation result', + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/Validator' + } + } + } + }, + 500: { + description: 'Error', + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/Error' + } + } + } + } + } + } + }, + '/feed': { + post: { + summary: 'Get feed content', + description: + "Get content of all GBFS's files. Usefull to avoid CORS errors.", + requestBody: { + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/FeedRequest' + } + } + }, + required: true + }, + responses: { + 200: { + description: 'lorem', + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/Feed' + } + } + } + }, + 500: { + description: 'Error', + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/Error' + } + } + } + } + } + } + } + }, + components: { + schemas: { + Error: { + type: 'object' + }, + Validator: { + required: ['summary', 'files'], + type: 'object', + properties: { + summary: { + type: 'object' + }, + files: { + type: 'array', + items: { + $ref: '#/components/schemas/ValidatedFile' + } + } + } + }, + ValidatedFile: { + type: 'object', + properties: { + schema: { + type: 'object' + }, + errors: { + type: 'array', + items: { + $ref: '#/components/schemas/JSONError' + } + }, + url: { + type: 'string' + }, + version: { + type: 'string' + }, + recommanded: { + type: 'boolean' + }, + required: { + type: 'boolean' + }, + exists: { + type: 'boolean' + }, + file: { + type: 'string' + }, + hasErrors: { + type: 'boolean' + }, + errorsCount: { + type: 'number' + } + }, + required: ['required', 'exists', 'file', 'hasErrors', 'errorsCount'] + }, + JSONError: { + type: 'object', + properties: { + instancePath: { + type: 'string' + }, + schemaPath: { + type: 'string' + }, + keyword: { + type: 'string' + }, + params: { + type: 'object' + }, + message: { + type: 'string' + } + } + }, + ValidatorRequest: { + required: ['url'], + type: 'object', + properties: { + url: { + type: 'string' + }, + options: { + type: 'object', + properties: { + freefloating: { + type: 'boolean' + }, + docked: { + type: 'boolean' + }, + version: { + type: 'string' + }, + auth: { + type: 'object', + properties: { + type: { + type: 'string' + }, + basicAuth: { + type: 'object', + properties: { + user: { + type: 'string' + }, + password: { + type: 'string' + } + } + }, + bearerToken: { + type: 'object', + properties: { + token: { + type: 'string' + } + } + }, + oauthClientCredentialsGrant: { + type: 'object', + properties: { + user: { + type: 'string' + }, + password: { + type: 'string' + }, + tokenUrl: { + type: 'string' + } + } + } + } + } + } + } + } + }, + FeedRequest: { + required: ['url'], + type: 'object', + properties: { + url: { + type: 'string' + } + } + }, + Feed: { + type: 'object', + properties: { + summary: { + type: 'object' + }, + gbfsResult: { + type: 'object' + }, + gbfsVersion: { + type: 'string' + }, + files: { + type: 'array', + items: { + type: 'object' + } + } + } + } + } + } +} diff --git a/functions/validator.js b/functions/validator.js index 44a1daf..3fcf726 100644 --- a/functions/validator.js +++ b/functions/validator.js @@ -1,6 +1,6 @@ const GBFS = require('gbfs-validator') -exports.handler = function(event, context, callback) { +exports.handler = function (event, context, callback) { let body try { @@ -16,13 +16,13 @@ exports.handler = function(event, context, callback) { gbfs .validation() - .then(result => { + .then((result) => { callback(null, { statusCode: 200, body: JSON.stringify(result) }) }) - .catch(err => { + .catch((err) => { callback(null, { statusCode: 500, body: JSON.stringify(err.message) diff --git a/gbfs-validator/gbfs.js b/gbfs-validator/gbfs.js index 6542081..29ed6b5 100644 --- a/gbfs-validator/gbfs.js +++ b/gbfs-validator/gbfs.js @@ -7,7 +7,7 @@ const validatorVersion = process.env.COMMIT_REF function hasErrors(data, required) { let hasError = false - data.forEach(el => { + data.forEach((el) => { if (Array.isArray(el)) { if (hasErrors(el, required)) { hasError = true @@ -30,7 +30,7 @@ function countErrors(file) { count = file.errors.length } else if (file.languages) { if (file.required) { - count += file.languages.filter(l => !l.exists).length + count += file.languages.filter((l) => !l.exists).length } count += file.languages.reduce((acc, l) => { @@ -63,8 +63,8 @@ function getPartialSchema(version, partial, data = {}) { function getVehicleTypes({ body }) { if (Array.isArray(body)) { return body.reduce((acc, lang) => { - lang.body?.data?.vehicle_types.map(vt => { - if (!acc.find(f => f.vehicle_type_id === vt.vehicle_type_id)) { + lang.body?.data?.vehicle_types.map((vt) => { + if (!acc.find((f) => f.vehicle_type_id === vt.vehicle_type_id)) { acc.push({ vehicle_type_id: vt.vehicle_type_id, form_factor: vt.form_factor, @@ -76,7 +76,7 @@ function getVehicleTypes({ body }) { return acc }, []) } else { - return body?.data?.vehicle_types.map(vt => ({ + return body?.data?.vehicle_types.map((vt) => ({ vehicle_type_id: vt.vehicle_type_id, form_factor: vt.form_factor, propulsion_type: vt.propulsion_type @@ -87,8 +87,8 @@ function getVehicleTypes({ body }) { function getPricingPlans({ body }) { if (Array.isArray(body)) { return body.reduce((acc, lang) => { - lang.body?.data?.plans.map(pp => { - if (!acc.find(f => f.plan_id === pp.plan_id)) { + lang.body?.data?.plans.map((pp) => { + if (!acc.find((f) => f.plan_id === pp.plan_id)) { acc.push(pp) } }) @@ -102,27 +102,31 @@ function getPricingPlans({ body }) { function hadVehicleTypeId({ body }) { if (Array.isArray(body)) { - return body.some(lang => lang.body.data.bikes.find(b => b.vehicle_type_id)) + return body.some((lang) => + lang.body.data.bikes.find((b) => b.vehicle_type_id) + ) } else { - return body.data.bikes.some(b => b.vehicle_type_id) + return body.data.bikes.some((b) => b.vehicle_type_id) } } function hasPricingPlanId({ body }) { if (Array.isArray(body)) { - return body.some(lang => lang.body.data.bikes.find(b => b.pricing_plan_id)) + return body.some((lang) => + lang.body.data.bikes.find((b) => b.pricing_plan_id) + ) } else { - return body.data.bikes.some(b => b.pricing_plan_id) + return body.data.bikes.some((b) => b.pricing_plan_id) } } function hasRentalUris({ body }, key, store) { if (Array.isArray(body)) { - return body.some(lang => - lang.body.data[key].find(b => b.rental_uris?.[store]) + return body.some((lang) => + lang.body.data[key].find((b) => b.rental_uris?.[store]) ) } else { - return body.data[key].some(b => b.rental_uris?.[store]) + return body.data[key].some((b) => b.rental_uris?.[store]) } } @@ -134,7 +138,7 @@ function fileExist(file) { if (file.exists) { return true } else if (Array.isArray(file.body)) { - return file.body.some(lang => lang.exists) + return file.body.some((lang) => lang.exists) } return false @@ -189,7 +193,7 @@ class GBFS { return got .get(url, this.gotOptions) .json() - .then(body => { + .then((body) => { if (typeof body !== 'object') { return { recommanded: true, @@ -240,7 +244,7 @@ class GBFS { return got .get(this.url, this.gotOptions) .json() - .then(body => { + .then((body) => { if (typeof body !== 'object') { return this.alternativeAutoDiscovery( new URL('gbfs.json', this.url).toString() @@ -269,7 +273,7 @@ class GBFS { hasErrors: !!errors } }) - .catch(e => { + .catch((e) => { if (!this.url.match(/gbfs.json$/)) { return this.alternativeAutoDiscovery( new URL('gbfs.json', this.url).toString() @@ -303,43 +307,43 @@ class GBFS { getFile(type, required) { if (this.autoDiscovery) { - const urls = Object.entries(this.autoDiscovery.data).map(key => { + const urls = Object.entries(this.autoDiscovery.data).map((key) => { return Object.assign( { lang: key[0] }, - this.autoDiscovery.data[key[0]].feeds.find(f => f.name === type) + this.autoDiscovery.data[key[0]].feeds.find((f) => f.name === type) ) }) return Promise.all( - urls.map( - lang => - lang && lang.url - ? got - .get(lang.url, this.gotOptions) - .json() - .then(body => { - return { - body, - exists: true, - lang: lang.lang, - url: lang.url - } - }) - .catch(() => ({ - body: null, - exists: false, + urls.map((lang) => + lang && lang.url + ? got + .get(lang.url, this.gotOptions) + .json() + .then((body) => { + return { + body, + exists: true, lang: lang.lang, url: lang.url - })) - : { + } + }) + .catch(() => ({ body: null, exists: false, lang: lang.lang, - url: null - } + url: lang.url + })) + : { + body: null, + exists: false, + lang: lang.lang, + url: null + } ) - ).then(bodies => { + ).then((bodies) => { return { + file: `${type}.json`, body: bodies, required, type @@ -349,13 +353,15 @@ class GBFS { return got .get(`${this.url}/${type}.json`, this.gotOptions) .json() - .then(body => ({ + .then((body) => ({ + file: `${type}.json`, body, required, exists: true, type })) - .catch(err => ({ + .catch((err) => ({ + file: `${type}.json`, body: null, required, errors: required ? err : null, @@ -367,10 +373,12 @@ class GBFS { validationFile(body, version, type, required, options) { if (Array.isArray(body)) { - body = body.filter(b => b.exists || b.required).map(b => ({ - ...b, - ...this.validateFile(version, type, b.body, options) - })) + body = body + .filter((b) => b.exists || b.required) + .map((b) => ({ + ...b, + ...this.validateFile(version, type, b.body, options) + })) return { languages: body, @@ -385,7 +393,6 @@ class GBFS { return { required, ...this.validateFile(version, type, body, options), - exists: !!body, file: `${type}.json`, url: `${this.url}/${type}.json` @@ -402,14 +409,14 @@ class GBFS { body: 'grant_type=client_credentials' }) .json() - .then(auth => { + .then((auth) => { this.gotOptions.headers = { Authorization: `Bearer ${auth.access_token}` } }) } - async validation() { + async getFiles() { if (this.auth && this.auth.type === 'oauth_client_credentials_grant') { await this.getToken() } @@ -419,6 +426,8 @@ class GBFS { if (!gbfsResult.version) { return { summary: { + gbfsResult, + gbfsVersion: gbfsResult.version, validatorVersion, versionUnimplemented: true } @@ -427,16 +436,35 @@ class GBFS { const gbfsVersion = this.options.version || gbfsResult.version - let files = require(`./versions/v${gbfsVersion}.js`).files(this.options) - - const t = await Promise.all( - files.map(f => this.getFile(f.file, f.required)) + let filesRequired = require(`./versions/v${gbfsVersion}.js`).files( + this.options ) - const vehicleTypesFile = t.find(a => a.type === 'vehicle_types') - const freeBikeStatusFile = t.find(a => a.type === 'free_bike_status') - const stationInformationFile = t.find(a => a.type === 'station_information') - const stationPricingPlans = t.find(a => a.type === 'system_pricing_plans') + return { + summary: {}, + gbfsResult, + gbfsVersion, + files: await Promise.all( + filesRequired.map((f) => this.getFile(f.file, f.required)) + ) + } + } + + async validation() { + const { gbfsResult, gbfsVersion, files, summary } = await this.getFiles() + + if (summary?.versionUnimplemented) { + return { summary } + } + + const vehicleTypesFile = files.find((a) => a.type === 'vehicle_types') + const freeBikeStatusFile = files.find((a) => a.type === 'free_bike_status') + const stationInformationFile = files.find( + (a) => a.type === 'station_information' + ) + const stationPricingPlans = files.find( + (a) => a.type === 'system_pricing_plans' + ) let vehicleTypes, pricingPlans, @@ -475,7 +503,7 @@ class GBFS { pricingPlans = getPricingPlans(stationPricingPlans) } - t.forEach(f => { + files.forEach((f) => { const addSchema = [] let required = f.required @@ -542,6 +570,7 @@ class GBFS { addSchema.push(partial) } } + break default: break } @@ -553,7 +582,7 @@ class GBFS { ) }) - const filesResult = result.map(file => ({ + const filesResult = result.map((file) => ({ ...file, errorsCount: countErrors(file) })) diff --git a/netlify.toml b/netlify.toml index 9a2a9a0..afbd56b 100644 --- a/netlify.toml +++ b/netlify.toml @@ -6,4 +6,6 @@ directory = "functions" [dev] + command = "yarn run dev:website" + autoLaunch = false functions = "functions" diff --git a/package.json b/package.json index e23bba8..da93b7d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "scripts": { - "dev:website": "cd website && yarn run serve", + "dev:website": "cd website && yarn run dev", "dev": "netlify dev", "start": "yarn run dev", "lint": "eslint --ext .js,.vue website/src", diff --git a/website/.env.exemple b/website/.env.exemple new file mode 100644 index 0000000..c2fc9df --- /dev/null +++ b/website/.env.exemple @@ -0,0 +1,5 @@ +# Mapbox token (Required for visualization / public scopes) +VITE_MAPBOX_API_KEY= + +# Google Analytics ID (Optionnal) +VITE_GOOGLE_ANALYTICS_ID= diff --git a/website/index.html b/website/index.html index 439d72a..a1b2fda 100644 --- a/website/index.html +++ b/website/index.html @@ -3,14 +3,17 @@ - - - - - - - - + + + + + + + + GBFS-Validator - A GBFS feed validator diff --git a/website/package.json b/website/package.json index 038bbe6..315f94c 100644 --- a/website/package.json +++ b/website/package.json @@ -13,13 +13,22 @@ "test": "echo \"Error: no test specified\" && exit 0" }, "dependencies": { + "@deck.gl/core": "^8.9.18", + "@deck.gl/layers": "^8.9.18", + "@deck.gl/mapbox": "^8.9.18", + "@turf/bbox": "^6.5.0", + "@vue/compat": "^3.3.2", + "@vueuse/core": "^10.1.2", "bootstrap": "^5.0.1", "bootstrap-vue": "^2.23.1", + "maplibre-gl": "^3.0.1", + "maplibregl-mapbox-request-transformer": "^0.0.2", + "mitt": "^3.0.0", "sass": "^1.34.1", "sass-loader": "^12.0.0", "vue": "^3.3.2", - "@vue/compat": "^3.3.2", - "vue-gtag": "1.16.1" + "vue-gtag": "1.16.1", + "vue-router": "^4.2.2" }, "devDependencies": { "@rushstack/eslint-patch": "^1.2.0", diff --git a/website/src/App.vue b/website/src/App.vue index a80fbec..9b14342 100644 --- a/website/src/App.vue +++ b/website/src/App.vue @@ -1,28 +1,39 @@