ADD: SQLite Export With Python

README.md

@@ -1,17 +1,38 @@
 # Yangon Bus Service – YBS Data
+
 YBS Data is improved update-to-date version of officially released version.
 
 Data are gathered around Facebook/Groups/Pages that are related to YBS. Data validity is by best effort.
 
 ### ./data
+
 - **./routes/** are json files that contain bus line information by ID, name, bus route as line shape and bus stop IDs in sequence.
-	-  Bus stop IDs are equals to ID in `stops.tsv`
-	- `shape.geometry` is in geojson LineString format.
-	- separated into respective json file for easier git versioning
+  - Bus stop IDs are equals to ID in `stops.tsv`
+  - `shape.geometry` is in geojson LineString format.
+  - separated into respective json file for easier git versioning
 - **stops.tsv** is in tsv format contains stop ID along with name, road, township.
 
 ### ./tools
+
 - python scripts for importing/exporting to/from mongo
 
+#### Export to SQLite DB
+
+```bash
+$ cd tools
+$ python import_stops.py
+$ python import_routes.py
+
+# or
+
+$ cd tools
+$ ./export_db.sh
+```
+
+_DB Output - data/ybs.db_
+
+---
+
 ## Thanks
+
 - Soe Moe - for helping with data updates

data/.gitignore

@@ -0,0 +1 @@
+*.db

tools/export_db.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+python import_stops.py
+python import_routes.py

tools/import_routes.py

@@ -0,0 +1,32 @@
+import json
+import sqlite3
+import os
+
+db_path = './../data/ybs.db'
+con = sqlite3.connect(db_path)
+cur = con.cursor()
+cur.execute("DROP TABLE IF EXISTS routes;")
+cur.execute("DROP TABLE IF EXISTS coordinates;")
+cur.execute("DROP TABLE IF EXISTS route_stops;")
+cur.execute(
+    "CREATE TABLE routes (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, route_id_name TEXT NOT NULL, color TEXT(6) NOT NULL, name TEXT NOT NULL);")
+cur.execute(
+    "CREATE TABLE coordinates (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,route_id BIGINT UNSIGNED NOT NULL, lat Decimal(8,6) NOT NULL, lng Decimal(9,6) NOT NULL, FOREIGN KEY (route_id) REFERENCES routes(id));")
+cur.execute(
+    "CREATE TABLE route_stops (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,route_id BIGINT UNSIGNED NOT NULL, stop_id BIGINT UNSIGNED NOT NULL, FOREIGN KEY (route_id) REFERENCES routes(id), FOREIGN KEY (stop_id) REFERENCES stops(id));")
+
+route_directory = './../data/routes/'
+for filename in os.listdir(route_directory):
+    if filename.endswith(".json"):
+        with open(route_directory + filename) as data_file:
+            data = json.load(data_file)
+            cur.execute(
+                "INSERT INTO routes (route_id_name, color, name) VALUES (?, ?, ?);", (data['route_id'], data['color'], data['name']))
+            route_id = cur.lastrowid
+            cur.executemany(
+                "INSERT INTO coordinates (route_id, lat, lng) VALUES (?, ?, ?);",
+                [(route_id, coord[0], coord[1]) for coord in data['shape']['geometry']['coordinates']])
+            cur.executemany(
+                "INSERT INTO route_stops (route_id, stop_id) VALUES (?, ?);", [(route_id, shop_id) for shop_id in data['stops']])
+con.commit()
+con.close()

tools/import_stops.py

@@ -0,0 +1,19 @@
+import csv
+import sqlite3
+
+db_path = './../data/ybs.db'
+con = sqlite3.connect(db_path)
+cur = con.cursor()
+cur.execute("DROP TABLE IF EXISTS stops;")
+cur.execute("CREATE TABLE stops (id INTEGER NOT NULL PRIMARY KEY, lat Decimal(8,6) NOT NULL, lng Decimal(9,6) NOT NULL, name_en TEXT NOT NULL, name_mm TEXT NOT NULL, road_en TEXT NOT NULL, road_mm TEXT NOT NULL, township_en TEXT NOT NULL, township_mm TEXT NOT NULL);")
+stops_tsv_path = './../data/stops.tsv'
+
+with open(stops_tsv_path, 'r') as stops_file:
+    reader = csv.reader(stops_file, delimiter="\t")
+    next(reader)
+    cur.executemany(
+        "INSERT INTO stops (id, lat, lng, name_en, name_mm, road_en, road_mm, township_en, township_mm) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);",
+        reader
+    )
+con.commit()
+con.close()