From 833ad0a53d399e90f0f1ad5b0a7935b0784eb884 Mon Sep 17 00:00:00 2001
From: Chit Ye Aung <chitye.aung@wavemoney.com.mm>
Date: Sun, 19 Jun 2022 13:46:07 +0630
Subject: [PATCH] ADD: SQLite Export With Python

---
 README.md              | 27 ++++++++++++++++++++++++---
 data/.gitignore        |  1 +
 tools/export_db.sh     |  3 +++
 tools/import_routes.py | 32 ++++++++++++++++++++++++++++++++
 tools/import_stops.py  | 19 +++++++++++++++++++
 5 files changed, 79 insertions(+), 3 deletions(-)
 create mode 100644 data/.gitignore
 create mode 100755 tools/export_db.sh
 create mode 100644 tools/import_routes.py
 create mode 100644 tools/import_stops.py

diff --git a/README.md b/README.md
index 1e51519..e67d6b1 100644
--- a/README.md
+++ b/README.md
@@ -1,17 +1,38 @@
 # Yangon Bus Service – YBS Data
+
 YBS Data is improved update-to-date version of officially released version.
 
 Data are gathered around Facebook/Groups/Pages that are related to YBS. Data validity is by best effort.
 
 ### ./data
+
 - **./routes/** are json files that contain bus line information by ID, name, bus route as line shape and bus stop IDs in sequence.
-	-  Bus stop IDs are equals to ID in `stops.tsv`
-	- `shape.geometry` is in geojson LineString format.
-	- separated into respective json file for easier git versioning
+  - Bus stop IDs are equals to ID in `stops.tsv`
+  - `shape.geometry` is in geojson LineString format.
+  - separated into respective json file for easier git versioning
 - **stops.tsv** is in tsv format contains stop ID along with name, road, township.
 
 ### ./tools
+
 - python scripts for importing/exporting to/from mongo
 
+#### Export to SQLite DB
+
+```bash
+$ cd tools
+$ python import_stops.py
+$ python import_routes.py
+
+# or
+
+$ cd tools
+$ ./export_db.sh
+```
+
+_DB Output - data/ybs.db_
+
+---
+
 ## Thanks
+
 - Soe Moe - for helping with data updates
diff --git a/data/.gitignore b/data/.gitignore
new file mode 100644
index 0000000..3997bea
--- /dev/null
+++ b/data/.gitignore
@@ -0,0 +1 @@
+*.db
\ No newline at end of file
diff --git a/tools/export_db.sh b/tools/export_db.sh
new file mode 100755
index 0000000..b6d584b
--- /dev/null
+++ b/tools/export_db.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+python import_stops.py
+python import_routes.py
\ No newline at end of file
diff --git a/tools/import_routes.py b/tools/import_routes.py
new file mode 100644
index 0000000..2c0fb65
--- /dev/null
+++ b/tools/import_routes.py
@@ -0,0 +1,32 @@
+import json
+import sqlite3
+import os
+
+db_path = './../data/ybs.db'
+con = sqlite3.connect(db_path)
+cur = con.cursor()
+cur.execute("DROP TABLE IF EXISTS routes;")
+cur.execute("DROP TABLE IF EXISTS coordinates;")
+cur.execute("DROP TABLE IF EXISTS route_stops;")
+cur.execute(
+    "CREATE TABLE routes (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, route_id_name TEXT NOT NULL, color TEXT(6) NOT NULL, name TEXT NOT NULL);")
+cur.execute(
+    "CREATE TABLE coordinates (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,route_id BIGINT UNSIGNED NOT NULL, lat Decimal(8,6) NOT NULL, lng Decimal(9,6) NOT NULL, FOREIGN KEY (route_id) REFERENCES routes(id));")
+cur.execute(
+    "CREATE TABLE route_stops (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,route_id BIGINT UNSIGNED NOT NULL, stop_id BIGINT UNSIGNED NOT NULL, FOREIGN KEY (route_id) REFERENCES routes(id), FOREIGN KEY (stop_id) REFERENCES stops(id));")
+
+route_directory = './../data/routes/'
+for filename in os.listdir(route_directory):
+    if filename.endswith(".json"):
+        with open(route_directory + filename) as data_file:
+            data = json.load(data_file)
+            cur.execute(
+                "INSERT INTO routes (route_id_name, color, name) VALUES (?, ?, ?);", (data['route_id'], data['color'], data['name']))
+            route_id = cur.lastrowid
+            cur.executemany(
+                "INSERT INTO coordinates (route_id, lat, lng) VALUES (?, ?, ?);",
+                [(route_id, coord[0], coord[1]) for coord in data['shape']['geometry']['coordinates']])
+            cur.executemany(
+                "INSERT INTO route_stops (route_id, stop_id) VALUES (?, ?);", [(route_id, shop_id) for shop_id in data['stops']])
+con.commit()
+con.close()
diff --git a/tools/import_stops.py b/tools/import_stops.py
new file mode 100644
index 0000000..7967067
--- /dev/null
+++ b/tools/import_stops.py
@@ -0,0 +1,19 @@
+import csv
+import sqlite3
+
+db_path = './../data/ybs.db'
+con = sqlite3.connect(db_path)
+cur = con.cursor()
+cur.execute("DROP TABLE IF EXISTS stops;")
+cur.execute("CREATE TABLE stops (id INTEGER NOT NULL PRIMARY KEY, lat Decimal(8,6) NOT NULL, lng Decimal(9,6) NOT NULL, name_en TEXT NOT NULL, name_mm TEXT NOT NULL, road_en TEXT NOT NULL, road_mm TEXT NOT NULL, township_en TEXT NOT NULL, township_mm TEXT NOT NULL);")
+stops_tsv_path = './../data/stops.tsv'
+
+with open(stops_tsv_path, 'r') as stops_file:
+    reader = csv.reader(stops_file, delimiter="\t")
+    next(reader)
+    cur.executemany(
+        "INSERT INTO stops (id, lat, lng, name_en, name_mm, road_en, road_mm, township_en, township_mm) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);",
+        reader
+    )
+con.commit()
+con.close()