From 2623ed165de53a9a2523e3b239d0e28bc32a0a9f Mon Sep 17 00:00:00 2001 From: jojocys <yuyilobay@gmail.com> Date: Wed, 14 Sep 2022 10:55:25 +0800 Subject: [PATCH 1/2] refactor: delete useless sub repo --- apps/rath-py-connector/__init__.py | 0 apps/rath-py-connector/database.py | 16 --- apps/rath-py-connector/main.py | 122 -------------------- apps/rath-py-connector/models/__init__.py | 0 apps/rath-py-connector/models/connection.py | 17 --- apps/testServer/index.js | 91 --------------- apps/testServer/package.json | 11 -- 7 files changed, 257 deletions(-) delete mode 100644 apps/rath-py-connector/__init__.py delete mode 100644 apps/rath-py-connector/database.py delete mode 100644 apps/rath-py-connector/main.py delete mode 100644 apps/rath-py-connector/models/__init__.py delete mode 100644 apps/rath-py-connector/models/connection.py delete mode 100644 apps/testServer/index.js delete mode 100644 apps/testServer/package.json diff --git a/apps/rath-py-connector/__init__.py b/apps/rath-py-connector/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/apps/rath-py-connector/database.py b/apps/rath-py-connector/database.py deleted file mode 100644 index b80c950b..00000000 --- a/apps/rath-py-connector/database.py +++ /dev/null @@ -1,16 +0,0 @@ -from sqlalchemy import create_engine -from sqlalchemy.orm import scoped_session, sessionmaker -from sqlalchemy.ext.declarative import declarative_base - -engine = create_engine('sqlite:///./test.db') -db_session = scoped_session(sessionmaker(autocommit=False, - autoflush=False, - bind=engine)) -Base = declarative_base() -Base.query = db_session.query_property() - -def init_db(): - # 在这里导入定义模型所需要的所有模块,这样它们就会正确的注册在 - # 元数据上。否则你就必须在调用 init_db() 之前导入它们。 - import models.connection - Base.metadata.create_all(bind=engine) \ No newline at end of file diff --git a/apps/rath-py-connector/main.py b/apps/rath-py-connector/main.py deleted file mode 100644 index 6f188c5c..00000000 --- a/apps/rath-py-connector/main.py +++ /dev/null @@ -1,122 +0,0 @@ -from crypt import methods -import json -from database import init_db, db_session -from flask import Flask, request -from flask_cors import CORS -from sqlalchemy import create_engine, MetaData, inspect -from sqlalchemy import types -from models.connection import Connection - - -app = Flask(__name__) -CORS(app, supports_credentials=True) - -@app.route("/api/schema_list", methods=['GET']) -def schema_list(): - engine = create_engine('clickhouse+native://localhost/default') - inspector = inspect(engine) - schemas = inspector.get_schema_names() - return { - "success": True, - "data": schemas - } - -@app.route("/api/table_list", methods=['GET']) -def table_list(): - props = json.loads(request.data) - print(props) - engine = create_engine('clickhouse+native://localhost/default') - inspector = inspect(engine) - # schemas = inspector.get_schema_names() - tables = inspector.get_table_names(schema=props["schema"]) - return { - "success": True, - "data": tables - } - -@app.route("/api/column_list", methods=['POST']) -def column_list(): - props = json.loads(request.data) - print(props) - engine = create_engine('clickhouse+native://localhost/default') - inspector = inspect(engine) - columns = inspector.get_columns(props["table"], schema=props["schema"]) - # print(columns, type(columns[0]['type'])) - # print(vars(columns[0]['type'])) - # print(columns[0]['type'].Unicode()) - # tt = columns[0]['type'].copy() - print('!!', str(columns[0]['type']), type(columns[0]['type'].copy())) - ser_columns = [] - for col in columns: - ser_col = col.copy() - ser_col['type'] = str(ser_col['type']) - ser_columns.append(ser_col) - - return { - "success": True, - "data": ser_columns - } - -@app.route("/api/preview", methods=['POST']) -def preview_data(): - props = json.loads(request.data) - print(props) - engine = create_engine('clickhouse+native://localhost/default') - with engine.connect() as con: - res = con.execute('SELECT * from ' + props['schema'] + '.' + props['table']) - ans = [] - for row in res: - ans.append(row.values()) - return { - "success": True, - "data": ans - } - -@app.route("/api/connection_list", methods=['GET']) -def get_connection_list(): - connection_list = Connection.query.all() - # print(help(connection_list)) - print(dict(connection_list[0])) - return "" - -@app.route("/api/connection/create", methods=['POST']) -def create_connection(): - props = json.loads(request.data) - print(props) - newCon = Connection(props["name"], props["uri"], props["source_type"]) - db_session.add(newCon) - db_session.commit() - connection_list = Connection.query.all() - print(help(connection_list)) - print(list(map(lambda x: dict(x), connection_list))) - res = list(map(lambda x: dict(x), connection_list)) - return { - "success": True, - "data": res - } - -@app.route("/") -def hello_world(): - return "<p>Hello, World!</p>" - -@app.route("/ping") -def ping(): - return { - "success": True - } - -@app.teardown_appcontext -def shutdown_session(exception=None): - db_session.remove() - -if __name__ == '__main__': - # init_db() - init_db() - # connection_list = Connection.query.all() - # print(connection_list) - # from database import db_session - # from models.connection import Connection - # u = Connection('admin', '') - # db_session.add(u) - # db_session.commit() - app.run(host='127.0.0.1',port=8000) \ No newline at end of file diff --git a/apps/rath-py-connector/models/__init__.py b/apps/rath-py-connector/models/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/apps/rath-py-connector/models/connection.py b/apps/rath-py-connector/models/connection.py deleted file mode 100644 index a2267543..00000000 --- a/apps/rath-py-connector/models/connection.py +++ /dev/null @@ -1,17 +0,0 @@ -from sqlalchemy import Column, Integer, String -from database import Base - -class Connection(Base): - __tablename__ = 'connection' - id = Column(Integer, primary_key=True) - name = Column(String(50), unique=False) - uri = Column(String(255), unique=False) - source_type = Column(String(50), unique=False) - - def __init__(self, name=None, uri=None, source_type=None): - self.name = name - self.uri = uri - self.source_type = source_type - - def __repr__(self): - return f'<Connection {self.name!r}>' \ No newline at end of file diff --git a/apps/testServer/index.js b/apps/testServer/index.js deleted file mode 100644 index 49ffc865..00000000 --- a/apps/testServer/index.js +++ /dev/null @@ -1,91 +0,0 @@ -var express = require("express"); -var fs = require("fs"); -var app = express(); -var bodyParser = require("body-parser"); -var cookieParser = require("cookie-parser"); -app.use(express.json({ limit: Infinity })); - -app.use(bodyParser.json()); -app.use(cookieParser()); -app.use(bodyParser.urlencoded({ extended: false })); - -app.all("*", function (req, res, next) { - // res.header("Access-Control-Allow-Origin", ["http://kanaries-app.s3.ap-northeast-1.amazonaws.com", "http://localhost:8000"]); - res.header("Access-Control-Allow-Origin", '*'); - // res.header("Access-Control-Allow-Credentials", true); - res.header( - "Access-Control-Allow-Headers", - "Content-Type, Content-Length, Authorization, Accept, X-Requested-With , yourHeaderFeild" - ); - res.header("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS"); - - if (req.method == "OPTIONS") { - res.send(200); - /让options请求快速返回/; - } else { - next(); - } -}); -app.post("/associate", function (req, res) { - // console.log("[/assocaite]", req); - const fields = req.body.fields; - const returns = JSON.parse(fs.readFileSync('/Users/chenhao/Downloads/result(3).json').toString()) - const result1 = returns.data.t1; - for (let i = 0; i < result1.length; i++) { - result1[i].dimensions = result1[i].dimensions.map(d => { - const target = fields.find(f => f.fid.split('_')[1] === d.split('_')[1]); - return target.fid; - }).filter(f => Boolean(f)); - result1[i].measures = result1[i].measures.map(d => { - const target = fields.find(f => f.fid.split('_')[1] === d.split('_')[1]); - return target.fid; - }).filter(f => Boolean(f)); - } - const result2 = returns.data.t2; - for (let i = 0; i < result2.length; i++) { - result2[i].dimensions = result2[i].dimensions.map(d => { - const target = fields.find(f => f.fid.split('_')[1] === d.split('_')[1]); - return target.fid; - }).filter(f => Boolean(f)); - result2[i].measures = result2[i].measures.map(d => { - const target = fields.find(f => f.fid.split('_')[1] === d.split('_')[1]); - return target.fid; - }).filter(f => Boolean(f)); - } - res.json(returns); -}); - -app.post("/start", function (req, res) { - console.log("[/start]", req); - const fields = req.body.fields; - res.json({ - success: true, - data: { - fields: req.body.fields, - dataSource: req.body.dataSource, - insightSpaces: [ - { - dimensions: fields.filter((f) => f.analyticType === "dimension").slice(0, 2).map(f => f.fid), - measures: fields.filter((f) => f.analyticType === "measure").slice(0, 2).map(f => f.fid), - // significance: 1, - score: 1, - // impurity: 1 - }, - { - dimensions: fields.filter((f) => f.analyticType === "dimension").slice(1, 3).map(f => f.fid), - measures: fields.filter((f) => f.analyticType === "measure").slice(1, 2).map(f => f.fid), - // significance: 1, - score: 1, - // impurity: 1 - } - ] - }, - }); -}); - -var server = app.listen(8000, function () { - var host = server.address().address; - var port = server.address().port; - - console.log("应用实例,访问地址为 http://%s:%s", host, port); -}); diff --git a/apps/testServer/package.json b/apps/testServer/package.json deleted file mode 100644 index 5a839e43..00000000 --- a/apps/testServer/package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "dependencies": { - "body-parser": "^1.19.2", - "cookie-parser": "^1.4.6", - "express": "^4.17.3" - }, - "prettier": { - "tabWidth": 4, - "printWidth": 120 - } -} From 0b869444e6c47172a495c6bf497d34e7ac94545b Mon Sep 17 00:00:00 2001 From: jojocys <yuyilobay@gmail.com> Date: Wed, 14 Sep 2022 10:55:36 +0800 Subject: [PATCH 2/2] doc: update connector info --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 17fb7b5c..ed23f6e1 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,9 @@ RATH is an OpenSource automated data exploration tool that can help you automate It is not only an open source replacement of tableau but a version with more automation power, including auto insights discovery, predictive interaction, and visual recommendation. -Try RATH [here](https://kanaries.net/) +Try RATH [here](https://rath.kanaries.net) + +or visit our official website[kanaries website](https://kanaries.net) ## Introduction @@ -103,6 +105,9 @@ Details of the test result can be accessed [here](https://www.yuque.com/chenhao- Rath now runs all the computation tasks on webworker. For some large datasets, Rath will use indexedDB to avoid too much memory cost of browsers. For larger datasets (>100MB), Rath can put some of its computation to computation engine support SQL query(such as clickhouse). If you are interested in a server version, check the older version or contact us. +#### deploy connector service +RATH requires a database connector to connect common databases. You need to deploy `apps/connector` and then you can connect a number of common databases as datasource. + ## Documentation + [Tutorial: Using Rath to find deep insight in your data](https://www.yuque.com/docs/share/3f32e044-3530-4ebe-9b01-287bfbdb7ce0?#)