fix: ER_TRUNCATED_WRONG_VALUE: Truncated incorrect datetime value

Fixes #309

.circleci/config.yml

@@ -10,6 +10,9 @@ jobs:
           TEST_LOCAL: true
           TEST_DB_PASSWORD: Test1test
       - image: circleci/redis:5.0.5
+      - image: circleci/mysql:5.7
+        environment:
+          MYSQL_ROOT_PASSWORD: Test1test
       - image: circleci/postgres:9.6.8
         environment:
           POSTGRES_USER: root

packages/cubejs-schema-compiler/adapter/MysqlQuery.js

@@ -32,6 +32,10 @@ class MysqlQuery extends BaseQuery {
     return `TIMESTAMP(convert_tz(${value}, '+00:00', @@session.time_zone))`;
   }
 
+  inDbTimeZone(date) {
+    return this.inIntegrationTimeZone(date).clone().utc().format(moment.HTML5_FMT.DATETIME_LOCAL_MS);
+  }
+
   dateTimeCast(value) {
     return `TIMESTAMP(${value})`;
   }

packages/cubejs-schema-compiler/package.json

@@ -38,6 +38,7 @@
     "eslint-plugin-node": "^5.2.1",
     "mocha": "^3.4.2",
     "mssql": "^5.1.0",
+    "mysql": "^2.17.1",
     "pg-promise": "^7.3.2",
     "request": "^2.88.0",
     "request-promise": "^4.2.4",

packages/cubejs-schema-compiler/test/MySqlDbRunner.js

@@ -0,0 +1,89 @@
+const { GenericContainer } = require("testcontainers");
+const mysql = require('mysql');
+const { promisify } = require('util');
+
+const BaseDbRunner = require('./BaseDbRunner');
+
+class MSSqlDbRunner extends BaseDbRunner {
+  async connectionLazyInit(port) {
+    return {
+      testQueries: async (queries, fixture) => {
+        const conn = mysql.createConnection({
+          host: 'localhost',
+          port,
+          user: 'root',
+          database: 'mysql',
+          password: this.password()
+        });
+        const connect = promisify(conn.connect.bind(conn));
+
+        conn.execute = promisify(conn.query.bind(conn));
+
+        await connect();
+
+        try {
+          await this.prepareFixture(conn, fixture);
+          return await queries
+            .map(query => async () => JSON.parse(JSON.stringify(await conn.execute(query[0], query[1]))))
+            .reduce((a, b) => a.then(b), Promise.resolve());
+        } finally {
+          await promisify(conn.end.bind(conn))();
+        }
+      }
+    };
+  }
+
+  async prepareFixture(conn) {
+    const query = conn.execute;
+    await query('CREATE TEMPORARY TABLE visitors (id INT, amount INT, created_at datetime, updated_at datetime, status INT, source VARCHAR(255), latitude DECIMAL, longitude DECIMAL)');
+    await query('CREATE TEMPORARY TABLE visitor_checkins (id INT, visitor_id INT, created_at datetime, source VARCHAR(255))');
+    await query('CREATE TEMPORARY TABLE cards (id INT, visitor_id INT, visitor_checkin_id INT)');
+    await query(`
+    INSERT INTO
+    visitors
+    (id, amount, created_at, updated_at, status, source, latitude, longitude) VALUES
+    (1, 100, '2017-01-03', '2017-01-30', 1, 'some', 120.120, 40.60),
+    (2, 200, '2017-01-05', '2017-01-15', 1, 'some', 120.120, 58.60),
+    (3, 300, '2017-01-06', '2017-01-20', 2, 'google', 120.120, 70.60),
+    (4, 400, '2017-01-07', '2017-01-25', 2, NULL, 120.120, 10.60),
+    (5, 500, '2017-01-07', '2017-01-25', 2, NULL, 120.120, 58.10),
+    (6, 500, '2016-09-07', '2016-09-07', 2, NULL, 120.120, 58.10)
+    `);
+    await query(`
+    INSERT INTO
+    visitor_checkins
+    (id, visitor_id, created_at, source) VALUES
+    (1, 1, '2017-01-03', NULL),
+    (2, 1, '2017-01-04', NULL),
+    (3, 1, '2017-01-05', 'google'),
+    (4, 2, '2017-01-05', NULL),
+    (5, 2, '2017-01-05', NULL),
+    (6, 3, '2017-01-06', NULL)
+    `);
+    await query(`
+    INSERT INTO
+    cards
+    (id, visitor_id, visitor_checkin_id) VALUES
+    (1, 1, 1),
+    (2, 1, 2),
+    (3, 3, 6)
+    `);
+  }
+
+  password() {
+    return process.env.TEST_DB_PASSWORD || 'Test1test';
+  }
+
+  async containerLazyInit() {
+    return new GenericContainer("mysql", '5.7')
+      .withEnv("MYSQL_ROOT_PASSWORD", this.password())
+      .withExposedPorts(this.port())
+      .start();
+  }
+
+  port() {
+    return 3306;
+  }
+}
+
+module.exports = new MSSqlDbRunner();

packages/cubejs-schema-compiler/test/MySqlPreAggregationsTest.js

@@ -0,0 +1,161 @@
+/* globals describe,it,after */
+/* eslint-disable quote-props */
+const R = require('ramda');
+require('should');
+
+const MySqlQuery = require('../adapter/MysqlQuery');
+const { prepareCompiler } = require('./PrepareCompiler');
+
+const dbRunner = require('./MySqlDbRunner');
+
+describe('MySqlPreAggregations', function test() {
+  this.timeout(30000);
+
+  after(async () => {
+    await dbRunner.tearDown();
+  });
+
+  const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(`
+    cube(\`visitors\`, {
+      sql: \`
+      select * from visitors
+      \`,
+
+      measures: {
+        count: {
+          type: 'count'
+        },
+        
+        checkinsTotal: {
+          sql: \`\${checkinsCount}\`,
+          type: 'sum'
+        },
+        
+        uniqueSourceCount: {
+          sql: 'source',
+          type: 'countDistinct'
+        },
+        
+        countDistinctApprox: {
+          sql: 'id',
+          type: 'countDistinctApprox'
+        },
+        
+        ratio: {
+          sql: \`1.0 * \${uniqueSourceCount} / nullif(\${checkinsTotal}, 0)\`,
+          type: 'number'
+        }
+      },
+
+      dimensions: {
+        id: {
+          type: 'number',
+          sql: 'id',
+          primaryKey: true
+        },
+        source: {
+          type: 'string',
+          sql: 'source'
+        },
+        createdAt: {
+          type: 'time',
+          sql: 'created_at'
+        }
+      },
+      
+      segments: {
+        google: {
+          sql: \`source = 'google'\`
+        }
+      },
+      
+      preAggregations: {
+        partitioned: {
+          type: 'rollup',
+          measureReferences: [count],
+          dimensionReferences: [source],
+          timeDimensionReference: createdAt,
+          granularity: 'day',
+          partitionGranularity: 'month'
+        }
+      }
+    })
+    `);
+
+  function replaceTableName(query, preAggregation, suffix) {
+    const [toReplace, params] = query;
+    console.log(toReplace);
+    preAggregation = Array.isArray(preAggregation) ? preAggregation : [preAggregation];
+    return [
+      preAggregation.reduce((replacedQuery, desc) =>
+        replacedQuery.replace(new RegExp(desc.tableName, 'g'), desc.tableName + '_' + suffix), toReplace
+      ),
+      params
+    ];
+  }
+
+  function tempTablePreAggregations(preAggregationsDescriptions) {
+    return R.unnest(preAggregationsDescriptions.map(desc =>
+      desc.invalidateKeyQueries.concat([
+        [desc.loadSql[0].replace('CREATE TABLE', 'CREATE TEMPORARY TABLE'), desc.loadSql[1]]
+      ])
+    ));
+  }
+
+  it('partitioned', () => {
+    return compiler.compile().then(() => {
+      const query = new MySqlQuery({ joinGraph, cubeEvaluator, compiler }, {
+        measures: [
+          'visitors.count'
+        ],
+        dimensions: [
+          'visitors.source'
+        ],
+        timezone: 'America/Los_Angeles',
+        preAggregationsSchema: '',
+        timeDimensions: [{
+          dimension: 'visitors.createdAt',
+          granularity: 'day',
+          dateRange: ['2016-12-30', '2017-01-05']
+        }],
+        order: [{
+          id: 'visitors.createdAt'
+        }],
+      });
+
+      const queryAndParams = query.buildSqlAndParams();
+      console.log(queryAndParams);
+      const preAggregationsDescription = query.preAggregations.preAggregationsDescription();
+      console.log(preAggregationsDescription);
+
+      const queries = tempTablePreAggregations(preAggregationsDescription);
+
+      console.log(JSON.stringify(queries.concat(queryAndParams)));
+
+      return dbRunner.testQueries(
+        queries.concat([queryAndParams]).map(q => replaceTableName(q, preAggregationsDescription, 42))
+      ).then(res => {
+        console.log(JSON.stringify(res));
+        res.should.be.deepEqual(
+          [
+            {
+              "visitors__source": "some",
+              "visitors__created_at_day": "2017-01-02T00:00:00.000",
+              "visitors__count": 1
+            },
+            {
+              "visitors__source": "some",
+              "visitors__created_at_day": "2017-01-04T00:00:00.000",
+              "visitors__count": 1
+            },
+            {
+              "visitors__source": "google",
+              "visitors__created_at_day": "2017-01-05T00:00:00.000",
+              "visitors__count": 1
+            }
+          ]
+        );
+      });
+    });
+  });
+});