diff --git a/.travis.yml b/.travis.yml index 1e015a3..b785995 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ language: php php: - - 7 - 7.1 - 7.2 + - 7.3 env: - CLICKHOUSE_VERSION=latest @@ -18,4 +18,4 @@ before_script: - docker run -d -p 127.0.0.1:8123:8123 --name test-clickhouse-server --ulimit nofile=262144:262144 yandex/clickhouse-server:$CLICKHOUSE_VERSION - docker logs test-clickhouse-server -script: ./vendor/bin/phpunit \ No newline at end of file +script: ./vendor/bin/phpunit diff --git a/ArrayExpressionBuilder.php b/ArrayExpressionBuilder.php new file mode 100644 index 0000000..62f3351 --- /dev/null +++ b/ArrayExpressionBuilder.php @@ -0,0 +1,89 @@ +getValue(); + if ($value === null) { + return 'NULL'; + } + + if ($value instanceof Query) { + list ($sql, $params) = $this->queryBuilder->build($value, $params); + return $this->buildSubqueryArray($sql); + } + + $placeholders = $this->buildPlaceholders($expression, $params); + + return '[' . implode(', ', $placeholders) . ']'; + } + + /** + * Builds placeholders array out of $expression values + * @param ExpressionInterface|ArrayExpression $expression + * @param array $params the binding parameters. + * @return array + */ + protected function buildPlaceholders(ExpressionInterface $expression, &$params): array + { + $value = $expression->getValue(); + + $placeholders = []; + if ($value === null || (!is_array($value) && !$value instanceof Traversable)) { + return $placeholders; + } + + if ($expression->getDimension() > 1) { + foreach ($value as $item) { + $placeholders[] = $this->build($this->unnestArrayExpression($expression, $item), $params); + } + return $placeholders; + } + + foreach ($value as $item) { + if ($item instanceof Query) { + list ($sql, $params) = $this->queryBuilder->build($item, $params); + $placeholders[] = $this->buildSubqueryArray($sql); + continue; + } + + if ($item instanceof ExpressionInterface) { + $placeholders[] = $this->queryBuilder->buildExpression($item, $params); + continue; + } + + $placeholders[] = $this->queryBuilder->bindParam($item, $params); + } + + return $placeholders; + } + + private function unnestArrayExpression(ArrayExpression $expression, $value): ArrayExpression + { + $expressionClass = get_class($expression); + + return new $expressionClass($value, $expression->getType(), $expression->getDimension() - 1); + } + + protected function buildSubqueryArray($sql): string + { + return "array({$sql})"; + } +} diff --git a/ColumnSchema.php b/ColumnSchema.php index edd36a6..b8daf2d 100644 --- a/ColumnSchema.php +++ b/ColumnSchema.php @@ -2,6 +2,7 @@ namespace bashkarev\clickhouse; +use yii\db\ArrayExpression; use yii\db\Expression; /** @@ -20,6 +21,10 @@ public function dbTypecast($value) $value = new Expression($value); } + if (strpos($this->dbType, 'Array(') === 0) { + return new ArrayExpression($value, $this->type); + } + return parent::dbTypecast($value); } } diff --git a/Command.php b/Command.php index 5016957..0ea2496 100644 --- a/Command.php +++ b/Command.php @@ -7,7 +7,10 @@ namespace bashkarev\clickhouse; +use ClickHouseDB\Exception\DatabaseException; +use ClickHouseDB\Statement; use Yii; +use yii\db\Exception; /** * @property Connection $db @@ -23,12 +26,6 @@ protected function queryInternal($method, $fetchMode = null) { $rawSql = $this->getRawSql(); - // Add LIMIT 1 for single result SELECT queries to save transmission bandwidth and properly reuse socket - if (in_array($fetchMode, ['fetch', 'fetchColumn']) && !preg_match('/LIMIT\s+\d+$/i', $rawSql)) { - Yii::trace('LIMIT 1 added for single result query explicitly! Try to add LIMIT 1 manually', 'bashkarev\clickhouse\Command::query'); - $rawSql = $rawSql.' LIMIT 1'; - } - Yii::info($rawSql, 'bashkarev\clickhouse\Command::query'); if ($method !== '') { $info = $this->db->getQueryCacheInfo($this->queryCacheDuration, $this->queryCacheDependency); @@ -50,17 +47,19 @@ protected function queryInternal($method, $fetchMode = null) } } } - $generator = $this->db->execute(); + $token = $rawSql; try { Yii::beginProfile($token, 'bashkarev\clickhouse\Command::query'); - $generator->send($this->createRequest($rawSql, true)); - $generator->send(false); + $statement = $this->db->executeSelect($rawSql); if ($method === '') { - return $generator; + return $statement->rows(); } - $result = call_user_func_array([$this, $method], [$generator, $fetchMode]); + $result = call_user_func_array([$this, $method], [$statement, $fetchMode]); + Yii::endProfile($token, 'bashkarev\clickhouse\Command::query'); + } catch (DatabaseException $e) { Yii::endProfile($token, 'bashkarev\clickhouse\Command::query'); + throw new Exception($e->getMessage()); } catch (\Exception $e) { Yii::endProfile($token, 'bashkarev\clickhouse\Command::query'); throw $e; @@ -85,18 +84,16 @@ public function execute() if ($this->sql == '') { return 0; } - $generator = $this->db->execute(); + $token = $rawSql; try { Yii::beginProfile($token, __METHOD__); - $generator->send($this->createRequest($rawSql, false)); - $generator->send(false); - while ($generator->valid()) { - $generator->next(); - } - Yii::endProfile($token, __METHOD__); + $statement = $this->db->execute($rawSql); $this->refreshTableSchema(); - return 1; + return (int)(!$statement->isError()); + } catch (DatabaseException $e) { + Yii::endProfile($token, __METHOD__); + throw new Exception($e->getMessage()); } catch (\Exception $e) { Yii::endProfile($token, __METHOD__); throw $e; @@ -110,128 +107,79 @@ public function execute() */ public function queryBatchInternal($size) { - $rawSql = $this->getRawSql(); + // TODO: real batch select + $allRows = $this->queryAll(); + $count = 0; + $index = 0; $rows = []; - Yii::info($rawSql, 'bashkarev\clickhouse\Command::query'); - $generator = $this->db->execute(); - $token = $rawSql; - try { - Yii::beginProfile($token, 'bashkarev\clickhouse\Command::query'); - $generator->send($this->createRequest($rawSql, true)); - $generator->send(false); - $index = 0; - while ($generator->valid()) { - $count++; - $rows[$index] = $generator->current(); - if ($count >= $size) { - yield $rows; - $rows = []; - $count = 0; - } - ++$index; - $generator->next(); - } - if ($rows !== []) { + foreach ($allRows as $row) { + $count++; + $rows[$index] = $row; + if ($count >= $size) { yield $rows; + $rows = []; + $count = 0; } - Yii::endProfile($token, 'bashkarev\clickhouse\Command::query'); - } catch (\Exception $e) { - Yii::endProfile($token, 'bashkarev\clickhouse\Command::query'); - throw $e; + $index++; } - } - - /** - * @param string $table - * @param array $files - * @param array $columns - * @return InsertFiles - */ - public function batchInsertFiles($table, $files = [], $columns = []) - { - return new InsertFiles($this->db, $table, $files, $columns); - } - /** - * @param string $sql - * @param bool $forRead - * @return string - */ - protected function createRequest($sql, $forRead) - { - $data = $sql; - $url = $this->db->getConfiguration()->prepareUrl(); - if ($forRead === true) { - $data .= ' FORMAT JSONEachRow'; + if ($rows !== []) { + yield $rows; } - $header = "POST $url HTTP/1.1\r\n"; - $header .= "Content-Length: " . strlen($data) . "\r\n"; - $header .= "\r\n"; - $header .= $data; - - return $header; } /** - * @param \Generator $generator + * @param \ClickHouseDB\Statement $statement * @param int $mode * @return array */ - protected function fetchAll(\Generator $generator, $mode) + protected function fetchAll(Statement $statement, $mode) { - $result = []; - if ($mode === \PDO::FETCH_COLUMN) { - while ($generator->valid()) { - $result[] = current($generator->current()); - $generator->next(); - } - } else { - while ($generator->valid()) { - $result[] = $generator->current(); - $generator->next(); - } + $result = $statement->rows(); + + if ($result === null) { + return []; } - return $result; + + if ($mode !== \PDO::FETCH_COLUMN) { + return $result; + } + + $firstRow = current($result); + if ($firstRow === false) { + return []; + } + + $firstKey = current(array_keys($firstRow)); + + return array_column($result, $firstKey); + } /** - * @param \Generator $generator + * @param \ClickHouseDB\Statement $statement * @param $mode * @return bool|mixed */ - protected function fetchColumn(\Generator $generator, $mode) + protected function fetchColumn(Statement $statement, $mode) { - if (!$generator->valid()) { + $row = $statement->fetchOne(); + + if ($row === null) { return false; } - $result = current($generator->current()); - $this->readRest($generator); - return $result; + return current($row); } /** - * @param \Generator $generator + * @param \ClickHouseDB\Statement $statement * @param $mode * @return bool|mixed */ - protected function fetch(\Generator $generator, $mode) + protected function fetch(Statement $statement, $mode) { - if (!$generator->valid()) { - return false; - } - $result = $generator->current(); - $this->readRest($generator); - - return $result; - } - - private function readRest(\Generator $generator) - { - while ($generator->valid()) { - $generator->next(); - } + return $statement->fetchOne()??false; } - } \ No newline at end of file diff --git a/Configuration.php b/Configuration.php deleted file mode 100644 index 65c3436..0000000 --- a/Configuration.php +++ /dev/null @@ -1,98 +0,0 @@ - - */ -class Configuration -{ - /** - * @var string - */ - protected $address; - /** - * @var string - */ - protected $url = '/'; - /** - * @var array - */ - protected $options = []; - - /** - * Configuration constructor. - * @param string $dsn - * @param string|null $user - * @param string|null $password - */ - public function __construct($dsn, $user, $password) - { - $this->prepare($dsn); - if ($user !== null && $user !== '') { - $this->options['user'] = $user; - } - if ($password !== null && $password !== '') { - $this->options['password'] = $password; - } - } - - /** - * @param array $options - * @return string - */ - public function prepareUrl($options = []) - { - if ($options === [] && $this->options === []) { - return $this->url; - } - if (isset($options['user']) || isset($options['password'])) { - throw new InvalidParamException('Do not change user or password'); - } - return $this->url . '?' . http_build_query(array_merge($this->options, $options)); - } - - /** - * @return string Address to the socket to connect to. - */ - public function getAddress() - { - return $this->address; - } - - /** - * @param string $dsn - */ - protected function prepare($dsn) - { - foreach (explode(';', $dsn) as $item) { - if ($item === '' || strpos($item, '=') === false) { - continue; - } - list($key, $value) = explode('=', $item); - $this->options[$key] = $value; - } - - $this->address = 'tcp://'; - if (isset($this->options['host'])) { - $this->address .= $this->options['host']; - unset($this->options['host']); - } else { - $this->address .= '127.0.0.1'; - } - if (isset($this->options['port'])) { - $this->address .= ":{$this->options['port']}"; - unset($this->options['port']); - } else { - $this->address .= ':8123'; - } - } - -} \ No newline at end of file diff --git a/Connection.php b/Connection.php index 6f53b9f..d093b9f 100644 --- a/Connection.php +++ b/Connection.php @@ -7,7 +7,9 @@ namespace bashkarev\clickhouse; +use ClickHouseDB\Client; use Yii; +use yii\base\InvalidConfigException; use yii\base\NotSupportedException; /** @@ -20,23 +22,17 @@ class Connection extends \yii\db\Connection /** * @inheritdoc */ - public $schemaMap; + public $commandClass = 'bashkarev\clickhouse\Command'; + /** - * @inheritdoc + * @var Client */ - public $commandClass = 'bashkarev\clickhouse\Command'; + private $_client; + /** * @var Schema */ private $_schema; - /** - * @var ConnectionPool - */ - private $_pool; - /** - * @var Configuration - */ - private $_configuration; /** * @inheritdoc @@ -52,74 +48,96 @@ public function __construct($config = []) /** * @inheritdoc + * @throws InvalidConfigException */ public function open() { - $this->getPool()->open(); + if ($this->_client === null) { + $config = $this->parseDsn(); + + $this->_client = new Client([ + 'host' => $config['host'] ?? '127.0.0.1', + 'port' => $config['port'] ?? 8123, + 'username' => $this->username, + 'password' => $this->password, + ], + array_merge([ + 'database' => $config['database'] ?? 'default', + ], $this->attributes ?? []) + ); + } } /** - * @inheritdoc + * @throws InvalidConfigException */ - public function close() + private function parseDsn(): array { - if ($this->_pool !== null) { - $this->_pool->close(); + $parts = explode(';', $this->dsn); + $config = []; + foreach ($parts as $part) { + $paramValue = explode('=', $part); + + if (empty($paramValue[0])) { + throw new InvalidConfigException("Invalid (empty) param name in dsn"); + } + + if (empty($paramValue[1])) { + throw new InvalidConfigException("Invalid (empty) param '{$paramValue[0]}' value in dsn"); + } + + $config[$paramValue[0]] = $paramValue[1]; } + + return $config; } /** - * @return \Generator + * @inheritdoc */ - public function execute() + public function close() { - $socket = $this->getPool()->open(); - $socket->lock(); - while (true) { - $data = yield; - if ($data === false) { - break 1; - } - $socket->write($data); + if ($this->_client !== null) { + $this->_client = null; } - yield from (new Parser())->run($socket->getNative()); - $socket->unlock(); } /** - * @return Schema + * @param string $sql + * @return \ClickHouseDB\Statement */ - public function getSchema() + public function execute(string $sql) { - if ($this->_schema === null) { - $this->_schema = Yii::createObject([ - 'class' => 'bashkarev\clickhouse\Schema', - 'db' => $this - ]); - } - return $this->_schema; + $this->open(); + + return $this->_client->write($sql); } /** - * @return ConnectionPool + * @param string $sql + * @return \ClickHouseDB\Statement + * @throws \Exception */ - public function getPool() + public function executeSelect(string $sql) { - if ($this->_pool === null) { - $this->_pool = new ConnectionPool($this); - } - return $this->_pool; + $this->open(); + + return $this->_client->select($sql); } /** - * @return Configuration + * @return Schema + * @throws InvalidConfigException */ - public function getConfiguration() + public function getSchema() { - if ($this->_configuration === null) { - $this->_configuration = new Configuration($this->dsn, $this->username, $this->password); + if ($this->_schema === null) { + $this->_schema = Yii::createObject([ + 'class' => 'bashkarev\clickhouse\Schema', + 'db' => $this + ]); } - return $this->_configuration; + return $this->_schema; } /** @@ -127,7 +145,7 @@ public function getConfiguration() */ public function getIsActive() { - return ($this->_pool !== null && $this->_pool->total() !== 0); + return ($this->_client !== null); } /** diff --git a/ConnectionPool.php b/ConnectionPool.php deleted file mode 100644 index 8ef1772..0000000 --- a/ConnectionPool.php +++ /dev/null @@ -1,93 +0,0 @@ - - */ -class ConnectionPool -{ - /** - * @var Connection - */ - public $db; - /** - * @var Socket[] - */ - private $_sockets = []; - /** - * @var Socket - */ - private $_main; - - public function __construct(Connection $db) - { - $this->db = $db; - } - - /** - * @return Socket - * @throws Exception - */ - public function open() - { - $main = $this->getMain(); - if ($main->isLocked() === false) { - return $main; - } - foreach ($this->_sockets as $socket) { - if ($socket->isLocked() === false) { - return $socket; - } - } - $socket = new Socket($this->db->getConfiguration()); - $this->_sockets[] = $socket; - return $socket; - } - - /** - * @return Socket - */ - protected function getMain() - { - if ($this->_main === null) { - $this->_main = new Socket($this->db->getConfiguration()); - } - return $this->_main; - } - - /** - * Close all socket connection - */ - public function close() - { - if ($this->_main === null) { - return; - } - $this->_main->close(); - foreach ($this->_sockets as $socket) { - $socket->close(); - } - $this->_main = null; - $this->_sockets = []; - } - - /** - * @return int total open socket connections - */ - public function total() - { - if ($this->_main === null) { - return 0; - } - return count($this->_sockets) + 1; - } - -} \ No newline at end of file diff --git a/FileNotFoundException.php b/FileNotFoundException.php deleted file mode 100644 index e01bb82..0000000 --- a/FileNotFoundException.php +++ /dev/null @@ -1,24 +0,0 @@ - - */ -class FileNotFoundException extends InvalidParamException -{ - /** - * @inheritdoc - */ - public function getName() - { - return 'File not Found'; - } -} \ No newline at end of file diff --git a/InsertFiles.php b/InsertFiles.php deleted file mode 100644 index cd59db3..0000000 --- a/InsertFiles.php +++ /dev/null @@ -1,182 +0,0 @@ - - */ -class InsertFiles -{ - /** - * @var int Maximum chunk size. - */ - protected $chunkSize = 4096; - /** - * @var Connection - */ - protected $db; - /** - * @var string - */ - protected $url; - /** - * @var string - */ - protected $sql; - /** - * @var array - */ - protected $files = []; - - /** - * InsertFiles constructor. - * @param Connection $db - * @param string $table - * @param array $files - * @param array $columns - */ - public function __construct(Connection $db, $table, $files = [], $columns = []) - { - $this->db = $db; - $this->prepare($table, $columns); - if ($files !== []) { - $this->setFiles($files); - } - } - - /** - * @param mixed $files - * @return $this - * @throws \Exception - */ - public function setFiles($files) - { - if (!is_array($files)) { - $files = (array)$files; - } - foreach ($files as $file) { - if (is_resource($file)) { - rewind($file); - } else { - $file = \Yii::getAlias($file); - if (file_exists($file) === false) { - throw new FileNotFoundException("File: `{$file}` not found"); - } - } - $this->files[] = $file; - } - return $this; - } - - /** - * @param int $size - * @return $this - */ - public function setChunkSize($size) - { - $size = (int)$size; - if ($size < 1) { - throw new InvalidParamException('The size must be greater than 0'); - } - $this->chunkSize = $size; - return $this; - } - - /** - * @return array - */ - public function getFiles() - { - return $this->files; - } - - /** - * @param string $table - * @param array $columns - */ - protected function prepare($table, $columns) - { - $this->sql = 'INSERT INTO ' . $this->db->getSchema()->quoteTableName($table); - if ($columns !== []) { - $this->sql .= ' (' . implode(', ', $columns) . ')'; - } - $this->sql .= ' FORMAT CSV'; - $this->url = $this->db->getConfiguration()->prepareUrl(['query' => $this->sql]); - } - - /** - * @param string|resource $file - * @return \Generator - */ - protected function runItem($file) - { - if (is_resource($file)) { - $closeFile = false; - $handle = $file; - $token = $this->sql . ' `' . $file . '`'; - } else { - $closeFile = true; - $handle = fopen($file, 'rb'); - $token = $this->sql . ' `' . basename($file) . '`'; - } - - Yii::info($token, 'bashkarev\clickhouse\Command::query'); - Yii::beginProfile($token, 'bashkarev\clickhouse\Command::query'); - $generator = $this->db->execute(); - $generator->send("POST {$this->url} HTTP/1.1\r\n"); - $generator->send("Transfer-Encoding: chunked\r\n\r\n"); - while (true) { - $data = fread($handle, $this->chunkSize); - if ($data === false || ($length = strlen($data)) === 0) { - $generator->send("0\r\n\r\n"); - $generator->send(false); - break 1; - } - $generator->send(dechex($length) . "\r\n"); - $generator->send($data . "\r\n"); - yield; - } - - if ($closeFile === true) { - fclose($handle); - } - - while ($generator->valid()) { - $generator->next(); - yield; - } - Yii::endProfile($token, 'bashkarev\clickhouse\Command::query'); - } - - /** - * Execute - * @throws Exception - */ - public function execute() - { - $queue = new \SplQueue(); - foreach ($this->files as $file) { - $queue->enqueue($this->runItem($file)); - } - - while (!$queue->isEmpty()) { - $task = $queue->dequeue(); - $task->next(); - if ($task->valid()) { - $queue->enqueue($task); - } - } - $this->files = []; - } - - -} \ No newline at end of file diff --git a/Parser.php b/Parser.php deleted file mode 100644 index e54941b..0000000 --- a/Parser.php +++ /dev/null @@ -1,157 +0,0 @@ - - */ -class Parser -{ - const POS_HEADER = 0x01; - const POS_LENGTH = 0x02; - const POS_CONTENT = 0x03; - const POS_END = 0x04; - - const CRLF = "\r\n"; - - /** - * @var int - */ - protected $position = 0x01; - /** - * @var int - */ - protected $httpCode; - /** - * @var int - */ - protected $length; - /** - * @var string - */ - protected $last; - - /** - * @param resource $socket - * @return \Generator - */ - public function run($socket) - { - while (true) { - if ($this->position === self::POS_HEADER) { - $line = fgets($socket, 1024); - if ($line === false) { - continue; - } - $this->parseHeader($line); - } - if ($this->position === self::POS_LENGTH) { - $line = fgets($socket, 11); - if ($line === false || $line === self::CRLF) { - continue; - } - $this->parseLength($line); - } - if ($this->position === self::POS_CONTENT) { - yield from $this->parseContent(fread($socket, $this->length)); - } - if ($this->position === self::POS_END) { - fseek($socket, 2, SEEK_CUR); // \r\n end - if (($last = $this->getLastContent()) !== null) { - yield $last; - } - break 1; - } - } - } - - /** - * @param $buffer - * @return \Generator - * @throws Exception - */ - protected function parseContent($buffer) - { - if ($this->httpCode !== 200) { - throw new Exception($buffer); - } - $lines = explode("\n", $buffer); - $count = count($lines) - 1; - for ($i = 0; ; $i++) { - if ($i === $count) { - $this->last = $lines[$i]; - break 1; - } - - $line = $lines[$i]; - if ($i === 0 && $this->last !== null) { - $line = $this->last . $line; - $this->last = null; - } - $value = $this->parseContentLine($line); - if ($value !== null) { - yield $value; - } - } - $this->position = self::POS_LENGTH; - } - - /** - * @param $value - * @return array|null - */ - protected function parseContentLine($value) - { - return json_decode($value, true); - } - - /** - * @return mixed - */ - protected function getLastContent() - { - if ($this->last === null || $this->last === '') { - return null; - } - return $this->parseContentLine($this->last); - } - - /** - * @param $line - */ - protected function parseHeader($line) - { - if ($this->httpCode === null) { - $this->parseCode($line); - } - - if ($line === self::CRLF || $line === PHP_EOL) { - $this->position = self::POS_LENGTH; - } - } - - /** - * @param $line - */ - protected function parseLength($line) - { - $this->length = hexdec($line); - $this->position = ($this->length === 0) ? self::POS_END : self::POS_CONTENT; - } - - /** - * @param string $line - */ - protected function parseCode($line) - { - $this->httpCode = (int)substr($line, 9, 3); - } - -} diff --git a/QueryBuilder.php b/QueryBuilder.php index b83d75e..c8cd0b0 100644 --- a/QueryBuilder.php +++ b/QueryBuilder.php @@ -7,6 +7,7 @@ namespace bashkarev\clickhouse; +use yii\db\ArrayExpression; use yii\db\Exception; /** @@ -42,6 +43,13 @@ class QueryBuilder extends \yii\db\QueryBuilder Schema::TYPE_JSON => 'String' ]; + protected function defaultExpressionBuilders() + { + return array_merge(parent::defaultExpressionBuilders(), [ + ArrayExpression::class => ArrayExpressionBuilder::class, + ]); + } + /** * @inheritdoc */ @@ -86,4 +94,4 @@ public function addColumn($table, $column, $type) . ' ADD COLUMN ' . $this->db->quoteColumnName($column) . ' ' . $this->getColumnType($type); } -} \ No newline at end of file +} diff --git a/Socket.php b/Socket.php deleted file mode 100644 index 0c5f373..0000000 --- a/Socket.php +++ /dev/null @@ -1,131 +0,0 @@ - - */ -class Socket -{ - /** - * @var resource - */ - protected $socket; - /** - * @var Configuration - */ - protected $config; - /** - * @var bool - */ - private $_lock = false; - - public function __construct(Configuration $config) - { - $this->config = $config; - $this->open(); - } - - public function __wakeup() - { - $this->open(); - } - - /** - * Open socket connection - * @throws SocketException - */ - public function open() - { - $this->socket = @stream_socket_client($this->config->getAddress(), $code, $message); - if ($this->socket === false) { - throw new SocketException($message, [], $code); - } - if (stream_set_blocking($this->socket, false) === false) { - throw new SocketException('Failed set non blocking socket'); - } - if (YII_DEBUG) { - Yii::trace("Opening clickhouse DB connection: " . $this->config->getAddress() . " ($this->socket)", __METHOD__); - } - } - - /** - * Close socket connection - */ - public function close() - { - if (YII_DEBUG) { - Yii::trace("Closing clickhouse DB connection: " . $this->config->getAddress() . " ($this->socket)", __METHOD__); - } - stream_socket_shutdown($this->socket, STREAM_SHUT_RDWR); - } - - /** - * @param string $string - * @param null|int $length - * @throws SocketException - */ - public function write($string, $length = null) - { - if ($length === null) { - $length = strlen($string); - } - - while (true) { - $bytes = @fwrite($this->socket, $string); - if ($bytes === false || $bytes === 0) { - $message = "Failed to write to socket"; - if ($error = error_get_last()) { - $message .= sprintf(" Errno: %d; %s", $error["type"], $error["message"]); - } - throw new SocketException($message); - } - - if ($bytes < $length) { - $string = substr($string, $bytes); - $length -= $bytes; - } else { - break 1; - } - } - } - - /** - * Lock - */ - public function lock() - { - $this->_lock = true; - } - - /** - * Unlock - */ - public function unlock() - { - $this->_lock = false; - } - - /** - * @return bool - */ - public function isLocked() - { - return $this->_lock; - } - - /** - * @return resource - */ - public function getNative() - { - return $this->socket; - } -} \ No newline at end of file diff --git a/SocketException.php b/SocketException.php deleted file mode 100644 index f75f128..0000000 --- a/SocketException.php +++ /dev/null @@ -1,41 +0,0 @@ - - */ -class SocketException extends Exception -{ - - /** - * @inheritdoc - */ - public function __construct($message, $errorInfo = [], $code = 0, \Exception $previous = null) - { - if ($code === 0) { - $code = $this->parseCode($message); - } - parent::__construct($message, $errorInfo, $code, $previous); - } - - /** - * @param $message - * @return int - */ - protected function parseCode($message) - { - if (preg_match('/errno=(\d+)/', $message, $out)) { - return (int)$out[1]; - } - return 0; - } - -} \ No newline at end of file diff --git a/composer.json b/composer.json index 60978ce..95427cd 100644 --- a/composer.json +++ b/composer.json @@ -5,8 +5,9 @@ "type": "yii2-extension", "license": "MIT", "require": { - "php": ">=7.0", - "yiisoft/yii2": "~2.0.13" + "php": "^7.1", + "yiisoft/yii2": "~2.0.13", + "smi2/phpclickhouse": "^1.3.3" }, "require-dev": { "phpunit/phpunit": "~5.7" @@ -20,6 +21,10 @@ { "name": "bashkarev", "email": "dmitry@bashkarev.com" + }, + { + "name": "sartor", + "email": "sartorua@gmail.com" } ], "repositories": [ diff --git a/helpers/Csv.php b/helpers/Csv.php deleted file mode 100644 index 8b2618b..0000000 --- a/helpers/Csv.php +++ /dev/null @@ -1,127 +0,0 @@ - - */ -class Csv -{ - - const EOL = "\n"; - - /** - * @param mixed $value - * @return string - */ - public static function toString($value) - { - $str = null; - foreach ($value as $item) { - if ($str !== null) { - $str .= ','; - } - $type = 'set' . gettype($item); - $str .= static::$type($item); - } - return $str; - } - - /** - * @param resource $handle - * @param array $fields - * @return int|bool - */ - public static function write($handle, $fields) - { - $line = static::toString($fields); - if ($line === '') { - return 0; - } - return fwrite($handle, $line . self::EOL); - } - - /** - * @param bool $value - * @return int - */ - protected static function setBoolean($value) - { - return (int)$value; - } - - /** - * @param null $value - * @return string - */ - protected static function setNull($value) - { - return ''; - } - - /** - * @param int $value - * @return int - */ - protected static function setInteger($value) - { - return $value; - } - - /** - * @param float $value - * @return string - */ - protected static function setDouble($value) - { - return StringHelper::normalizeNumber($value); - } - - /** - * @param string $value - * @return string - */ - protected static function setString($value) - { - if ( - strpos($value, ',') !== false - || strpos($value, '"') !== false - ) { - $value = '"' . str_replace('"', '""', $value) . '"'; - } - return addcslashes($value, "\r\n"); - } - - /** - * toDo - * @param object|array $value - */ - protected static function setArray($value) - { - throw new \RuntimeException('Type `array` is not supported'); - } - - /** - * @param object $value - */ - protected static function setObject($value) - { - return static::setArray($value); - } - - /** - * @throws \RuntimeException - */ - protected static function setResource() - { - throw new \RuntimeException('Type `resource` is not supported'); - } - -} \ No newline at end of file diff --git a/tests/CommandTest.php b/tests/CommandTest.php index 7ccc89c..1767df4 100644 --- a/tests/CommandTest.php +++ b/tests/CommandTest.php @@ -49,7 +49,7 @@ public function testQuery() // query $sql = 'SELECT * FROM {{customer}}'; $reader = $db->createCommand($sql)->query(); - $this->assertInstanceOf(\Generator::class, $reader); + $this->assertEquals(true, is_array($reader)); // queryAll $rows = $db->createCommand('SELECT * FROM {{customer}} ORDER BY [[id]]')->queryAll(); @@ -104,4 +104,25 @@ public function testQueryBatchInternal() $data = iterator_to_array((new Query())->from('customer')->each(1, $db), false); $this->assertCount(3, $data); } -} \ No newline at end of file + + // todo nested + public function testArrays() + { + $db = $this->getConnection(); + + $dataForInsert = [ + 'Array_UInt8' => [5], + 'Array_Float64' => [5.5], + 'Array_String' => ['asdasd'], + 'Array_DateTime' => [date('Y-m-d H:i:s')], + 'Array_Nullable_Decimal' => [null, null], + 'Array_FixedString_empty' => [], + ]; + + $this->assertEquals(1, $db->createCommand()->insert('{{arrays}}', $dataForInsert)->execute()); + + $dataFromSelect = $db->createCommand('SELECT * FROM {{arrays}}')->queryOne(); + + $this->assertEquals($dataForInsert, $dataFromSelect); + } +} diff --git a/tests/ConfigurationTest.php b/tests/ConfigurationTest.php deleted file mode 100644 index 3e49458..0000000 --- a/tests/ConfigurationTest.php +++ /dev/null @@ -1,62 +0,0 @@ - - */ -class ConfigurationTest extends DatabaseTestCase -{ - - public function testConstruct() - { - $config = new Configuration('', '', ''); - $this->assertEquals('tcp://127.0.0.1:8123', $config->getAddress()); - $this->assertEquals('/', $config->prepareUrl()); - } - - public function testUnknownSetting() - { - $config = self::getParam('database'); - $config['dsn'] = 'unknown-setting=test'; - unset($config['fixture']); - $connection = $this->prepareDatabase($config, null, false); - $this->expectException(Exception::class); - $connection->createCommand('SELECT 123')->queryScalar(); - } - - public function testAddress() - { - $this->assertEquals('tcp://localhost:1010', (new Configuration('host=localhost;port=1010', '', ''))->getAddress()); - $this->assertEquals('tcp://127.0.0.1:1011', (new Configuration('port=1011', '', ''))->getAddress()); - $this->assertEquals('tcp://[2001:0db8:11a3:09d7:1f34:8a2e:07a0:765d]:8123', (new Configuration('host=[2001:0db8:11a3:09d7:1f34:8a2e:07a0:765d]', '', ''))->getAddress()); - $this->assertEquals('tcp://0.0.0.0:8123', (new Configuration('host=0.0.0.0', '', ''))->getAddress()); - } - - public function testPrepareUrl() - { - $this->assertEquals('/?ya=ya', (new Configuration('ya=ya', '', ''))->prepareUrl()); - $this->assertEquals('/?user=ya&password=ya', (new Configuration('', 'ya', 'ya'))->prepareUrl()); - - $this->assertEquals('/?user=ya&password=ya&temp=te+mp', (new Configuration('', 'ya', 'ya'))->prepareUrl(['temp' => 'te mp'])); - $this->assertEquals('/?ya=ya&temp=te+mp', (new Configuration('', '', ''))->prepareUrl(['ya' => 'ya', 'temp' => 'te mp'])); - - } - - public function testChangeUserOrPassword() - { - $this->assertEquals('/?user=ya&password=ya', (new Configuration('user=ttt;password=ttt', 'ya', 'ya'))->prepareUrl()); - - $this->expectException(\LogicException::class); - (new Configuration('', 'ya', 'ya'))->prepareUrl(['user' => 'ttt', 'password' => 'ttt']); - } - -} \ No newline at end of file diff --git a/tests/ConnectionTest.php b/tests/ConnectionTest.php index c1418d6..38e074f 100644 --- a/tests/ConnectionTest.php +++ b/tests/ConnectionTest.php @@ -8,6 +8,7 @@ namespace bashkarev\clickhouse\tests; use bashkarev\clickhouse\Connection; +use yii\base\InvalidConfigException; use yii\base\NotSupportedException; use yii\db\Exception; @@ -38,10 +39,33 @@ public function testOpenClose() $connection->close(); $this->assertFalse($connection->isActive); + } + + public function testAutoOpen() + { + $connection = $this->getConnection(false, false); + + $this->assertFalse($connection->isActive); + + $connection->execute("SELECT 1"); + $this->assertTrue($connection->isActive); + + $connection->close(); + $this->assertFalse($connection->isActive); + + $connection->executeSelect("SELECT 1"); + $this->assertTrue($connection->isActive); + } + + public function testInvalidConfig() + { + $connection = $this->getConnection(false, false); + + $this->assertFalse($connection->isActive); $connection = new Connection(); - $connection->dsn = 'port=unknown'; - $this->expectException(Exception::class); + $connection->dsn = 'port='; + $this->expectException(InvalidConfigException::class); $connection->open(); } diff --git a/tests/InsertFilesTest.php b/tests/InsertFilesTest.php deleted file mode 100644 index e31e787..0000000 --- a/tests/InsertFilesTest.php +++ /dev/null @@ -1,82 +0,0 @@ - - */ -class InsertFilesTest extends DatabaseTestCase -{ - - public function testSetFileAlias() - { - \Yii::setAlias('@InsertFilesTest', __DIR__); - $this->assertEquals([__FILE__], $this->getInsertFiles()->setFiles(['@InsertFilesTest/InsertFilesTest.php'])->getFiles()); - } - - public function testFileNotFound() - { - $this->expectException(FileNotFoundException::class); - $this->getInsertFiles()->setFiles(__DIR__ . '/FileNotFound'); - } - - public function testExecute() - { - $db = $this->getConnection(); - $this->getInsertFiles($db)->setFiles([ - '@data/csv/e1e747f9901e67ca121768b36921fbae.csv', - '@data/csv/ebe191dfc36d73aece91e92007d24e3e.csv', - '@data/csv/empty.csv' - ])->execute(); - $count = (new Query)->from('csv')->count('*', $db); - $this->assertContains('2000', $count); - } - - public function testSetFileResource() - { - $file = fopen(\Yii::getAlias('@data/csv/e1e747f9901e67ca121768b36921fbae.csv'), 'rb'); - fseek($file, 100); - $insert = $this->getInsertFiles()->setFiles($file); - $this->assertTrue((int)$file === (int)$insert->getFiles()[0], 'exist'); - $this->assertEquals(0, ftell($insert->getFiles()[0]), 'rewind'); - - $insert = $this->getInsertFiles()->setFiles([$file]); - $this->assertTrue((int)$file === (int)$insert->getFiles()[0], 'exist'); - } - - public function testInvalidChunkSize() - { - $this->expectException(\yii\base\InvalidParamException::class); - $this->getInsertFiles()->setChunkSize(0); - } - - public function testInvalidFile() - { - $this->expectException(Exception::class); - $this->getInsertFiles()->setFiles('@data/csv/not_valid.csv')->execute(); - } - - /** - * @param Connection|null $db - * @return InsertFiles - */ - protected function getInsertFiles($db = null) - { - if ($db === null) { - $db = $this->getConnection(false, false); - } - return new InsertFiles($db, 'csv'); - } - -} \ No newline at end of file diff --git a/tests/ParserTest.php b/tests/ParserTest.php deleted file mode 100644 index 3093497..0000000 --- a/tests/ParserTest.php +++ /dev/null @@ -1,49 +0,0 @@ - - */ -class ParserTest extends TestCase -{ - - public function testEmpty() - { - $this->assertEmpty(iterator_to_array($this->getParserGenerator('empty.txt'))); - } - - public function testValid() - { - $this->assertEquals([['123' => 123]], iterator_to_array($this->getParserGenerator('one.txt'))); - $this->assertEquals([['COUNT()' => '2000']], iterator_to_array($this->getParserGenerator('count.txt'))); - $this->assertCount(3, $this->getParserGenerator('ids.txt')); - $this->assertCount(3, $this->getParserGenerator('customers.txt')); - } - - public function testError() - { - $this->expectException(Exception::class); - iterator_to_array($this->getParserGenerator('error.txt')); - } - - /** - * @param string $file - * @return \Generator - */ - protected function getParserGenerator($file) - { - return (new Parser())->run(fopen(Yii::getAlias("@data/parser/$file"), 'rb')); - } - - -} \ No newline at end of file diff --git a/tests/SocketTest.php b/tests/SocketTest.php deleted file mode 100644 index 2dccec1..0000000 --- a/tests/SocketTest.php +++ /dev/null @@ -1,97 +0,0 @@ - - */ -class SocketTest extends TestCase -{ - private $server; - private $port; - - protected function setUp() - { - parent::setUp(); - - $this->prepareDummyServer(); - } - - protected function tearDown() - { - $this->shutdownDummyServer(); - - parent::tearDown(); - } - - private function prepareDummyServer() - { - $this->server = stream_socket_server("tcp://0.0.0.0:0"); - $name = stream_socket_get_name($this->server, 0); - - $this->port = substr($name, strpos($name, ':') + 1); - } - - private function shutdownDummyServer() - { - fclose($this->server); - } - - private function socket(): Socket - { - $config = new Configuration('host=0.0.0.0;port='.$this->port, '', ''); - - return new Socket($config); - } - - public function testOpen() - { - $socket = $this->socket(); - - $socket->close(); - } - - public function testWrite() - { - $socket = $this->socket(); - - $client = stream_socket_accept($this->server); - - $socket->write("Test string"); - - $socket->close(); - - $result = fgets($client); - - $this->assertEquals("Test string", $result); - } - - public function testBrokenPipe() - { - $this->expectException(SocketException::class); - $socket = $this->socket(); - - $socket->write("Write to valid pipe"); - - $socket->close(); - - $socket->write("Write to broken pipe"); - } - - public function testRead() - { - $socket = $this->socket(); - - $client = stream_socket_accept($this->server); - - fwrite($client, "Test read string"); - fclose($client); - - $result = fgets($socket->getNative()); - - $this->assertEquals("Test read string", $result); - } -} \ No newline at end of file diff --git a/tests/data/clickhouse.sql b/tests/data/clickhouse.sql index 7e340a4..071100d 100644 --- a/tests/data/clickhouse.sql +++ b/tests/data/clickhouse.sql @@ -1,6 +1,7 @@ DROP TABLE IF EXISTS `csv`; DROP TABLE IF EXISTS `customer`; DROP TABLE IF EXISTS `types`; +DROP TABLE IF EXISTS `arrays`; CREATE TABLE csv (d Date, a String, b String) ENGINE = MergeTree(d, d, 8192); @@ -39,4 +40,13 @@ CREATE TABLE `types` ( `Decimal9_2` Decimal(9, 2), `Decimal18_4` Decimal(18, 4), `Decimal38_10` Decimal(38, 10) -) ENGINE=Memory; \ No newline at end of file +) ENGINE=Memory; + + CREATE TABLE `arrays` ( + `Array_UInt8` Array(UInt8), + `Array_Float64` Array(Float64), + `Array_String` Array(String), + `Array_DateTime` Array(DateTime), + `Array_Nullable_Decimal` Array(Nullable(Decimal32(4))), + `Array_FixedString_empty` Array(FixedString(2)) + ) ENGINE=Memory; diff --git a/tests/data/parser/count.txt b/tests/data/parser/count.txt deleted file mode 100644 index 76d774b..0000000 --- a/tests/data/parser/count.txt +++ /dev/null @@ -1,11 +0,0 @@ -HTTP/1.1 200 OK -Date: Thu, 11 May 2017 19:00:21 GMT -Connection: Keep-Alive -Content-Type: text/plain; charset=UTF-8 -Transfer-Encoding: chunked -Keep-Alive: timeout=3 - -13 -{"COUNT()":"2000"} -0 - diff --git a/tests/data/parser/customers.txt b/tests/data/parser/customers.txt deleted file mode 100644 index 348f55e..0000000 --- a/tests/data/parser/customers.txt +++ /dev/null @@ -1,13 +0,0 @@ -HTTP/1.1 200 OK -Date: Thu, 11 May 2017 19:00:20 GMT -Connection: Keep-Alive -Content-Type: text/plain; charset=UTF-8 -Transfer-Encoding: chunked -Keep-Alive: timeout=3 - -129 -{"id":1,"email":"user1@example.com","name":"user1","address":"address1","status":1,"profile_id":1} -{"id":2,"email":"user2@example.com","name":"user2","address":"address2","status":1,"profile_id":0} -{"id":3,"email":"user3@example.com","name":"user3","address":"address3","status":2,"profile_id":2} -0 - diff --git a/tests/data/parser/empty.txt b/tests/data/parser/empty.txt deleted file mode 100644 index 194e9ac..0000000 --- a/tests/data/parser/empty.txt +++ /dev/null @@ -1,8 +0,0 @@ -HTTP/1.1 200 OK -Date: Thu, 11 May 2017 15:49:00 GMT -Connection: Keep-Alive -Content-Type: text/plain; charset=UTF-8 -Transfer-Encoding: chunked -Keep-Alive: timeout=3 - -0 diff --git a/tests/data/parser/error.txt b/tests/data/parser/error.txt deleted file mode 100644 index 5acb630..0000000 --- a/tests/data/parser/error.txt +++ /dev/null @@ -1,11 +0,0 @@ -HTTP/1.1 500 Internal Server Error -Date: Thu, 11 May 2017 19:00:20 GMT -Connection: Keep-Alive -Content-Type: text/plain; charset=UTF-8 -Transfer-Encoding: chunked -Keep-Alive: timeout=3 - -84 -Code: 62, e.displayText() = DB::Exception: Syntax error: failed at position 1: bad SQL, expected OPTIMIZE, e.what() = DB::Exception -0 - diff --git a/tests/data/parser/ids.txt b/tests/data/parser/ids.txt deleted file mode 100644 index 7787798..0000000 --- a/tests/data/parser/ids.txt +++ /dev/null @@ -1,13 +0,0 @@ -HTTP/1.1 200 OK -Date: Thu, 11 May 2017 19:00:20 GMT -Connection: Keep-Alive -Content-Type: text/plain; charset=UTF-8 -Transfer-Encoding: chunked -Keep-Alive: timeout=3 - -1B -{"id":1} -{"id":2} -{"id":3} -0 - diff --git a/tests/data/parser/one.txt b/tests/data/parser/one.txt deleted file mode 100644 index 35e0567..0000000 --- a/tests/data/parser/one.txt +++ /dev/null @@ -1,11 +0,0 @@ -HTTP/1.1 200 OK -Date: Thu, 11 May 2017 15:49:00 GMT -Connection: Keep-Alive -Content-Type: text/plain; charset=UTF-8 -Transfer-Encoding: chunked -Keep-Alive: timeout=3 - -C -{"123":123} -0 - diff --git a/tests/helpers/CsvTest.php b/tests/helpers/CsvTest.php deleted file mode 100644 index 3c5c2cf..0000000 --- a/tests/helpers/CsvTest.php +++ /dev/null @@ -1,59 +0,0 @@ - - */ -class CsvTest extends TestCase -{ - - public function testNull() - { - $this->assertEquals('', Csv::toString([null])); - } - - public function testInt() - { - $this->assertEquals('1,2,3', Csv::toString([1, 2, 3])); - } - - public function testFloat() - { - $this->assertEquals('1.22', Csv::toString([1.22])); - setlocale(LC_ALL, 'de_DE.UTF-8'); - $this->assertEquals('1.22', Csv::toString([1.22])); - setlocale(LC_ALL, 'en_US.UTF-8'); - } - - public function testString() - { - $this->assertEquals('2017-04-04', Csv::toString(['2017-04-04'])); - $this->assertEquals('2017-04-04 10:56:16', Csv::toString(['2017-04-04 10:56:16'])); - $this->assertEquals('"""test"', Csv::toString(['"test'])); - $this->assertEquals('"""test"""', Csv::toString(['"test"'])); - } - - public function testEmptyString() - { - $this->assertEquals(',,', Csv::toString(['', '', ''])); - } - - /** - * toDo - */ - public function testArray() - { - //$this->assertEquals('[222]', Csv::toString([[222]])); - } - -} \ No newline at end of file