diff --git a/README.md b/README.md index a87c1bb..a373ce5 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,51 @@ You can install duckdb extensions too. DB::connection('my_duckdb') ->select("SELECT * FROM read_csv_auto('s3://my-bucket/test-datasets/example1/us-gender-data-2022.csv') LIMIT 10") ``` +### Writing a migration +```php +return new class extends Migration { + protected $connection = 'my_duckdb'; + public function up(): void + { + DB::connection('my_duckdb')->statement('CREATE SEQUENCE people_sequence'); + Schema::create('people', function (Blueprint $table) { + $table->id()->default(new \Illuminate\Database\Query\Expression("nextval('people_sequence')")); + $table->string('name'); + $table->integer('age'); + $table->integer('rank'); + $table->timestamps(); + }); + } + + public function down(): void + { + Schema::dropIfExists('people'); + DB::connection('my_duckdb')->statement('DROP SEQUENCE people_sequence'); + } +}; +``` + +### Readonly Connection - A solution to concurrent query. +- in `database.php` +```php + 'connections' => [ + 'my_duckdb' => [ + 'driver' => 'duckdb', + 'cli_path' => env('DUCKDB_CLI_PATH', base_path('vendor/bin/duckdb')), + 'cli_timeout' => 0, + 'dbfile' => env('DUCKDB_DB_FILE', storage_path('app/duckdb/duck_main.db')), + 'schema' => 'main', + 'read_only' => true, + 'pre_queries' => [ + "SET s3_region='".env('AWS_DEFAULT_REGION')."'", + "SET s3_access_key_id='".env('AWS_ACCESS_KEY_ID')."'", + "SET s3_secret_access_key='".env('AWS_SECRET_ACCESS_KEY')."'", + ], + 'extensions' => ['httpfs', 'postgres_scanner'], + ], + ... +``` + ## Testing @@ -104,6 +149,10 @@ DB::connection('my_duckdb') composer test ``` +## Limitations & FAQ + +- https://duckdb.org/faq + ## Changelog Please see [CHANGELOG](CHANGELOG.md) for more information on what has changed recently. diff --git a/src/Commands/ConnectDuckdbCliCommand.php b/src/Commands/ConnectDuckdbCliCommand.php index f96f7b9..f688845 100644 --- a/src/Commands/ConnectDuckdbCliCommand.php +++ b/src/Commands/ConnectDuckdbCliCommand.php @@ -7,7 +7,7 @@ class ConnectDuckdbCliCommand extends Command { - protected $signature = 'laravel-duckdb:connect {connection_name}'; + protected $signature = 'laravel-duckdb:connect {connection_name} {--readonly=true}'; protected $description = 'Connect with duckdb cli to interactive query and development.'; @@ -16,11 +16,17 @@ class ConnectDuckdbCliCommand extends Command public function handle(): void { $connection = config('database.connections.'.$this->argument('connection_name')); + $isReadonly = filter_var($this->option('readonly'), FILTER_VALIDATE_BOOLEAN); if(!$connection || ($connection['driver']??'') !== 'duckdb') throw new \Exception("DuckDB connection named `".$this->argument('connection_name')."` not found!"); - $cmd = $connection['cli_path']." ".$connection['dbfile']; - $this->info('Connecting to duckdb cli `'.$cmd.'`'); - $this->process = Process::fromShellCommandline($cmd); + $cmd = [ + $connection['cli_path'], + $connection['dbfile'] + ]; + if($isReadonly) array_splice($cmd, 1, 0, '--readonly'); + + $this->info('Connecting to duckdb cli `'.implode(" ", $cmd).'`'); + $this->process = new Process($cmd); $this->process->setTimeout(0); $this->process->setIdleTimeout(0); $this->process->setTty(Process::isTtySupported()); diff --git a/src/LaravelDuckdbConnection.php b/src/LaravelDuckdbConnection.php index 43e6698..fe67c06 100644 --- a/src/LaravelDuckdbConnection.php +++ b/src/LaravelDuckdbConnection.php @@ -19,9 +19,9 @@ class LaravelDuckdbConnection extends PostgresConnection private $installed_extensions = []; public function __construct($config) { + $this->database = $config['database']; $this->config = $config; $this->config['dbfile'] = $config['dbfile']; - //$this->setDatabaseName($this->config['dbfile']); $this->useDefaultPostProcessor(); $this->useDefaultSchemaGrammar(); @@ -75,6 +75,7 @@ private function getDuckDBCommand($query, $bindings = [], $safeMode=false){ $this->config['cli_path'], $this->config['dbfile'], ]; + if($this->config['read_only']) array_splice($cmdParams, 1, 0, '--readonly'); if(!$safeMode) $cmdParams = array_merge($cmdParams, $preQueries); $cmdParams = array_merge($cmdParams, [ "$escapeQuery", @@ -103,7 +104,7 @@ private function installExtensions(){ } if(!empty($sql)) Cache::forget($cacheKey); foreach ($sql as $ext_name=>$sExtQuery) { - $this->statement($sExtQuery); + $this->executeDuckCliSql($sExtQuery, [], true); } $this->installed_extensions=$tobe_installed_extensions; } @@ -125,7 +126,6 @@ private function ensureDuckdbDirectory(){ private function executeDuckCliSql($sql, $bindings = [], $safeMode=false){ $command = $this->getDuckDBCommand($sql, $bindings, $safeMode); - //$process = Process::fromShellCommandline($command); $process = new Process($command); $process->setTimeout($this->config['cli_timeout']); $process->setIdleTimeout(0); @@ -145,32 +145,36 @@ private function executeDuckCliSql($sql, $bindings = [], $safeMode=false){ return json_decode($raw_output, true)??[]; } - public function statement($query, $bindings = []) - { + private function runQueryWithLog($query, $bindings=[]){ $start = microtime(true); //execute - $this->executeDuckCliSql($query, $bindings); + $result = $this->executeDuckCliSql($query, $bindings); $this->logQuery( $query, [], $this->getElapsedTime($start) ); - return true; + return $result; } - public function select($query, $bindings = [], $useReadPdo = true) + public function statement($query, $bindings = []) { - $start = microtime(true); + $this->runQueryWithLog($query, $bindings); - //execute - $result = $this->executeDuckCliSql($query, $bindings); + return true; + } - $this->logQuery( - $query, [], $this->getElapsedTime($start) - ); + public function select($query, $bindings = [], $useReadPdo = true) + { + return $this->runQueryWithLog($query, $bindings); + } - return $result; + public function affectingStatement($query, $bindings = []) + { + //for update/delete + //todo: we have to use : returning * to get list of affected rows; currently causing error; + return $this->runQueryWithLog($query, $bindings); } private function getDefaultQueryBuilder(){ diff --git a/src/LaravelDuckdbServiceProvider.php b/src/LaravelDuckdbServiceProvider.php index d3701cf..72b1b29 100644 --- a/src/LaravelDuckdbServiceProvider.php +++ b/src/LaravelDuckdbServiceProvider.php @@ -13,12 +13,17 @@ public function register(): void 'cli_path' => base_path('vendor/bin/duckdb'), 'cli_timeout' => 60, 'dbfile' => storage_path('app/duckdb/duck_main.db'), + //'database' => 'duck_main' //default to filename of dbfile, in most case no need to specify manually + 'schema' => 'main', + 'read_only' => false, 'pre_queries' => [], 'extensions' => [] ]; $this->app->resolving('db', function ($db) use ($defaultConfig) { $db->extend('duckdb', function ($config, $name) use ($defaultConfig) { + $defaultConfig['database'] = pathinfo($config['dbfile'], PATHINFO_FILENAME); + $config = array_merge($defaultConfig, $config); $config['name'] = $name; return new LaravelDuckdbConnection($config); diff --git a/src/Query/Grammar.php b/src/Query/Grammar.php index 5ae8037..556046c 100644 --- a/src/Query/Grammar.php +++ b/src/Query/Grammar.php @@ -30,4 +30,9 @@ private function wrapFromClause($value, $prefixAlias = false){ } return ($prefixAlias?$this->tablePrefix:'').$value; } + + public function compileTruncate(Builder $query) + { + return ['truncate '.$this->wrapTable($query->from) => []]; + } } diff --git a/src/Query/Processor.php b/src/Query/Processor.php index 774bd01..54019f0 100644 --- a/src/Query/Processor.php +++ b/src/Query/Processor.php @@ -2,9 +2,9 @@ namespace Harish\LaravelDuckdb\Query; -use Illuminate\Database\Query\Processors\Processor as BaseProcessor; +use Illuminate\Database\Query\Processors\PostgresProcessor; -class Processor extends BaseProcessor +class Processor extends PostgresProcessor { } diff --git a/src/Schema/Blueprint.php b/src/Schema/Blueprint.php new file mode 100644 index 0000000..59a58a3 --- /dev/null +++ b/src/Schema/Blueprint.php @@ -0,0 +1,8 @@ +blueprintResolver(function ($table, $callback, $prefix) { + return new Blueprint($table, $callback, $prefix); + }); + return parent::createBlueprint($table, $callback); + } } diff --git a/src/Schema/Grammar.php b/src/Schema/Grammar.php index 215cbbf..b0d9a13 100644 --- a/src/Schema/Grammar.php +++ b/src/Schema/Grammar.php @@ -2,9 +2,25 @@ namespace Harish\LaravelDuckdb\Schema; -use Illuminate\Database\Query\Grammars\PostgresGrammar; +use Illuminate\Database\Schema\Grammars\PostgresGrammar; +use Illuminate\Support\Fluent; class Grammar extends PostgresGrammar { + protected $transactions = false; + protected function typeInteger(Fluent $column) + { + return 'integer'; + } + + protected function typeBigInteger(Fluent $column) + { + return 'bigint'; + } + + protected function typeSmallInteger(Fluent $column) + { + return 'smallint'; + } } diff --git a/tests/Feature/DuckDBBasicTest.php b/tests/Feature/DuckDBBasicTest.php index f5220f2..b77860f 100644 --- a/tests/Feature/DuckDBBasicTest.php +++ b/tests/Feature/DuckDBBasicTest.php @@ -60,6 +60,5 @@ public function test_eloquent_model(){ public function test_query_exception(){ $this->expectException(QueryException::class); $rs = DB::connection('my_duckdb')->selectOne('select * from non_existing_tbl01 where foo=1 limit 1'); - dd($rs); } } diff --git a/tests/Feature/DuckDBSchemaStatementTest.php b/tests/Feature/DuckDBSchemaStatementTest.php new file mode 100644 index 0000000..52cd894 --- /dev/null +++ b/tests/Feature/DuckDBSchemaStatementTest.php @@ -0,0 +1,80 @@ + fake()->name(), + 'age' => fake()->numberBetween(13, 50), + 'rank' => fake()->numberBetween(1, 10), + 'salary' => fake()->randomFloat(null, 10000, 90000) + ]; + } +} +class Person extends \Harish\LaravelDuckdb\LaravelDuckdbModel{ + use \Illuminate\Database\Eloquent\Factories\HasFactory; + protected $connection = 'my_duckdb'; + protected $table = 'people'; + protected $guarded = ['id']; + + protected static function newFactory() + { + return PersonFactory::new(); + } +} +class DuckDBSchemaStatementTest extends TestCase +{ + public function test_migration(){ + + Schema::connection('my_duckdb')->dropIfExists('people'); + DB::connection('my_duckdb')->statement('DROP SEQUENCE IF EXISTS people_sequence'); + + DB::connection('my_duckdb')->statement('CREATE SEQUENCE people_sequence'); + Schema::connection('my_duckdb')->create('people', function (Blueprint $table) { + $table->id()->default(new \Illuminate\Database\Query\Expression("nextval('people_sequence')")); + $table->string('name'); + $table->integer('age'); + $table->integer('rank'); + $table->unsignedDecimal('salary')->nullable(); + $table->timestamps(); + }); + + $this->assertTrue(Schema::hasTable('people')); + } + + + public function test_model(){ + //truncate + Person::truncate(); + + //create + $singlePerson = Person::factory()->make()->toArray(); + $newPerson = Person::create($singlePerson); + + //batch insert + $manyPersons = Person::factory()->count(10)->make()->toArray(); + Person::insert($manyPersons); + + //update + $personToUpdate = Person::where('id', $newPerson->id)->first(); + $personToUpdate->name = 'Harish81'; + $personToUpdate->save(); + $this->assertSame(Person::where('name', 'Harish81')->count(), 1); + + //delete + Person::where('name', 'Harish81')->delete(); + + //assertion count + $this->assertCount( 10, Person::all()); + } +} \ No newline at end of file diff --git a/tests/TestCase.php b/tests/TestCase.php index 55db5be..451396b 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -34,5 +34,8 @@ protected function getEnvironmentSetUp($app) 'cli_timeout' => 0, 'dbfile' => '/tmp/duck_main.db', ]); + + //default database just for schema testing, no need for production + $app['config']->set('database.default', 'my_duckdb'); } }