From 5308cc21ae51939931db69d6550dd021c7d2014f Mon Sep 17 00:00:00 2001 From: Egor Shokurov Date: Sun, 2 Jan 2022 22:30:48 +0300 Subject: [PATCH] Switched to Microsoft SQLite implementation Debugging under Apple Silicon --- .gitlab-ci.yml | 3 +- ETLBox/ETLBox.csproj | 3 +- .../SQLiteConnectionString.cs | 10 +-- .../Native/SqliteConnectionManager.cs | 62 ++++++++++++------- ETLBox/src/Toolbox/DataFlow/DBDestination.cs | 3 +- .../default.config.json-template | 6 +- .../default.config.json-template | 2 +- .../src/CreateDatabaseTaskTests.cs | 6 +- .../src/CreateIndexTaskTests.cs | 2 +- .../default.config.json-template | 6 +- .../DBDestinationTransactionTests.cs | 4 +- .../default.config.json-template | 2 +- TestNonParallel/default.config.json-template | 2 +- TestNonParallel/docker.config.json-template | 2 +- .../default.config.json-template | 2 +- TestPerformance/default.config.json-template | 2 +- TestShared/src/Helper/TestPathHelper.cs | 25 ++++++++ .../default.config.json-template | 2 +- global.json | 4 +- test/docker/build-images.ps1 | 2 +- test/docker/mssql/Dockerfile | 4 +- test/docker/run-containers.ps1 | 10 ++- 22 files changed, 105 insertions(+), 59 deletions(-) create mode 100644 TestShared/src/Helper/TestPathHelper.cs diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 271bc6fa..87c91c01 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -77,10 +77,11 @@ test_job: - pushd test - pwsh ./Set-Configuration.ps1 gitlab-ci - popd + - dotnet build ETLBox.sln -c Release # - dotnet test ${DOT_NET_TEST_OPTIONS} ./TestConnectionManager # - dotnet test ${DOT_NET_TEST_OPTIONS} ./TestControlFlowTasks # - dotnet test ${DOT_NET_TEST_OPTIONS} ./TestDatabaseConnectors - - pushd ./TestFlatFileConnectors; dotnet test ${DOT_NET_TEST_OPTIONS} .; popd + - pushd ./TestFlatFileConnectors; dotnet test --no-build ${DOT_NET_TEST_OPTIONS} .; popd # - dotnet test ${DOT_NET_TEST_OPTIONS} ./TestHelper # - dotnet test ${DOT_NET_TEST_OPTIONS} ./TestNonParallel # - dotnet test ${DOT_NET_TEST_OPTIONS} ./TestOtherConnectors diff --git a/ETLBox/ETLBox.csproj b/ETLBox/ETLBox.csproj index 2e33306e..36f3ee39 100644 --- a/ETLBox/ETLBox.csproj +++ b/ETLBox/ETLBox.csproj @@ -33,6 +33,7 @@ + @@ -40,8 +41,6 @@ - - diff --git a/ETLBox/src/Definitions/ConnectionStrings/SQLiteConnectionString.cs b/ETLBox/src/Definitions/ConnectionStrings/SQLiteConnectionString.cs index d4e92bd0..999d5115 100644 --- a/ETLBox/src/Definitions/ConnectionStrings/SQLiteConnectionString.cs +++ b/ETLBox/src/Definitions/ConnectionStrings/SQLiteConnectionString.cs @@ -1,16 +1,16 @@ -using System.Data.SQLite; + +using Microsoft.Data.Sqlite; namespace ALE.ETLBox { /// - /// A helper class for encapsulating a conection string in an object. + /// A helper class for encapsulating a connection string in an object. /// Internally the SQLiteConnectionStringBuilder is used to access the values of the given connection string. /// public class SQLiteConnectionString : - DbConnectionString + DbConnectionString { - public SQLiteConnectionString() : - base() + public SQLiteConnectionString() { } public SQLiteConnectionString(string value) : base(value) diff --git a/ETLBox/src/Toolbox/ConnectionManager/Native/SqliteConnectionManager.cs b/ETLBox/src/Toolbox/ConnectionManager/Native/SqliteConnectionManager.cs index 50623c2c..85fbe076 100644 --- a/ETLBox/src/Toolbox/ConnectionManager/Native/SqliteConnectionManager.cs +++ b/ETLBox/src/Toolbox/ConnectionManager/Native/SqliteConnectionManager.cs @@ -1,8 +1,8 @@ using System; using System.Data; -using System.Data.SQLite; using System.Globalization; using System.Linq; +using Microsoft.Data.Sqlite; namespace ALE.ETLBox.ConnectionManager { @@ -16,7 +16,7 @@ namespace ALE.ETLBox.ConnectionManager /// "Data Source=.\db\SQLite.db;Version=3;")); /// /// - public class SQLiteConnectionManager : DbConnectionManager + public class SQLiteConnectionManager : DbConnectionManager { public override ConnectionManagerType ConnectionManagerType { get; } = ConnectionManagerType.SQLite; public override string QB { get; } = @""""; @@ -29,9 +29,17 @@ public class SQLiteConnectionManager : DbConnectionManager public bool ModifyDBSettings { get; set; } = false; - public SQLiteConnectionManager() : base() { } - public SQLiteConnectionManager(SQLiteConnectionString connectionString) : base(connectionString) { } - public SQLiteConnectionManager(string connectionString) : base(new SQLiteConnectionString(connectionString)) { } + public SQLiteConnectionManager() : base() + { + } + + public SQLiteConnectionManager(SQLiteConnectionString connectionString) : base(connectionString) + { + } + + public SQLiteConnectionManager(string connectionString) : base(new SQLiteConnectionString(connectionString)) + { + } string Synchronous { get; set; } string JournalMode { get; set; } @@ -39,11 +47,11 @@ public SQLiteConnectionManager(string connectionString) : base(new SQLiteConnect public override void BulkInsert(ITableData data, string tableName) { var sourceColumnNames = data.ColumnMapping.Cast().Select(cm => cm.SourceColumn).ToList(); - var sourceColumnValues = data.ColumnMapping.Cast().Select(cm => "?").ToList(); + var paramNames = data.ColumnMapping.Cast().Select((_, i) => $"$p{i}").ToArray(); var destColumnNames = data.ColumnMapping.Cast().Select(cm => cm.DataSetColumn).ToList(); - SQLiteTransaction existingTransaction = Transaction as SQLiteTransaction; - SQLiteTransaction bulkTransaction = null; + var existingTransaction = Transaction as SqliteTransaction; + SqliteTransaction bulkTransaction = null; if (existingTransaction == null) bulkTransaction = this.DbConnection.BeginTransaction(); using (bulkTransaction) @@ -51,25 +59,24 @@ public override void BulkInsert(ITableData data, string tableName) { command.Transaction = existingTransaction ?? bulkTransaction; command.CommandText = - $@"INSERT INTO {tableName} -({String.Join(",", sourceColumnNames)}) -VALUES ({String.Join(",", sourceColumnValues)}) - "; + $@"INSERT INTO {tableName} +({string.Join(",", sourceColumnNames)}) +VALUES ({string.Join(",", paramNames)})"; command.Prepare(); while (data.Read()) { - foreach (var mapping in destColumnNames) - { - SQLiteParameter par = new SQLiteParameter(); - par.Value = data.GetValue(data.GetOrdinal(mapping)); - command.Parameters.Add(par); - } + command.Parameters.AddRange( + destColumnNames.Select((n, i) => new SqliteParameter + { + ParameterName = paramNames[i], + Value = data.GetValue(data.GetOrdinal(n)) + })); command.ExecuteNonQuery(); command.Parameters.Clear(); } + bulkTransaction?.Commit(); } - } public override void PrepareBulkInsert(string tablename) @@ -89,6 +96,7 @@ public override void PrepareBulkInsert(string tablename) } } } + public override void CleanUpBulkInsert(string tablename) { if (ModifyDBSettings) @@ -98,17 +106,23 @@ public override void CleanUpBulkInsert(string tablename) this.ExecuteNonQuery($"PRAGMA synchronous = {Synchronous}"); this.ExecuteNonQuery($"PRAGMA journal_mode = {JournalMode}"); } - catch { } + catch + { + } } } - public override void BeforeBulkInsert(string tableName) { } + public override void BeforeBulkInsert(string tableName) + { + } - public override void AfterBulkInsert(string tableName) { } + public override void AfterBulkInsert(string tableName) + { + } public override IConnectionManager Clone() { - SQLiteConnectionManager clone = new SQLiteConnectionManager((SQLiteConnectionString)ConnectionString) + var clone = new SQLiteConnectionManager((SQLiteConnectionString)ConnectionString) { MaxLoginAttempts = this.MaxLoginAttempts, ModifyDBSettings = this.ModifyDBSettings @@ -116,4 +130,4 @@ public override IConnectionManager Clone() return clone; } } -} +} \ No newline at end of file diff --git a/ETLBox/src/Toolbox/DataFlow/DBDestination.cs b/ETLBox/src/Toolbox/DataFlow/DBDestination.cs index 76f7824f..ab9f48a0 100644 --- a/ETLBox/src/Toolbox/DataFlow/DBDestination.cs +++ b/ETLBox/src/Toolbox/DataFlow/DBDestination.cs @@ -104,8 +104,7 @@ protected override void TryBulkInsertData(TInput[] data) DisableLogging = true, ConnectionManager = BulkInsertConnectionManager }; - sql - .BulkInsert(TableData, DestinationTableDefinition.Name); + sql.BulkInsert(TableData, DestinationTableDefinition.Name); } catch (Exception e) { diff --git a/TestConnectionManager/default.config.json-template b/TestConnectionManager/default.config.json-template index b9591b96..2f02cdf5 100644 --- a/TestConnectionManager/default.config.json-template +++ b/TestConnectionManager/default.config.json-template @@ -2,7 +2,7 @@ "DataFlow": { "SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlow", "SqlOdbcConnectionString": "Driver={SQL Server};Server=${CFG_MSSQL_IP};Database=ETLBox_DataFlow;Uid=sa;Pwd=YourStrong@Passw0rd;", - "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlow.db;Version=3;", + "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlow.db;", "AccessOdbcConnectionString": "Driver={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=.${CFG_PD}db${CFG_PD}AccessDataFlow.accdb", "MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_DataFlow;Uid=root;Pwd=etlboxpassword;", "PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_DataFlow;User Id=postgres;Password=etlboxpassword;", @@ -10,13 +10,13 @@ }, "DataFlowSource": { "SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlowSource", - "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlowSource.db;Version=3;", + "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlowSource.db;", "MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_DataFlowSource;Uid=root;Pwd=etlboxpassword;", "PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_DataFlowSource;User Id=postgres;Password=etlboxpassword;" }, "DataFlowDestination": { "SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlowDestination", - "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlowDestination.db;Version=3;", + "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlowDestination.db;", "MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_DataFlowDestination;Uid=root;Pwd=etlboxpassword;", "PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_DataFlowDestination;User Id=postgres;Password=etlboxpassword;" }, diff --git a/TestControlFlowTasks/default.config.json-template b/TestControlFlowTasks/default.config.json-template index a54a4c1c..af5693d1 100644 --- a/TestControlFlowTasks/default.config.json-template +++ b/TestControlFlowTasks/default.config.json-template @@ -2,7 +2,7 @@ "ControlFlow": { "SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_ControlFlow", "SSASConnectionString": "Data Source=.;Integrated Security=SSPI;Initial Catalog=ETLBox_ControlFlow", - "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteControlFlow.db;Version=3;", + "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteControlFlow.db;", "AccessOdbcConnectionString": "Driver={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=.${CFG_PD}db${CFG_PD}AccessControlFlow.accdb", "SqlOdbcConnectionString": "Driver={SQL Server};Server=${CFG_MSSQL_IP};Database=ETLBox_ControlFlow;Uid=sa;Pwd=YourStrong@Passw0rd;", "MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_ControlFlow;Uid=root;Pwd=etlboxpassword;", diff --git a/TestControlFlowTasks/src/CreateDatabaseTaskTests.cs b/TestControlFlowTasks/src/CreateDatabaseTaskTests.cs index dfdef992..25e239b6 100644 --- a/TestControlFlowTasks/src/CreateDatabaseTaskTests.cs +++ b/TestControlFlowTasks/src/CreateDatabaseTaskTests.cs @@ -25,7 +25,7 @@ public CreateDatabaseTaskTests() public void CreateSimple(IConnectionManager connection) { //Arrange - string dbName = "ETLBox_"+HashHelper.RandomString(10); + var dbName = "ETLBox_"+HashHelper.RandomString(10); var dbListBefore = GetDatabaseListTask.List(connection); Assert.DoesNotContain(dbName, dbListBefore); @@ -44,8 +44,8 @@ public void CreateSimple(IConnectionManager connection) public void CreateWithCollation(IConnectionManager connection) { //Arrange - string dbName = "ETLBox_" + HashHelper.RandomString(10); - string collation = "Latin1_General_CS_AS"; + var dbName = "ETLBox_" + HashHelper.RandomString(10); + var collation = "Latin1_General_CS_AS"; if (connection.GetType() == typeof(PostgresConnectionManager)) collation = "en_US.utf8"; if (connection.GetType() == typeof(MySqlConnectionManager)) diff --git a/TestControlFlowTasks/src/CreateIndexTaskTests.cs b/TestControlFlowTasks/src/CreateIndexTaskTests.cs index cd5c11e7..2cc8808f 100644 --- a/TestControlFlowTasks/src/CreateIndexTaskTests.cs +++ b/TestControlFlowTasks/src/CreateIndexTaskTests.cs @@ -13,7 +13,7 @@ namespace ALE.ETLBoxTests.ControlFlowTests [Collection("ControlFlow")] public class CreateIndexTaskTests { - public SqlConnectionManager SqlConnection => Config.SqlConnection.ConnectionManager("ControlFlow"); + private SqlConnectionManager SqlConnection => Config.SqlConnection.ConnectionManager("ControlFlow"); public static IEnumerable Connections => Config.AllSqlConnections("ControlFlow"); public CreateIndexTaskTests(ControlFlowDatabaseFixture dbFixture) diff --git a/TestDatabaseConnectors/default.config.json-template b/TestDatabaseConnectors/default.config.json-template index 8676ec1a..16fa446e 100644 --- a/TestDatabaseConnectors/default.config.json-template +++ b/TestDatabaseConnectors/default.config.json-template @@ -2,7 +2,7 @@ "DataFlow": { "SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlow", "SqlOdbcConnectionString": "Driver={SQL Server};Server=${CFG_MSSQL_IP};Database=ETLBox_DataFlow;Uid=sa;Pwd=YourStrong@Passw0rd;", - "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlow.db;Version=3;", + "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlow.db;", "AccessOdbcConnectionString": "Driver={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=.${CFG_PD}db${CFG_PD}AccessDataFlow.accdb", "MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_DataFlow;Uid=root;Pwd=etlboxpassword;", "PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_DataFlow;User Id=postgres;Password=etlboxpassword;", @@ -10,13 +10,13 @@ }, "DataFlowSource": { "SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlowSource", - "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlowSource.db;Version=3;", + "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlowSource.db;", "MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_DataFlowSource;Uid=root;Pwd=etlboxpassword;", "PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_DataFlowSource;User Id=postgres;Password=etlboxpassword;" }, "DataFlowDestination": { "SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlowDestination", - "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlowDestination.db;Version=3;", + "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlowDestination.db;", "MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_DataFlowDestination;Uid=root;Pwd=etlboxpassword;", "PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_DataFlowDestination;User Id=postgres;Password=etlboxpassword;" } diff --git a/TestDatabaseConnectors/src/DBDestination/DBDestinationTransactionTests.cs b/TestDatabaseConnectors/src/DBDestination/DBDestinationTransactionTests.cs index edff905a..e97692af 100644 --- a/TestDatabaseConnectors/src/DBDestination/DBDestinationTransactionTests.cs +++ b/TestDatabaseConnectors/src/DBDestination/DBDestinationTransactionTests.cs @@ -15,10 +15,10 @@ namespace ALE.ETLBoxTests.DataFlowTests { [Collection("DataFlow")] - public class DbDestinationTransactionTests + public class DbDestinationTransactionTests : IClassFixture { public static IEnumerable Connections => Config.AllSqlConnections("DataFlow"); - public DbDestinationTransactionTests(DataFlowDatabaseFixture dbFixture) + public DbDestinationTransactionTests(DataFlowDatabaseFixture _) { } diff --git a/TestFlatFileConnectors/default.config.json-template b/TestFlatFileConnectors/default.config.json-template index d31d5559..8bb1b677 100644 --- a/TestFlatFileConnectors/default.config.json-template +++ b/TestFlatFileConnectors/default.config.json-template @@ -2,7 +2,7 @@ "DataFlow": { "SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlow", "SqlOdbcConnectionString": "Driver={SQL Server};Server=${CFG_MSSQL_IP};Database=ETLBox_DataFlow;Uid=sa;Pwd=YourStrong@Passw0rd;", - "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlow.db;Version=3;", + "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlow.db;", "AccessOdbcConnectionString": "Driver={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=.${CFG_PD}db${CFG_PD}AccessDataFlow.accdb", "MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_DataFlow;Uid=root;Pwd=etlboxpassword;", "PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_DataFlow;User Id=postgres;Password=etlboxpassword;", diff --git a/TestNonParallel/default.config.json-template b/TestNonParallel/default.config.json-template index 08f24150..f67b6268 100644 --- a/TestNonParallel/default.config.json-template +++ b/TestNonParallel/default.config.json-template @@ -1,7 +1,7 @@ { "Logging": { "SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_Logging", - "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteLogging.db;Version=3;", + "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteLogging.db;", "MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_Logging;Uid=root;Pwd=etlboxpassword;", "PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_Logging;User Id=postgres;Password=etlboxpassword;" }, diff --git a/TestNonParallel/docker.config.json-template b/TestNonParallel/docker.config.json-template index 08f24150..f67b6268 100644 --- a/TestNonParallel/docker.config.json-template +++ b/TestNonParallel/docker.config.json-template @@ -1,7 +1,7 @@ { "Logging": { "SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_Logging", - "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteLogging.db;Version=3;", + "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteLogging.db;", "MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_Logging;Uid=root;Pwd=etlboxpassword;", "PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_Logging;User Id=postgres;Password=etlboxpassword;" }, diff --git a/TestOtherConnectors/default.config.json-template b/TestOtherConnectors/default.config.json-template index d31d5559..8bb1b677 100755 --- a/TestOtherConnectors/default.config.json-template +++ b/TestOtherConnectors/default.config.json-template @@ -2,7 +2,7 @@ "DataFlow": { "SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlow", "SqlOdbcConnectionString": "Driver={SQL Server};Server=${CFG_MSSQL_IP};Database=ETLBox_DataFlow;Uid=sa;Pwd=YourStrong@Passw0rd;", - "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlow.db;Version=3;", + "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlow.db;", "AccessOdbcConnectionString": "Driver={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=.${CFG_PD}db${CFG_PD}AccessDataFlow.accdb", "MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_DataFlow;Uid=root;Pwd=etlboxpassword;", "PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_DataFlow;User Id=postgres;Password=etlboxpassword;", diff --git a/TestPerformance/default.config.json-template b/TestPerformance/default.config.json-template index 56c3cccf..81148a0b 100644 --- a/TestPerformance/default.config.json-template +++ b/TestPerformance/default.config.json-template @@ -5,6 +5,6 @@ "AccessOdbcConnectionString": "Driver={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=.${CFG_PD}db${CFG_PD}AccessPerformance.mdb", "MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_Performance;Uid=root;Pwd=etlboxpassword;", "PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_Performance;User Id=postgres;Password=etlboxpassword;", - "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLitePerformance.db;Version=3;" + "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLitePerformance.db;" } } diff --git a/TestShared/src/Helper/TestPathHelper.cs b/TestShared/src/Helper/TestPathHelper.cs new file mode 100644 index 00000000..ba4f2ace --- /dev/null +++ b/TestShared/src/Helper/TestPathHelper.cs @@ -0,0 +1,25 @@ +using System.IO; +using System.Reflection; + +namespace ALE.ETLBox.Helper; + +public static class TestPathHelper +{ + public static string GetFullPathToFile(string pathRelativeUnitTestingFile) + { + var folderProjectLevel = GetPathToCurrentUnitTestProject(); + return Path.Combine(folderProjectLevel, pathRelativeUnitTestingFile); + } + + /// + /// Get the path to the current unit testing project. + /// + /// + private static string GetPathToCurrentUnitTestProject() + { + var pathAssembly = Assembly.GetExecutingAssembly().Location; + var folderAssembly = Path.GetDirectoryName(pathAssembly) ?? Directory.GetCurrentDirectory(); + var folderProjectLevel = Directory.GetParent(folderAssembly)?.Parent?.Parent?.FullName; + return folderProjectLevel; + } +} \ No newline at end of file diff --git a/TestTransformations/default.config.json-template b/TestTransformations/default.config.json-template index d31d5559..8bb1b677 100644 --- a/TestTransformations/default.config.json-template +++ b/TestTransformations/default.config.json-template @@ -2,7 +2,7 @@ "DataFlow": { "SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlow", "SqlOdbcConnectionString": "Driver={SQL Server};Server=${CFG_MSSQL_IP};Database=ETLBox_DataFlow;Uid=sa;Pwd=YourStrong@Passw0rd;", - "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlow.db;Version=3;", + "SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlow.db;", "AccessOdbcConnectionString": "Driver={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=.${CFG_PD}db${CFG_PD}AccessDataFlow.accdb", "MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_DataFlow;Uid=root;Pwd=etlboxpassword;", "PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_DataFlow;User Id=postgres;Password=etlboxpassword;", diff --git a/global.json b/global.json index 97dd8739..e80e4280 100644 --- a/global.json +++ b/global.json @@ -1,7 +1,7 @@ { "sdk": { "version": "6.0", - "rollForward": "latestMajor", + "rollForward": "latestMinor", "allowPrerelease": false } -} \ No newline at end of file +} diff --git a/test/docker/build-images.ps1 b/test/docker/build-images.ps1 index af5f5d6d..15eb077b 100755 --- a/test/docker/build-images.ps1 +++ b/test/docker/build-images.ps1 @@ -1,6 +1,6 @@ #!/usr/bin/env pwsh -$docker='nerdctl' +$docker='docker' & $docker login #$env:DOCKER_BUILDKIT=1 diff --git a/test/docker/mssql/Dockerfile b/test/docker/mssql/Dockerfile index 1d0c739b..d4f3f256 100644 --- a/test/docker/mssql/Dockerfile +++ b/test/docker/mssql/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/mssql/server +FROM mcr.microsoft.com/azure-sql-edge USER root @@ -6,4 +6,4 @@ RUN /opt/mssql/bin/mssql-conf traceflag 3979 on && \ /opt/mssql/bin/mssql-conf set control.alternatewritethrough 0 && \ /opt/mssql/bin/mssql-conf set control.writethrough 0 -USER mssql \ No newline at end of file +USER mssql diff --git a/test/docker/run-containers.ps1 b/test/docker/run-containers.ps1 index bfd6610a..5e5b2288 100755 --- a/test/docker/run-containers.ps1 +++ b/test/docker/run-containers.ps1 @@ -1,3 +1,11 @@ #!/usr/bin/env pwsh -nerdctl run -e "ACCEPT_EULA=Y" -e "SA_PASSWORD=YourStrong@Passw0rd" -e "MSSQL_PID=Developer" -p 1433:1433 --name localmssql -d etlbox-mssql +& docker run --rm -d --cap-add SYS_PTRACE -e "ACCEPT_EULA=Y" -e "SA_PASSWORD=YourStrong@Passw0rd" -e "MSSQL_PID=Developer" -p 1433:1433 --name localmssql -d mcr.microsoft.com/azure-sql-edge +if ( $LastExitCode -ne 0 ) { exit $LastExitCode } + + +& docker run --rm -d -e "MYSQL_ROOT_HOST=%" -e "MYSQL_ROOT_PASSWORD=etlboxpassword" -p 3306:3306 --name localmysql -d mysql/mysql-server +if ( $LastExitCode -ne 0 ) { exit $LastExitCode } + +& docker run --rm -d -e "POSTGRES_PASSWORD=etlboxpassword" -p 5432:5432 --name localpostgres -d postgres +if ( $LastExitCode -ne 0 ) { exit $LastExitCode }