Skip to content

Commit

Permalink
Switched to Microsoft SQLite implementation
Browse files Browse the repository at this point in the history
Debugging under Apple Silicon
  • Loading branch information
rpsft committed Jan 2, 2022
1 parent 72805b4 commit 5308cc2
Show file tree
Hide file tree
Showing 22 changed files with 105 additions and 59 deletions.
3 changes: 2 additions & 1 deletion .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -77,10 +77,11 @@ test_job:
- pushd test
- pwsh ./Set-Configuration.ps1 gitlab-ci
- popd
- dotnet build ETLBox.sln -c Release
# - dotnet test ${DOT_NET_TEST_OPTIONS} ./TestConnectionManager
# - dotnet test ${DOT_NET_TEST_OPTIONS} ./TestControlFlowTasks
# - dotnet test ${DOT_NET_TEST_OPTIONS} ./TestDatabaseConnectors
- pushd ./TestFlatFileConnectors; dotnet test ${DOT_NET_TEST_OPTIONS} .; popd
- pushd ./TestFlatFileConnectors; dotnet test --no-build ${DOT_NET_TEST_OPTIONS} .; popd
# - dotnet test ${DOT_NET_TEST_OPTIONS} ./TestHelper
# - dotnet test ${DOT_NET_TEST_OPTIONS} ./TestNonParallel
# - dotnet test ${DOT_NET_TEST_OPTIONS} ./TestOtherConnectors
Expand Down
3 changes: 1 addition & 2 deletions ETLBox/ETLBox.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,14 @@
<PackageReference Include="CsvHelper" Version="27.1.1" />
<PackageReference Include="ExcelDataReader" Version="3.6.0" />
<PackageReference Include="Microsoft.Data.SqlClient" Version="1.1.2" />
<PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.1" />
<PackageReference Include="MySql.Data" Version="8.0.19" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="NLog" Version="4.7.0" />
<PackageReference Include="NLog.Extensions.Logging" Version="1.6.2" />
<PackageReference Include="Npgsql" Version="4.1.3.1" />
<PackageReference Include="System.Data.Common" Version="4.3.0" />
<PackageReference Include="System.Data.Odbc" Version="4.7.0" />
<PackageReference Include="System.Data.SqlClient" Version="4.8.1" />
<PackageReference Include="System.Data.SQLite.Core" Version="1.0.112.1" />
<PackageReference Include="System.Threading.Tasks.Dataflow" Version="4.11.0" />
<PackageReference Include="TSQL.Parser" Version="1.5.0" />
<PackageReference Include="Unofficial.Microsoft.AnalysisServices.AdomdClientNetCore" Version="15.3.1.2" />
Expand Down
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
using System.Data.SQLite;

using Microsoft.Data.Sqlite;

namespace ALE.ETLBox
{
/// <summary>
/// A helper class for encapsulating a conection string in an object.
/// A helper class for encapsulating a connection string in an object.
/// Internally the SQLiteConnectionStringBuilder is used to access the values of the given connection string.
/// </summary>
public class SQLiteConnectionString :
DbConnectionString<SQLiteConnectionString, SQLiteConnectionStringBuilder>
DbConnectionString<SQLiteConnectionString, SqliteConnectionStringBuilder>
{
public SQLiteConnectionString() :
base()
public SQLiteConnectionString()
{ }
public SQLiteConnectionString(string value) :
base(value)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
using System;
using System.Data;
using System.Data.SQLite;
using System.Globalization;
using System.Linq;
using Microsoft.Data.Sqlite;

namespace ALE.ETLBox.ConnectionManager
{
Expand All @@ -16,7 +16,7 @@ namespace ALE.ETLBox.ConnectionManager
/// "Data Source=.\db\SQLite.db;Version=3;"));
/// </code>
/// </example>
public class SQLiteConnectionManager : DbConnectionManager<SQLiteConnection>
public class SQLiteConnectionManager : DbConnectionManager<SqliteConnection>
{
public override ConnectionManagerType ConnectionManagerType { get; } = ConnectionManagerType.SQLite;
public override string QB { get; } = @"""";
Expand All @@ -29,47 +29,54 @@ public class SQLiteConnectionManager : DbConnectionManager<SQLiteConnection>

public bool ModifyDBSettings { get; set; } = false;

public SQLiteConnectionManager() : base() { }
public SQLiteConnectionManager(SQLiteConnectionString connectionString) : base(connectionString) { }
public SQLiteConnectionManager(string connectionString) : base(new SQLiteConnectionString(connectionString)) { }
public SQLiteConnectionManager() : base()
{
}

public SQLiteConnectionManager(SQLiteConnectionString connectionString) : base(connectionString)
{
}

public SQLiteConnectionManager(string connectionString) : base(new SQLiteConnectionString(connectionString))
{
}

string Synchronous { get; set; }
string JournalMode { get; set; }

public override void BulkInsert(ITableData data, string tableName)
{
var sourceColumnNames = data.ColumnMapping.Cast<IColumnMapping>().Select(cm => cm.SourceColumn).ToList();
var sourceColumnValues = data.ColumnMapping.Cast<IColumnMapping>().Select(cm => "?").ToList();
var paramNames = data.ColumnMapping.Cast<IColumnMapping>().Select((_, i) => $"$p{i}").ToArray();
var destColumnNames = data.ColumnMapping.Cast<IColumnMapping>().Select(cm => cm.DataSetColumn).ToList();

SQLiteTransaction existingTransaction = Transaction as SQLiteTransaction;
SQLiteTransaction bulkTransaction = null;
var existingTransaction = Transaction as SqliteTransaction;
SqliteTransaction bulkTransaction = null;
if (existingTransaction == null)
bulkTransaction = this.DbConnection.BeginTransaction();
using (bulkTransaction)
using (var command = this.DbConnection.CreateCommand())
{
command.Transaction = existingTransaction ?? bulkTransaction;
command.CommandText =
$@"INSERT INTO {tableName}
({String.Join(",", sourceColumnNames)})
VALUES ({String.Join(",", sourceColumnValues)})
";
$@"INSERT INTO {tableName}
({string.Join(",", sourceColumnNames)})
VALUES ({string.Join(",", paramNames)})";
command.Prepare();
while (data.Read())
{
foreach (var mapping in destColumnNames)
{
SQLiteParameter par = new SQLiteParameter();
par.Value = data.GetValue(data.GetOrdinal(mapping));
command.Parameters.Add(par);
}
command.Parameters.AddRange(
destColumnNames.Select((n, i) => new SqliteParameter
{
ParameterName = paramNames[i],
Value = data.GetValue(data.GetOrdinal(n))
}));
command.ExecuteNonQuery();
command.Parameters.Clear();
}

bulkTransaction?.Commit();
}

}

public override void PrepareBulkInsert(string tablename)
Expand All @@ -89,6 +96,7 @@ public override void PrepareBulkInsert(string tablename)
}
}
}

public override void CleanUpBulkInsert(string tablename)
{
if (ModifyDBSettings)
Expand All @@ -98,22 +106,28 @@ public override void CleanUpBulkInsert(string tablename)
this.ExecuteNonQuery($"PRAGMA synchronous = {Synchronous}");
this.ExecuteNonQuery($"PRAGMA journal_mode = {JournalMode}");
}
catch { }
catch
{
}
}
}

public override void BeforeBulkInsert(string tableName) { }
public override void BeforeBulkInsert(string tableName)
{
}

public override void AfterBulkInsert(string tableName) { }
public override void AfterBulkInsert(string tableName)
{
}

public override IConnectionManager Clone()
{
SQLiteConnectionManager clone = new SQLiteConnectionManager((SQLiteConnectionString)ConnectionString)
var clone = new SQLiteConnectionManager((SQLiteConnectionString)ConnectionString)
{
MaxLoginAttempts = this.MaxLoginAttempts,
ModifyDBSettings = this.ModifyDBSettings
};
return clone;
}
}
}
}
3 changes: 1 addition & 2 deletions ETLBox/src/Toolbox/DataFlow/DBDestination.cs
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,7 @@ protected override void TryBulkInsertData(TInput[] data)
DisableLogging = true,
ConnectionManager = BulkInsertConnectionManager
};
sql
.BulkInsert(TableData, DestinationTableDefinition.Name);
sql.BulkInsert(TableData, DestinationTableDefinition.Name);
}
catch (Exception e)
{
Expand Down
6 changes: 3 additions & 3 deletions TestConnectionManager/default.config.json-template
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,21 @@
"DataFlow": {
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlow",
"SqlOdbcConnectionString": "Driver={SQL Server};Server=${CFG_MSSQL_IP};Database=ETLBox_DataFlow;Uid=sa;Pwd=YourStrong@Passw0rd;",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlow.db;Version=3;",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlow.db;",
"AccessOdbcConnectionString": "Driver={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=.${CFG_PD}db${CFG_PD}AccessDataFlow.accdb",
"MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_DataFlow;Uid=root;Pwd=etlboxpassword;",
"PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_DataFlow;User Id=postgres;Password=etlboxpassword;",
"AzureSqlConnectionString": "Server=tcp:etlbox.database.windows.net,1433;Initial Catalog=etlbox;Persist Security Info=False;User ID=etlboxadmin;Password=ETLBoxPassw0rd!;"
},
"DataFlowSource": {
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlowSource",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlowSource.db;Version=3;",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlowSource.db;",
"MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_DataFlowSource;Uid=root;Pwd=etlboxpassword;",
"PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_DataFlowSource;User Id=postgres;Password=etlboxpassword;"
},
"DataFlowDestination": {
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlowDestination",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlowDestination.db;Version=3;",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlowDestination.db;",
"MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_DataFlowDestination;Uid=root;Pwd=etlboxpassword;",
"PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_DataFlowDestination;User Id=postgres;Password=etlboxpassword;"
},
Expand Down
2 changes: 1 addition & 1 deletion TestControlFlowTasks/default.config.json-template
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"ControlFlow": {
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_ControlFlow",
"SSASConnectionString": "Data Source=.;Integrated Security=SSPI;Initial Catalog=ETLBox_ControlFlow",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteControlFlow.db;Version=3;",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteControlFlow.db;",
"AccessOdbcConnectionString": "Driver={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=.${CFG_PD}db${CFG_PD}AccessControlFlow.accdb",
"SqlOdbcConnectionString": "Driver={SQL Server};Server=${CFG_MSSQL_IP};Database=ETLBox_ControlFlow;Uid=sa;Pwd=YourStrong@Passw0rd;",
"MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_ControlFlow;Uid=root;Pwd=etlboxpassword;",
Expand Down
6 changes: 3 additions & 3 deletions TestControlFlowTasks/src/CreateDatabaseTaskTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ public CreateDatabaseTaskTests()
public void CreateSimple(IConnectionManager connection)
{
//Arrange
string dbName = "ETLBox_"+HashHelper.RandomString(10);
var dbName = "ETLBox_"+HashHelper.RandomString(10);
var dbListBefore = GetDatabaseListTask.List(connection);
Assert.DoesNotContain<string>(dbName, dbListBefore);

Expand All @@ -44,8 +44,8 @@ public void CreateSimple(IConnectionManager connection)
public void CreateWithCollation(IConnectionManager connection)
{
//Arrange
string dbName = "ETLBox_" + HashHelper.RandomString(10);
string collation = "Latin1_General_CS_AS";
var dbName = "ETLBox_" + HashHelper.RandomString(10);
var collation = "Latin1_General_CS_AS";
if (connection.GetType() == typeof(PostgresConnectionManager))
collation = "en_US.utf8";
if (connection.GetType() == typeof(MySqlConnectionManager))
Expand Down
2 changes: 1 addition & 1 deletion TestControlFlowTasks/src/CreateIndexTaskTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ namespace ALE.ETLBoxTests.ControlFlowTests
[Collection("ControlFlow")]
public class CreateIndexTaskTests
{
public SqlConnectionManager SqlConnection => Config.SqlConnection.ConnectionManager("ControlFlow");
private SqlConnectionManager SqlConnection => Config.SqlConnection.ConnectionManager("ControlFlow");
public static IEnumerable<object[]> Connections => Config.AllSqlConnections("ControlFlow");

public CreateIndexTaskTests(ControlFlowDatabaseFixture dbFixture)
Expand Down
6 changes: 3 additions & 3 deletions TestDatabaseConnectors/default.config.json-template
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,21 @@
"DataFlow": {
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlow",
"SqlOdbcConnectionString": "Driver={SQL Server};Server=${CFG_MSSQL_IP};Database=ETLBox_DataFlow;Uid=sa;Pwd=YourStrong@Passw0rd;",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlow.db;Version=3;",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlow.db;",
"AccessOdbcConnectionString": "Driver={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=.${CFG_PD}db${CFG_PD}AccessDataFlow.accdb",
"MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_DataFlow;Uid=root;Pwd=etlboxpassword;",
"PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_DataFlow;User Id=postgres;Password=etlboxpassword;",
"AzureSqlConnectionString": "Server=tcp:etlbox.database.windows.net,1433;Initial Catalog=etlbox;Persist Security Info=False;User ID=etlboxadmin;Password=ETLBoxPassw0rd!;"
},
"DataFlowSource": {
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlowSource",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlowSource.db;Version=3;",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlowSource.db;",
"MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_DataFlowSource;Uid=root;Pwd=etlboxpassword;",
"PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_DataFlowSource;User Id=postgres;Password=etlboxpassword;"
},
"DataFlowDestination": {
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlowDestination",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlowDestination.db;Version=3;",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlowDestination.db;",
"MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_DataFlowDestination;Uid=root;Pwd=etlboxpassword;",
"PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_DataFlowDestination;User Id=postgres;Password=etlboxpassword;"
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@
namespace ALE.ETLBoxTests.DataFlowTests
{
[Collection("DataFlow")]
public class DbDestinationTransactionTests
public class DbDestinationTransactionTests : IClassFixture<DataFlowDatabaseFixture>
{
public static IEnumerable<object[]> Connections => Config.AllSqlConnections("DataFlow");
public DbDestinationTransactionTests(DataFlowDatabaseFixture dbFixture)
public DbDestinationTransactionTests(DataFlowDatabaseFixture _)
{
}

Expand Down
2 changes: 1 addition & 1 deletion TestFlatFileConnectors/default.config.json-template
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"DataFlow": {
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlow",
"SqlOdbcConnectionString": "Driver={SQL Server};Server=${CFG_MSSQL_IP};Database=ETLBox_DataFlow;Uid=sa;Pwd=YourStrong@Passw0rd;",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlow.db;Version=3;",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlow.db;",
"AccessOdbcConnectionString": "Driver={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=.${CFG_PD}db${CFG_PD}AccessDataFlow.accdb",
"MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_DataFlow;Uid=root;Pwd=etlboxpassword;",
"PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_DataFlow;User Id=postgres;Password=etlboxpassword;",
Expand Down
2 changes: 1 addition & 1 deletion TestNonParallel/default.config.json-template
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"Logging": {
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_Logging",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteLogging.db;Version=3;",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteLogging.db;",
"MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_Logging;Uid=root;Pwd=etlboxpassword;",
"PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_Logging;User Id=postgres;Password=etlboxpassword;"
},
Expand Down
2 changes: 1 addition & 1 deletion TestNonParallel/docker.config.json-template
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"Logging": {
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_Logging",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteLogging.db;Version=3;",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteLogging.db;",
"MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_Logging;Uid=root;Pwd=etlboxpassword;",
"PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_Logging;User Id=postgres;Password=etlboxpassword;"
},
Expand Down
2 changes: 1 addition & 1 deletion TestOtherConnectors/default.config.json-template
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"DataFlow": {
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlow",
"SqlOdbcConnectionString": "Driver={SQL Server};Server=${CFG_MSSQL_IP};Database=ETLBox_DataFlow;Uid=sa;Pwd=YourStrong@Passw0rd;",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlow.db;Version=3;",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlow.db;",
"AccessOdbcConnectionString": "Driver={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=.${CFG_PD}db${CFG_PD}AccessDataFlow.accdb",
"MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_DataFlow;Uid=root;Pwd=etlboxpassword;",
"PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_DataFlow;User Id=postgres;Password=etlboxpassword;",
Expand Down
2 changes: 1 addition & 1 deletion TestPerformance/default.config.json-template
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,6 @@
"AccessOdbcConnectionString": "Driver={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=.${CFG_PD}db${CFG_PD}AccessPerformance.mdb",
"MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_Performance;Uid=root;Pwd=etlboxpassword;",
"PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_Performance;User Id=postgres;Password=etlboxpassword;",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLitePerformance.db;Version=3;"
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLitePerformance.db;"
}
}
Loading

0 comments on commit 5308cc2

Please sign in to comment.