Skip to content

Commit

Permalink
feat(csharp/src/Drivers): introduce Interop.FlightSql driver (#2214)
Browse files Browse the repository at this point in the history
- Introduces the Interop.FlightSql driver to use the FlightSQL Go driver
from C#
- Adds support for testing against multiple FlightSQL environments
simultaneously in the configuration (something we may want to carry
forward to other projects)
- Has a first set of test sample data specific to Dremio, DuckDB and
SQLite using https://github.com/voltrondata/SQLFlite as the server(s).
- Adds support for the IntervalType in a generic capacity

---------

Co-authored-by: David Coe <coedavid@umich.edu>
Co-authored-by: Leslie Fung <v-lesliefung@microsoft.com>
Co-authored-by: Matt Topol <zotthewizard@gmail.com>
  • Loading branch information
4 people authored Dec 4, 2024
1 parent 73bbc8a commit 08ac053
Show file tree
Hide file tree
Showing 42 changed files with 2,449 additions and 378 deletions.
42 changes: 28 additions & 14 deletions csharp/Apache.Arrow.Adbc.sln
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{5BD04C26
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Drivers", "Drivers", "{FEB257A0-4FD3-495E-9A47-9E1649755445}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Adbc.Drivers.FlightSql", "src\Drivers\FlightSql\Apache.Arrow.Adbc.Drivers.FlightSql.csproj", "{19AA450A-2F87-49BD-9122-8AD07D4C6DCE}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Drivers", "Drivers", "{C7290227-E925-47E7-8B6B-A8B171645D58}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Adbc.Tests.Drivers.FlightSql", "test\Drivers\FlightSql\Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj", "{5C15E79C-19C4-4FF4-BB82-28754FE3966B}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Client", "Client", "{B6111602-2DC4-4B2F-9598-E3EE1972D3E4}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Adbc.Client", "src\Client\Apache.Arrow.Adbc.Client.csproj", "{A405F4A0-5938-4139-B2DF-ED9A05EC3D7C}"
Expand All @@ -32,6 +28,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Adbc.Drivers.A
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Adbc.Tests.Drivers.Apache", "test\Drivers\Apache\Apache.Arrow.Adbc.Tests.Drivers.Apache.csproj", "{714F0BD2-3A92-4D1A-8FAC-D0C0599BE3E3}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Adbc.Drivers.Interop.FlightSql", "src\Drivers\Interop\FlightSql\Apache.Arrow.Adbc.Drivers.Interop.FlightSql.csproj", "{4076D7E9-728D-4DF4-999F-658784957648}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Adbc.Tests.Drivers.Interop.FlightSql", "test\Drivers\Interop\FlightSql\Apache.Arrow.Adbc.Tests.Drivers.Interop.FlightSql.csproj", "{C5503227-C5A7-406F-83AA-681F292EA61F}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Adbc.Drivers.FlightSql", "src\Drivers\FlightSql\Apache.Arrow.Adbc.Drivers.FlightSql.csproj", "{77D5A92F-4136-4DE7-81F4-43B981223280}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Adbc.Tests.Drivers.FlightSql", "test\Drivers\FlightSql\Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj", "{5B27FB02-D4AE-4ACB-AD88-5E64EEB61729}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand All @@ -46,14 +50,6 @@ Global
{00C143BA-F1CF-4117-9DE6-E73DC4D208F8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{00C143BA-F1CF-4117-9DE6-E73DC4D208F8}.Release|Any CPU.ActiveCfg = Release|Any CPU
{00C143BA-F1CF-4117-9DE6-E73DC4D208F8}.Release|Any CPU.Build.0 = Release|Any CPU
{19AA450A-2F87-49BD-9122-8AD07D4C6DCE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{19AA450A-2F87-49BD-9122-8AD07D4C6DCE}.Debug|Any CPU.Build.0 = Debug|Any CPU
{19AA450A-2F87-49BD-9122-8AD07D4C6DCE}.Release|Any CPU.ActiveCfg = Release|Any CPU
{19AA450A-2F87-49BD-9122-8AD07D4C6DCE}.Release|Any CPU.Build.0 = Release|Any CPU
{5C15E79C-19C4-4FF4-BB82-28754FE3966B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5C15E79C-19C4-4FF4-BB82-28754FE3966B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5C15E79C-19C4-4FF4-BB82-28754FE3966B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5C15E79C-19C4-4FF4-BB82-28754FE3966B}.Release|Any CPU.Build.0 = Release|Any CPU
{A405F4A0-5938-4139-B2DF-ED9A05EC3D7C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A405F4A0-5938-4139-B2DF-ED9A05EC3D7C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A405F4A0-5938-4139-B2DF-ED9A05EC3D7C}.Release|Any CPU.ActiveCfg = Release|Any CPU
Expand Down Expand Up @@ -82,22 +78,40 @@ Global
{714F0BD2-3A92-4D1A-8FAC-D0C0599BE3E3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{714F0BD2-3A92-4D1A-8FAC-D0C0599BE3E3}.Release|Any CPU.ActiveCfg = Release|Any CPU
{714F0BD2-3A92-4D1A-8FAC-D0C0599BE3E3}.Release|Any CPU.Build.0 = Release|Any CPU
{4076D7E9-728D-4DF4-999F-658784957648}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{4076D7E9-728D-4DF4-999F-658784957648}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4076D7E9-728D-4DF4-999F-658784957648}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4076D7E9-728D-4DF4-999F-658784957648}.Release|Any CPU.Build.0 = Release|Any CPU
{C5503227-C5A7-406F-83AA-681F292EA61F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C5503227-C5A7-406F-83AA-681F292EA61F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C5503227-C5A7-406F-83AA-681F292EA61F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C5503227-C5A7-406F-83AA-681F292EA61F}.Release|Any CPU.Build.0 = Release|Any CPU
{77D5A92F-4136-4DE7-81F4-43B981223280}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{77D5A92F-4136-4DE7-81F4-43B981223280}.Debug|Any CPU.Build.0 = Debug|Any CPU
{77D5A92F-4136-4DE7-81F4-43B981223280}.Release|Any CPU.ActiveCfg = Release|Any CPU
{77D5A92F-4136-4DE7-81F4-43B981223280}.Release|Any CPU.Build.0 = Release|Any CPU
{5B27FB02-D4AE-4ACB-AD88-5E64EEB61729}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5B27FB02-D4AE-4ACB-AD88-5E64EEB61729}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5B27FB02-D4AE-4ACB-AD88-5E64EEB61729}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5B27FB02-D4AE-4ACB-AD88-5E64EEB61729}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{00C143BA-F1CF-4117-9DE6-E73DC4D208F8} = {5BD04C26-CE52-4893-8C1A-479705195CEF}
{19AA450A-2F87-49BD-9122-8AD07D4C6DCE} = {FEB257A0-4FD3-495E-9A47-9E1649755445}
{C7290227-E925-47E7-8B6B-A8B171645D58} = {5BD04C26-CE52-4893-8C1A-479705195CEF}
{5C15E79C-19C4-4FF4-BB82-28754FE3966B} = {C7290227-E925-47E7-8B6B-A8B171645D58}
{A405F4A0-5938-4139-B2DF-ED9A05EC3D7C} = {B6111602-2DC4-4B2F-9598-E3EE1972D3E4}
{A748041C-EF9A-4E88-B6FB-9F2D6CB79170} = {FEB257A0-4FD3-495E-9A47-9E1649755445}
{EA43BB7C-BC00-4701-BDF4-367880C2495C} = {C7290227-E925-47E7-8B6B-A8B171645D58}
{30024B6F-7BC1-4574-BE5A-924FBD6EAF83} = {FEB257A0-4FD3-495E-9A47-9E1649755445}
{8BE1EECC-3ACF-41B2-AF7D-1A67196FF6C7} = {C7290227-E925-47E7-8B6B-A8B171645D58}
{6C0D8BE1-4A23-4C2F-88B1-D2FBEA0B1903} = {FEB257A0-4FD3-495E-9A47-9E1649755445}
{714F0BD2-3A92-4D1A-8FAC-D0C0599BE3E3} = {C7290227-E925-47E7-8B6B-A8B171645D58}
{4076D7E9-728D-4DF4-999F-658784957648} = {FEB257A0-4FD3-495E-9A47-9E1649755445}
{C5503227-C5A7-406F-83AA-681F292EA61F} = {C7290227-E925-47E7-8B6B-A8B171645D58}
{77D5A92F-4136-4DE7-81F4-43B981223280} = {FEB257A0-4FD3-495E-9A47-9E1649755445}
{5B27FB02-D4AE-4ACB-AD88-5E64EEB61729} = {C7290227-E925-47E7-8B6B-A8B171645D58}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {4795CF16-0FDB-4BE0-9768-5CF31564DC03}
Expand Down
98 changes: 98 additions & 0 deletions csharp/src/Apache.Arrow.Adbc/AdbcDriverLoader.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

using System;
using System.IO;
using System.Runtime.InteropServices;
using Apache.Arrow.Adbc.C;

namespace Apache.Arrow.Adbc
{
/// <summary>
/// Lightweight class for loading an Interop driver to .NET.
/// </summary>
public abstract class AdbcDriverLoader
{
readonly string driverShortName;
readonly string entryPoint;

/// <summary>
/// Initializes the driver loader with the driver name and entry point.
/// </summary>
/// <param name="driverShortName">Short driver name, with no extension.</param>
/// <param name="entryPoint">The entry point. Defaults to `AdbcDriverInit` if not provided.</param>
/// <exception cref="ArgumentException"></exception>
protected AdbcDriverLoader(string driverShortName, string entryPoint = "AdbcDriverInit")
{
if (string.IsNullOrEmpty(driverShortName))
throw new ArgumentException("cannot be null or empty", nameof(driverShortName));

if (string.IsNullOrEmpty(entryPoint))
throw new ArgumentException("cannot be null or empty", nameof(entryPoint));

this.driverShortName = driverShortName;
this.entryPoint = entryPoint;
}

/// <summary>
/// Loads the Interop from the current directory using the default name and entry point.
/// </summary>
/// <returns>An <see cref="AdbcDriver"/> based on the Flight SQL Go driver.</returns>
/// <exception cref="FileNotFoundException"></exception>
protected AdbcDriver FindAndLoadDriver()
{
string root = "runtimes";
string native = "native";
string architecture = RuntimeInformation.OSArchitecture.ToString().ToLower();
string fileName = driverShortName;
string file;

// matches extensions in https://github.com/apache/arrow-adbc/blob/main/go/adbc/pkg/Makefile
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
file = Path.Combine(root, $"linux-{architecture}", native, $"{fileName}.so");
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
file = Path.Combine(root, $"win-{architecture}", native, $"{fileName}.dll");
else
file = Path.Combine(root, $"osx-{architecture}", native, $"{fileName}.dylib");

if (File.Exists(file))
{
// get the full path because some .NET versions need it
file = Path.GetFullPath(file);
}
else
{
throw new FileNotFoundException($"Could not find {file}");
}

return LoadDriver(file, entryPoint);
}

/// <summary>
/// Loads the Interop driver from the current directory using the default name and entry point.
/// </summary>
/// <param name="file">The file to load.</param>
/// <param name="entryPoint">The entry point of the file.</param>
/// <returns>An <see cref="AdbcDriver"/>.</returns>
public static AdbcDriver LoadDriver(string file, string entryPoint)
{
AdbcDriver driver = CAdbcDriverImporter.Load(file, entryPoint);

return driver;
}
}
}
104 changes: 0 additions & 104 deletions csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,8 @@
*/

using System;
using System.IO;
using System.Threading.Tasks;
using Apache.Arrow.Ipc;
using Apache.Arrow.Types;

namespace Apache.Arrow.Adbc
{
Expand Down Expand Up @@ -157,108 +155,6 @@ public virtual void Dispose()
{
}

/// <summary>
/// Gets a value from the Arrow array at the specified index,
/// using the Field metadata for information.
/// </summary>
/// <param name="arrowArray">
/// The Arrow array.
/// </param>
/// <param name="field">
/// The <see cref="Field"/> from the <see cref="Schema"/> that can
/// be used for metadata inspection.
/// </param>
/// <param name="index">
/// The index in the array to get the value from.
/// </param>
public virtual object? GetValue(IArrowArray arrowArray, int index)
{
if (arrowArray == null) throw new ArgumentNullException(nameof(arrowArray));
if (index < 0) throw new ArgumentOutOfRangeException(nameof(index));

switch (arrowArray)
{
case BooleanArray booleanArray:
return booleanArray.GetValue(index);
case Date32Array date32Array:
return date32Array.GetDateTime(index);
case Date64Array date64Array:
return date64Array.GetDateTime(index);
case Decimal128Array decimal128Array:
return decimal128Array.GetSqlDecimal(index);
case Decimal256Array decimal256Array:
return decimal256Array.GetString(index);
case DoubleArray doubleArray:
return doubleArray.GetValue(index);
case FloatArray floatArray:
return floatArray.GetValue(index);
#if NET5_0_OR_GREATER
case PrimitiveArray<Half> halfFloatArray:
return halfFloatArray.GetValue(index);
#endif
case Int8Array int8Array:
return int8Array.GetValue(index);
case Int16Array int16Array:
return int16Array.GetValue(index);
case Int32Array int32Array:
return int32Array.GetValue(index);
case Int64Array int64Array:
return int64Array.GetValue(index);
case StringArray stringArray:
return stringArray.GetString(index);
#if NET6_0_OR_GREATER
case Time32Array time32Array:
return time32Array.GetTime(index);
case Time64Array time64Array:
return time64Array.GetTime(index);
#else
case Time32Array time32Array:
int? time32 = time32Array.GetValue(index);
if (time32 == null) { return null; }
return ((Time32Type)time32Array.Data.DataType).Unit switch
{
TimeUnit.Second => TimeSpan.FromSeconds(time32.Value),
TimeUnit.Millisecond => TimeSpan.FromMilliseconds(time32.Value),
_ => throw new InvalidDataException("Unsupported time unit for Time32Type")
};
case Time64Array time64Array:
long? time64 = time64Array.GetValue(index);
if (time64 == null) { return null; }
return ((Time64Type)time64Array.Data.DataType).Unit switch
{
TimeUnit.Microsecond => TimeSpan.FromTicks(time64.Value * 10),
TimeUnit.Nanosecond => TimeSpan.FromTicks(time64.Value / 100),
_ => throw new InvalidDataException("Unsupported time unit for Time64Type")
};
#endif
case TimestampArray timestampArray:
return timestampArray.GetTimestamp(index);
case UInt8Array uInt8Array:
return uInt8Array.GetValue(index);
case UInt16Array uInt16Array:
return uInt16Array.GetValue(index);
case UInt32Array uInt32Array:
return uInt32Array.GetValue(index);
case UInt64Array uInt64Array:
return uInt64Array.GetValue(index);

case BinaryArray binaryArray:
if (!binaryArray.IsNull(index))
return binaryArray.GetBytes(index).ToArray();

return null;

// not covered:
// -- struct array
// -- dictionary array
// -- fixed size binary
// -- list array
// -- union array
}

return null;
}

/// <summary>
/// Attempts to cancel an in-progress operation on a connection.
/// </summary>
Expand Down
1 change: 1 addition & 0 deletions csharp/src/Apache.Arrow.Adbc/Apache.Arrow.Adbc.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Apache.Arrow" Version="18.1.0" />
<PackageReference Include="System.Text.Json" Version="8.0.5" />
</ItemGroup>
<ItemGroup Condition="$([MSBuild]::IsTargetFrameworkCompatible($(TargetFramework), 'net6.0'))">
<Compile Remove="C\NativeLibrary.cs" />
Expand Down
Loading

0 comments on commit 08ac053

Please sign in to comment.