Skip to content

Commit 5e7e764

Browse files
teo-tsirpaniseerhardt
teo-tsirpanis
and
eerhardt
authored
GH-25163: [C#] Support half-float arrays. (#34618)
### Rationale for this change .NET 5 introduced the [`System.Half`](https://devblogs.microsoft.com/dotnet/introducing-the-half-type/) type, which represents 16-bit floats. This PR adds support for them in Apache Arrow. ### What changes are included in this PR? I multi-targeted the `Apache.Arrow` project to .NET 6 (because .NET 5 is unsupported) and added a `HalfFloatArray` type with a very similar implementation as the other floating-point array types. I also updated the README. ### Are these changes tested? Yes. I also refactored the array tests to reduce duplication among the various numeric types. ### Are there any user-facing changes? Yes. * Closes: #25163 Lead-authored-by: Theodore Tsirpanis <teo@tsirpanis.gr> Co-authored-by: Eric Erhardt <eric.erhardt@microsoft.com> Signed-off-by: Eric Erhardt <eric.erhardt@microsoft.com>
1 parent 196fbe5 commit 5e7e764

12 files changed

Lines changed: 159 additions & 59 deletions

‎csharp/Directory.Build.props‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434

3535
<PropertyGroup>
3636
<EmbedUntrackedSources>true</EmbedUntrackedSources>
37-
<LangVersion>8.0</LangVersion>
37+
<LangVersion>latest</LangVersion>
3838
<SignAssembly>true</SignAssembly>
3939
<AssemblyOriginatorKeyFile>$(CSharpDir)ApacheArrow.snk</AssemblyOriginatorKeyFile>
4040
</PropertyGroup>

‎csharp/README.md‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ for currently available features.
7979

8080
- Int8, Int16, Int32, Int64
8181
- UInt8, UInt16, UInt32, UInt64
82-
- Float, Double
82+
- Float, Double, Half-float (.NET 6+)
8383
- Binary (variable-length)
8484
- String (utf-8)
8585
- Null
@@ -126,12 +126,10 @@ for currently available features.
126126
- Dictionary Encoding
127127
- Types
128128
- Tensor
129-
- Table
130129
- Arrays
131130
- Union
132131
- Dense
133132
- Sparse
134-
- Half-Float
135133
- Array Operations
136134
- Equality / Comparison
137135
- Casting

‎csharp/src/Apache.Arrow/Apache.Arrow.csproj‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
4-
<TargetFrameworks>netstandard1.3;netstandard2.0;netcoreapp3.1</TargetFrameworks>
4+
<TargetFrameworks>netstandard1.3;netstandard2.0;netcoreapp3.1;net6.0</TargetFrameworks>
55
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
66
<DefineConstants>$(DefineConstants);UNSAFE_BYTEBUFFER;BYTEBUFFER_NO_BOUNDS_CHECK;ENABLE_SPAN_T</DefineConstants>
77

@@ -41,4 +41,7 @@
4141
<Compile Remove="Extensions\StreamExtensions.netstandard.cs" />
4242
<Compile Remove="Extensions\TupleExtensions.netstandard.cs" />
4343
</ItemGroup>
44+
<ItemGroup Condition="!$([MSBuild]::IsTargetFrameworkCompatible($(TargetFramework), 'net5.0'))">
45+
<Compile Remove="Arrays\HalfFloatArray.cs" />
46+
</ItemGroup>
4447
</Project>

‎csharp/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,12 @@ internal static IArrowArrayBuilder<IArrowArray, IArrowArrayBuilder<IArrowArray>>
4242
return new UInt64Array.Builder();
4343
case ArrowTypeId.Int64:
4444
return new Int64Array.Builder();
45+
case ArrowTypeId.HalfFloat:
46+
#if NET5_0_OR_GREATER
47+
return new HalfFloatArray.Builder();
48+
#else
49+
throw new NotSupportedException("Half-float arrays are not supported by this target framework.");
50+
#endif
4551
case ArrowTypeId.Float:
4652
return new FloatArray.Builder();
4753
case ArrowTypeId.Double:
@@ -70,7 +76,6 @@ internal static IArrowArrayBuilder<IArrowArray, IArrowArrayBuilder<IArrowArray>>
7076
case ArrowTypeId.Union:
7177
case ArrowTypeId.Dictionary:
7278
case ArrowTypeId.FixedSizedBinary:
73-
case ArrowTypeId.HalfFloat:
7479
case ArrowTypeId.Interval:
7580
case ArrowTypeId.Map:
7681
default:

‎csharp/src/Apache.Arrow/Arrays/ArrowArrayFactory.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,11 @@ public static IArrowArray BuildArray(ArrayData data)
7676
case ArrowTypeId.Dictionary:
7777
return new DictionaryArray(data);
7878
case ArrowTypeId.HalfFloat:
79+
#if NET5_0_OR_GREATER
80+
return new HalfFloatArray(data);
81+
#else
82+
throw new NotSupportedException("Half-float arrays are not supported by this target framework.");
83+
#endif
7984
case ArrowTypeId.Interval:
8085
case ArrowTypeId.Map:
8186
default:

‎csharp/src/Apache.Arrow/Arrays/HalfFloatArray.cs‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one or more
2+
// contributor license agreements. See the NOTICE file distributed with
3+
// this work for additional information regarding copyright ownership.
4+
// The ASF licenses this file to You under the Apache License, Version 2.0
5+
// (the "License"); you may not use this file except in compliance with
6+
// the License. You may obtain a copy of the License at
7+
//
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
//
10+
// Unless required by applicable law or agreed to in writing, software
11+
// distributed under the License is distributed on an "AS IS" BASIS,
12+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
// See the License for the specific language governing permissions and
14+
// limitations under the License.
15+
16+
using Apache.Arrow.Types;
17+
using System;
18+
19+
namespace Apache.Arrow
20+
{
21+
public class HalfFloatArray : PrimitiveArray<Half>
22+
{
23+
public class Builder : PrimitiveArrayBuilder<Half, HalfFloatArray, Builder>
24+
{
25+
protected override HalfFloatArray Build(
26+
ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer,
27+
int length, int nullCount, int offset) =>
28+
new HalfFloatArray(valueBuffer, nullBitmapBuffer, length, nullCount, offset);
29+
}
30+
31+
public HalfFloatArray(
32+
ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer,
33+
int length, int nullCount, int offset)
34+
: this(new ArrayData(HalfFloatType.Default, length, nullCount, offset,
35+
new[] { nullBitmapBuffer, valueBuffer }))
36+
{ }
37+
38+
public HalfFloatArray(ArrayData data)
39+
: base(data)
40+
{
41+
data.EnsureDataType(ArrowTypeId.HalfFloat);
42+
}
43+
44+
public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor);
45+
}
46+
}

‎csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,9 @@ internal class ArrowRecordBatchFlatBufferBuilder :
3838
IArrowArrayVisitor<UInt16Array>,
3939
IArrowArrayVisitor<UInt32Array>,
4040
IArrowArrayVisitor<UInt64Array>,
41+
#if NET5_0_OR_GREATER
42+
IArrowArrayVisitor<HalfFloatArray>,
43+
#endif
4144
IArrowArrayVisitor<FloatArray>,
4245
IArrowArrayVisitor<DoubleArray>,
4346
IArrowArrayVisitor<BooleanArray>,
@@ -87,6 +90,9 @@ public ArrowRecordBatchFlatBufferBuilder()
8790
public void Visit(UInt16Array array) => CreateBuffers(array);
8891
public void Visit(UInt32Array array) => CreateBuffers(array);
8992
public void Visit(UInt64Array array) => CreateBuffers(array);
93+
#if NET5_0_OR_GREATER
94+
public void Visit(HalfFloatArray array) => CreateBuffers(array);
95+
#endif
9096
public void Visit(FloatArray array) => CreateBuffers(array);
9197
public void Visit(DoubleArray array) => CreateBuffers(array);
9298
public void Visit(TimestampArray array) => CreateBuffers(array);

‎csharp/src/Apache.Arrow/RecordBatch.Builder.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@ internal ArrayBuilder(MemoryAllocator allocator)
4242
public UInt16Array UInt16(Action<UInt16Array.Builder> action) => Build<UInt16Array, UInt16Array.Builder>(new UInt16Array.Builder(), action);
4343
public UInt32Array UInt32(Action<UInt32Array.Builder> action) => Build<UInt32Array, UInt32Array.Builder>(new UInt32Array.Builder(), action);
4444
public UInt64Array UInt64(Action<UInt64Array.Builder> action) => Build<UInt64Array, UInt64Array.Builder>(new UInt64Array.Builder(), action);
45+
#if NET5_0_OR_GREATER
46+
public HalfFloatArray HalfFloat(Action<HalfFloatArray.Builder> action) => Build<HalfFloatArray, HalfFloatArray.Builder>(new HalfFloatArray.Builder(), action);
47+
#endif
4548
public FloatArray Float(Action<FloatArray.Builder> action) => Build<FloatArray, FloatArray.Builder>(new FloatArray.Builder(), action);
4649
public DoubleArray Double(Action<DoubleArray.Builder> action) => Build<DoubleArray, DoubleArray.Builder>(new DoubleArray.Builder(), action);
4750
public Decimal128Array Decimal128(Decimal128Type type, Action<Decimal128Array.Builder> action) =>

‎csharp/test/Apache.Arrow.Tests/ArrayBuilderTests.cs‎

Lines changed: 40 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
using System;
1818
using System.Collections.Generic;
1919
using System.Linq;
20+
using System.Numerics;
2021
using Xunit;
2122

2223
namespace Apache.Arrow.Tests
@@ -28,36 +29,49 @@ public class ArrayBuilderTests
2829
[Fact]
2930
public void PrimitiveArrayBuildersProduceExpectedArray()
3031
{
31-
TestArrayBuilder<Int8Array, Int8Array.Builder>(x => x.Append(10).Append(20).Append(30));
32-
TestArrayBuilder<Int16Array, Int16Array.Builder>(x => x.Append(10).Append(20).Append(30));
33-
TestArrayBuilder<Int32Array, Int32Array.Builder>(x => x.Append(10).Append(20).Append(30));
34-
TestArrayBuilder<Int64Array, Int64Array.Builder>(x => x.Append(10).Append(20).Append(30));
35-
TestArrayBuilder<UInt8Array, UInt8Array.Builder>(x => x.Append(10).Append(20).Append(30));
36-
TestArrayBuilder<UInt16Array, UInt16Array.Builder>(x => x.Append(10).Append(20).Append(30));
37-
TestArrayBuilder<UInt32Array, UInt32Array.Builder>(x => x.Append(10).Append(20).Append(30));
38-
TestArrayBuilder<UInt64Array, UInt64Array.Builder>(x => x.Append(10).Append(20).Append(30));
39-
TestArrayBuilder<FloatArray, FloatArray.Builder>(x => x.Append(10).Append(20).Append(30));
40-
TestArrayBuilder<DoubleArray, DoubleArray.Builder>(x => x.Append(10).Append(20).Append(30));
41-
TestArrayBuilder<Time32Array, Time32Array.Builder>(x => x.Append(10).Append(20).Append(30));
42-
TestArrayBuilder<Time64Array, Time64Array.Builder>(x => x.Append(10).Append(20).Append(30));
32+
Test<sbyte, Int8Array, Int8Array.Builder>();
33+
Test<short, Int16Array, Int16Array.Builder>();
34+
Test<int, Int32Array, Int32Array.Builder>();
35+
Test<long, Int64Array, Int64Array.Builder>();
36+
Test<byte, UInt8Array, UInt8Array.Builder>();
37+
Test<ushort, UInt16Array, UInt16Array.Builder>();
38+
Test<uint, UInt32Array, UInt32Array.Builder>();
39+
Test<ulong, UInt64Array, UInt64Array.Builder>();
40+
Test<Half, HalfFloatArray, HalfFloatArray.Builder>();
41+
Test<float, FloatArray, FloatArray.Builder>();
42+
Test<double, DoubleArray, DoubleArray.Builder>();
43+
Test<int, Time32Array, Time32Array.Builder>();
44+
Test<long, Time64Array, Time64Array.Builder>();
45+
46+
static void Test<T, TArray, TBuilder>()
47+
where T : struct, INumber<T>
48+
where TArray : PrimitiveArray<T>
49+
where TBuilder : PrimitiveArrayBuilder<T, TArray, TBuilder>, new() =>
50+
TestArrayBuilder<TArray, TBuilder>(x => x.Append(T.CreateChecked(10)).Append(T.CreateChecked(20)).Append(T.CreateChecked(30)));
4351
}
4452

4553
[Fact]
4654
public void PrimitiveArrayBuildersProduceExpectedArrayWithNulls()
4755
{
48-
TestArrayBuilder<Int8Array, Int8Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(127), 4, 2, 0x09);
49-
TestArrayBuilder<Int16Array, Int16Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
50-
TestArrayBuilder<Int32Array, Int32Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
51-
TestArrayBuilder<Int64Array, Int64Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
52-
TestArrayBuilder<UInt8Array, UInt8Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(127), 4, 2, 0x09);
53-
TestArrayBuilder<UInt16Array, UInt16Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
54-
TestArrayBuilder<UInt32Array, UInt32Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
55-
TestArrayBuilder<UInt64Array, UInt64Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
56-
TestArrayBuilder<UInt64Array, UInt64Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
57-
TestArrayBuilder<FloatArray, FloatArray.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
58-
TestArrayBuilder<DoubleArray, DoubleArray.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
59-
TestArrayBuilder<Time32Array, Time32Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
60-
TestArrayBuilder<Time64Array, Time64Array.Builder>(x => x.Append(123).AppendNull().AppendNull().Append(456), 4, 2, 0x09);
56+
Test<sbyte, Int8Array, Int8Array.Builder>();
57+
Test<short, Int16Array, Int16Array.Builder>();
58+
Test<int, Int32Array, Int32Array.Builder>();
59+
Test<long, Int64Array, Int64Array.Builder>();
60+
Test<byte, UInt8Array, UInt8Array.Builder>();
61+
Test<ushort, UInt16Array, UInt16Array.Builder>();
62+
Test<uint, UInt32Array, UInt32Array.Builder>();
63+
Test<ulong, UInt64Array, UInt64Array.Builder>();
64+
Test<Half, HalfFloatArray, HalfFloatArray.Builder>();
65+
Test<float, FloatArray, FloatArray.Builder>();
66+
Test<double, DoubleArray, DoubleArray.Builder>();
67+
Test<int, Time32Array, Time32Array.Builder>();
68+
Test<long, Time64Array, Time64Array.Builder>();
69+
70+
static void Test<T, TArray, TBuilder>()
71+
where T : struct, INumber<T>
72+
where TArray : PrimitiveArray<T>
73+
where TBuilder : PrimitiveArrayBuilder<T, TArray, TBuilder>, new() =>
74+
TestArrayBuilder<TArray, TBuilder>(x => x.Append(T.CreateChecked(123)).AppendNull().AppendNull().Append(T.CreateChecked(127)), 4, 2, 0x09);
6175
}
6276

6377
[Fact]
@@ -138,7 +152,7 @@ List<string> ConvertStringArrayToList(StringArray array)
138152
[Fact]
139153
public void ListArrayBuilderValidityBuffer()
140154
{
141-
ListArray listArray = new ListArray.Builder(Int64Type.Default).Append().AppendNull().Build();
155+
ListArray listArray = new ListArray.Builder(Int64Type.Default).Append().AppendNull().Build();
142156
Assert.False(listArray.IsValid(2));
143157
}
144158

0 commit comments

Comments
 (0)