From 99fb7a35778c5016239026128b353cc668bd712a Mon Sep 17 00:00:00 2001 From: demetrius albuquerque Date: Sat, 25 Jul 2026 17:14:39 +0200 Subject: [PATCH 1/7] test: add arrow-testing submodule and test data resolver Add `apache/arrow-testing` as a Git submodule to provide a shared IPC conformance corpus used across all Arrow implementations. - Add `testing` submodule pointing to arrow-testing - Add `ArrowTestData` helper to locate corpus files via ARROW_TEST_DATA env var or fallback to submodule path - Add test that verifies path resolution - Exclude .gitmodules from RAT check This is the first step toward replacing the Go-based data generator with the official test corpus, enabling broader conformance coverage. --- .gitmodules | 3 + Tests/ArrowTests/ArrowTestData.swift | 96 ++++++++++++++++++++++++++++ dev/release/rat_exclude_files.txt | 1 + testing | 1 + 4 files changed, 101 insertions(+) create mode 100644 .gitmodules create mode 100644 Tests/ArrowTests/ArrowTestData.swift create mode 160000 testing diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..93832ac --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "testing"] + path = testing + url = https://github.com/apache/arrow-testing.git diff --git a/Tests/ArrowTests/ArrowTestData.swift b/Tests/ArrowTests/ArrowTestData.swift new file mode 100644 index 0000000..c803b9d --- /dev/null +++ b/Tests/ArrowTests/ArrowTestData.swift @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import XCTest + +/// Locates files in the `apache/arrow-testing` corpus. +/// +/// The corpus is resolved from, in order: +/// 1. The `ARROW_TEST_DATA` environment variable. +/// 2. The `testing/data` submodule directory. +/// +/// When neither is available the corpus is treated as absent and callers skip, +/// so that `swift test` succeeds on a checkout where the submodule has not +/// been initialised, and when tests run against an extracted source archive. +enum ArrowTestData { + /// Root of the corpus, or `nil` when it cannot be located. + static var root: URL? { + let manager = FileManager.default + + if let path = ProcessInfo.processInfo.environment["ARROW_TEST_DATA"], !path.isEmpty { + let url = URL(fileURLWithPath: path, isDirectory: true) + return manager.fileExists(atPath: url.path) ? url : nil + } + + let fallback = repositoryRoot + .appendingPathComponent("testing", isDirectory: true) + .appendingPathComponent("data", isDirectory: true) + return manager.fileExists(atPath: fallback.path) ? fallback : nil + } + + /// Absolute URL of a file inside the corpus, for example + /// `arrow-ipc-stream/integration/cpp-21.0.0/generated_primitive.stream`. + /// + /// Throws `XCTSkip` when the corpus is unavailable. Fails the calling test + /// when the corpus is present but does not contain the requested file, + /// which indicates a stale submodule rather than a missing corpus. + static func url( + _ relativePath: String, + file: StaticString = #filePath, + line: UInt = #line + ) throws -> URL { + guard let root else { + throw XCTSkip( + """ + arrow-testing corpus not found. Run \ + 'git submodule update --init', or set ARROW_TEST_DATA to the \ + data directory of an apache/arrow-testing checkout. + """) + } + + let url = root.appendingPathComponent(relativePath) + if !FileManager.default.fileExists(atPath: url.path) { + XCTFail( + "Missing file in arrow-testing corpus: \(relativePath)", + file: file, + line: line) + } + return url + } + + /// This file is at `/Tests/ArrowTests/ArrowTestData.swift`, so + /// the repository root is three levels up. `#filePath` is used rather than + /// `#file` because the latter can be shortened to a bare file name + /// depending on compiler settings. + private static var repositoryRoot: URL { + URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + } +} + +final class ArrowTestDataTests: XCTestCase { + /// Verifies path resolution only. Reading and validating corpus contents + /// is deliberately out of scope here. + func testResolvesCorpusPath() throws { + let url = try ArrowTestData.url( + "arrow-ipc-stream/integration/cpp-21.0.0/generated_primitive.stream") + XCTAssertTrue(FileManager.default.fileExists(atPath: url.path)) + } +} diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index f438f1e..9ebde90 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -19,4 +19,5 @@ */Package.resolved */go.sum .github/pull_request_template.md +.gitmodules Package.resolved diff --git a/testing b/testing new file mode 160000 index 0000000..9ff285c --- /dev/null +++ b/testing @@ -0,0 +1 @@ +Subproject commit 9ff285c88565f0f6abc855918c6a342e70e4909c From e24f998e881d2d8d75ced3676a12226d9f0ca6c5 Mon Sep 17 00:00:00 2001 From: demetrius albuquerque Date: Sat, 25 Jul 2026 18:52:26 +0200 Subject: [PATCH 2/7] test: Verify IPC stream reading against arrow-testing corpus Adds conformance tests reading generated_primitive.stream and its no-batches and zero-length variants from the arrow-testing corpus. The existing streaming tests write with ArrowWriter.writeStreaming and read the result back with ArrowReader.readStreaming, which establishes internal consistency but not conformance: a symmetric defect in the writer and reader would pass. Until now no test read a stream produced by another Arrow implementation. The shared schema expectations are lifted out of the file conformance tests so both classes use them. Note: testStreamAndFileAgree is disabled pending a fix to ArrowBuffer memory initialization (see inline comment for details); it will serve as the regression test when that issue is resolved. Part of #10 --- Tests/ArrowTests/IPCConformanceTests.swift | 241 +++++++++++++++++++++ 1 file changed, 241 insertions(+) create mode 100644 Tests/ArrowTests/IPCConformanceTests.swift diff --git a/Tests/ArrowTests/IPCConformanceTests.swift b/Tests/ArrowTests/IPCConformanceTests.swift new file mode 100644 index 0000000..64e2b40 --- /dev/null +++ b/Tests/ArrowTests/IPCConformanceTests.swift @@ -0,0 +1,241 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import XCTest +@testable import Arrow + +/// Expectations shared by the file and stream conformance tests, which read +/// the same logical cases in two different encodings. +private enum PrimitiveCase { + static let corpusPrefix = "arrow-ipc-stream/integration/cpp-21.0.0" + + /// The 22 fields of the `generated_primitive` cases, in file order. + static let fields: [(String, ArrowTypeId)] = [ + ("bool_nullable", .boolean), ("bool_nonnullable", .boolean), + ("int8_nullable", .int8), ("int8_nonnullable", .int8), + ("int16_nullable", .int16), ("int16_nonnullable", .int16), + ("int32_nullable", .int32), ("int32_nonnullable", .int32), + ("int64_nullable", .int64), ("int64_nonnullable", .int64), + ("uint8_nullable", .uint8), ("uint8_nonnullable", .uint8), + ("uint16_nullable", .uint16), ("uint16_nonnullable", .uint16), + ("uint32_nullable", .uint32), ("uint32_nonnullable", .uint32), + ("uint64_nullable", .uint64), ("uint64_nonnullable", .uint64), + ("float32_nullable", .float), ("float32_nonnullable", .float), + ("float64_nullable", .double), ("float64_nonnullable", .double) + ] + + static func assertSchema( + _ schema: ArrowSchema?, + file: StaticString = #filePath, + line: UInt = #line + ) { + guard let schema else { + XCTFail("schema is nil", file: file, line: line) + return + } + XCTAssertEqual(schema.fields.count, fields.count, file: file, line: line) + guard schema.fields.count == fields.count else { return } + for (index, expected) in fields.enumerated() { + XCTAssertEqual(schema.fields[index].name, expected.0, file: file, line: line) + XCTAssertEqual(schema.fields[index].type.id, expected.1, file: file, line: line) + } + } + + /// Every value in the result rendered as a string, with `nil` for nulls, + /// used by testStreamAndFileAgree (currently disabled). + /// Remove this when the test is re-enabled. + /* + private static func snapshot(_ result: ArrowReader.ArrowReaderResult) -> [String] { + var values: [String] = [] + for (batchIndex, batch) in result.batches.enumerated() { + for column in 0.. ArrowReader.ArrowReaderResult { + let url = try ArrowTestData.url("\(PrimitiveCase.corpusPrefix)/\(name)") + switch ArrowReader().fromFile(url) { + case .success(let result): + return result + case .failure(let error): + throw error + } + } + + func testPrimitiveFileSchemaAndShape() throws { + let result = try readFile("generated_primitive.arrow_file") + PrimitiveCase.assertSchema(result.schema) + XCTAssertEqual(result.batches.count, 2) + XCTAssertEqual(result.batches.reduce(0) { $0 + Int($1.length) }, 37) + for batch in result.batches { + XCTAssertEqual(batch.columns.count, PrimitiveCase.fields.count) + } + + let batch0 = result.batches[0] + XCTAssertEqual(batch0.length, 17) + + let col0 = batch0.column(0) + let arr0 = col0.array as! AsString // swiftlint:disable:this force_cast + XCTAssertNil(col0.array.asAny(0)) + XCTAssertNil(col0.array.asAny(1)) + XCTAssertEqual(arr0.asString(2), "true") + XCTAssertNil(col0.array.asAny(3)) + + let col2 = batch0.column(2) + let arr2 = col2.array as! AsString // swiftlint:disable:this force_cast + XCTAssertEqual(arr2.asString(0), "-128") + XCTAssertEqual(arr2.asString(1), "127") + XCTAssertEqual(arr2.asString(2), "27") + XCTAssertEqual(arr2.asString(3), "-90") + + let col4 = batch0.column(4) + let arr4 = col4.array as! AsString // swiftlint:disable:this force_cast + XCTAssertEqual(arr4.asString(0), "-32768") + XCTAssertEqual(arr4.asString(1), "32767") + + let col6 = batch0.column(6) + let arr6 = col6.array as! AsString // swiftlint:disable:this force_cast + XCTAssertEqual(arr6.asString(0), "-2147483648") + + let col10 = batch0.column(10) + let arr10 = col10.array as! AsString // swiftlint:disable:this force_cast + XCTAssertEqual(arr10.asString(0), "0") + XCTAssertEqual(arr10.asString(1), "255") + + let col18 = batch0.column(18) + let f0 = try XCTUnwrap(col18.array.asAny(0) as? Float) + XCTAssertEqual(f0, 641.818, accuracy: 0.001) + + let col20 = batch0.column(20) + let d0 = try XCTUnwrap(col20.array.asAny(0) as? Double) + let d1 = try XCTUnwrap(col20.array.asAny(1) as? Double) + XCTAssertEqual(d0, -955.504, accuracy: 0.001) + XCTAssertEqual(d1, -1746.99, accuracy: 0.001) + + let batch1 = result.batches[1] + XCTAssertEqual(batch1.length, 20) + + let b1col0 = batch1.column(0) + let b1arr0 = b1col0.array as! AsString // swiftlint:disable:this force_cast + XCTAssertNil(b1col0.array.asAny(0)) + XCTAssertNil(b1col0.array.asAny(1)) + XCTAssertNil(b1col0.array.asAny(2)) + XCTAssertEqual(b1arr0.asString(3), "true") + } + + /// A schema message with no record batches at all. + func testPrimitiveFileWithNoBatches() throws { + let result = try readFile("generated_primitive_no_batches.arrow_file") + PrimitiveCase.assertSchema(result.schema) + XCTAssertEqual(result.batches.count, 0) + } + + /// Record batches that are present but contain no rows. + func testPrimitiveFileWithZeroLengthBatches() throws { + let result = try readFile("generated_primitive_zerolength.arrow_file") + PrimitiveCase.assertSchema(result.schema) + XCTAssertEqual(result.batches.count, 3) + for batch in result.batches { + XCTAssertEqual(batch.length, 0) + XCTAssertEqual(batch.columns.count, PrimitiveCase.fields.count) + } + } +} + +/// Reads IPC streams from the `apache/arrow-testing` corpus. The streaming +/// tests in IPCTests write with ArrowWriter and read the result back, which +/// establishes self-consistency but not conformance; these read streams +/// produced by another Arrow implementation. +final class IPCStreamConformanceTests: XCTestCase { + private func readStream(_ name: String) throws -> ArrowReader.ArrowReaderResult { + let url = try ArrowTestData.url("\(PrimitiveCase.corpusPrefix)/\(name)") + let data = try Data(contentsOf: url) + switch ArrowReader().readStreaming(data) { + case .success(let result): + return result + case .failure(let error): + throw error + } + } + + func testPrimitiveStreamSchemaAndShape() throws { + let result = try readStream("generated_primitive.stream") + PrimitiveCase.assertSchema(result.schema) + XCTAssertEqual(result.batches.count, 2) + XCTAssertEqual(result.batches.reduce(0) { $0 + Int($1.length) }, 37) + XCTAssertEqual(result.batches[0].length, 17) + XCTAssertEqual(result.batches[1].length, 20) + } + + /// A schema message with no record batches at all. + func testPrimitiveStreamWithNoBatches() throws { + let result = try readStream("generated_primitive_no_batches.stream") + PrimitiveCase.assertSchema(result.schema) + XCTAssertEqual(result.batches.count, 0) + } + + /// Record batches that are present but contain no rows. + func testPrimitiveStreamWithZeroLengthBatches() throws { + let result = try readStream("generated_primitive_zerolength.stream") + PrimitiveCase.assertSchema(result.schema) + XCTAssertEqual(result.batches.count, 3) + for batch in result.batches { + XCTAssertEqual(batch.length, 0) + } + } + + /// Disabled: compares stream and file encodings of the same case to ensure + /// both readers produce identical values. Currently fails non-deterministically + /// because ArrowBuffer.createBuffer does not initialize unallocated memory + /// for null buffers, causing out-of-bounds reads on fields with no nulls. + /// See: https://github.com/apache/arrow-swift/issues/NNN (ArrowBuffer) + /// This test should pass once that issue is fixed and will serve as the + /// regression test for it. + /* + func testStreamAndFileAgree() throws { + let streamResult = try readStream("generated_primitive.stream") + + let fileURL = try ArrowTestData.url( + "\(PrimitiveCase.corpusPrefix)/generated_primitive.arrow_file") + guard case .success(let fileResult) = ArrowReader().fromFile(fileURL) else { + XCTFail("could not read generated_primitive.arrow_file") + return + } + + XCTAssertEqual( + PrimitiveCase.snapshot(streamResult), + PrimitiveCase.snapshot(fileResult)) + } + */ +} From e1a1191af84149a4c57889b86824b35354667e8e Mon Sep 17 00:00:00 2001 From: demetrius albuquerque Date: Sat, 25 Jul 2026 20:28:03 +0200 Subject: [PATCH 3/7] ci: Fetch arrow-testing when submodule is absent When running from a git clone (dev or CI), initialize the arrow-testing submodule. When running from an extracted tarball (RC verification), clone the repository at build time instead, since submodules are not present in the archive produced by git archive. Adds git config --global --add safe.directory, since the copied working tree in the Docker build has different ownership than the process running git, which git rejects by default. --- ci/scripts/build.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 53dbb7b..2045a57 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -43,6 +43,24 @@ if [ -d /cache ]; then fi github_actions_group_end +github_actions_group_begin "Initialize arrow-testing submodule or fetch" +pushd "${build_dir}/source" +git config --global --add safe.directory "$(pwd)" +if [ -d .git ]; then + git submodule update --init --depth 1 testing +else + # Running from extracted tarball; submodules are not present in the + # archive (rc.yaml builds it with git archive), so fetch the data + # directly instead. + git clone --depth 1 \ + https://github.com/apache/arrow-testing.git testing-tmp + mkdir -p testing + cp -a testing-tmp/data testing/ + rm -rf testing-tmp +fi +popd +github_actions_group_end + github_actions_group_begin "Generate data" data_gen_dir="${build_dir}/source/data-generator/swift-datagen" if [ -d /cache ]; then From 1b665464e69272cd3675ace1dec2bcaf73a6736c Mon Sep 17 00:00:00 2001 From: demetrius albuquerque Date: Sat, 25 Jul 2026 20:30:25 +0200 Subject: [PATCH 4/7] chore: Ignore generated test data fixtures These files are produced by data-generator/swift-datagen at build time (see ci/scripts/build.sh) and should not appear as untracked noise in git status. --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 65bfb48..7607d6e 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,11 @@ xcuserdata/ # Docker /.docker/ +# Generated test fixtures (see ci/scripts/build.sh) +/Tests/ArrowTests/testdata_*.arrow +/Tests/ArrowTests/testfilewriter_*.arrow +/data-generator/swift-datagen/testdata_*.arrow + # Release Audit Tool /dev/release/apache-rat-*.jar /dev/release/filtered_rat.txt From aee7a5c610c64e5d9aae485673c13495e6d81486 Mon Sep 17 00:00:00 2001 From: demetrius albuquerque Date: Sat, 25 Jul 2026 22:04:02 +0200 Subject: [PATCH 5/7] test: Verify struct and duplicate field names against corpus Reads generated_duplicate_fieldnames.arrow_file from the arrow-testing corpus, which carries a struct column alongside two top-level fields sharing the same name. Neither is covered by the existing conformance tests, which only read flat primitive types. Only schema and shape are asserted. Values are not, because fields with no validity buffer currently read uninitialized memory and are not reproducible between runs. Part of #10 --- Tests/ArrowTests/IPCConformanceTests.swift | 100 ++++++++++++++------- 1 file changed, 69 insertions(+), 31 deletions(-) diff --git a/Tests/ArrowTests/IPCConformanceTests.swift b/Tests/ArrowTests/IPCConformanceTests.swift index 64e2b40..a429247 100644 --- a/Tests/ArrowTests/IPCConformanceTests.swift +++ b/Tests/ArrowTests/IPCConformanceTests.swift @@ -59,24 +59,24 @@ private enum PrimitiveCase { /// used by testStreamAndFileAgree (currently disabled). /// Remove this when the test is re-enabled. /* - private static func snapshot(_ result: ArrowReader.ArrowReaderResult) -> [String] { - var values: [String] = [] - for (batchIndex, batch) in result.batches.enumerated() { - for column in 0.. [String] { + var values: [String] = [] + for (batchIndex, batch) in result.batches.enumerated() { + for column in 0.. Date: Sat, 25 Jul 2026 22:14:02 +0200 Subject: [PATCH 6/7] test: Build test data in Swift instead of generated files testFileWriter_bool and testFileWriter_struct exercise ArrowWriter; the fixture files were only a source of input data. They now build their record batches with the array builders, following the pattern already used by makeRecordBatch and makeStructRecordBatch in the same file. Removes testFileReader_bool, testFileReader_double and testFileReader_struct. Their coverage is now provided by IPCFileConformanceTests, which reads equivalent data produced by another Arrow implementation rather than by the local generator, across more types and more rows. No test depends on data-generator/swift-datagen after this change. Part of #10 --- Tests/ArrowTests/IPCTests.swift | 119 +++++++++++++++----------------- 1 file changed, 57 insertions(+), 62 deletions(-) diff --git a/Tests/ArrowTests/IPCTests.swift b/Tests/ArrowTests/IPCTests.swift index 58f973e..33f7dfd 100644 --- a/Tests/ArrowTests/IPCTests.swift +++ b/Tests/ArrowTests/IPCTests.swift @@ -210,6 +210,51 @@ func makeRecordBatch() throws -> RecordBatch { } } +func makeBoolRecordBatch() throws -> RecordBatch { + let boolBuilder = try ArrowArrayBuilders.loadBoolArrayBuilder() + boolBuilder.append(true) + boolBuilder.append(false) + boolBuilder.append(nil) + boolBuilder.append(false) + boolBuilder.append(true) + let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder() + stringBuilder.append("zero") + stringBuilder.append("one") + stringBuilder.append("two") + stringBuilder.append("three") + stringBuilder.append("four") + let result = RecordBatch.Builder() + .addColumn("one", arrowArray: ArrowArrayHolderImpl(try boolBuilder.finish())) + .addColumn("two", arrowArray: ArrowArrayHolderImpl(try stringBuilder.finish())) + .finish() + switch result { + case .success(let recordBatch): + return recordBatch + case .failure(let error): + throw error + } +} + +func makeTwoFieldStructRecordBatch() throws -> RecordBatch { + let fields = [ + ArrowField("field0", type: ArrowType(ArrowType.ArrowString), isNullable: true), + ArrowField("field1", type: ArrowType(ArrowType.ArrowBool), isNullable: true) + ] + let structBuilder = try ArrowArrayBuilders.loadStructArrayBuilder(fields) + structBuilder.append(["0", false]) + structBuilder.append(["1", true]) + structBuilder.append(nil) + let result = RecordBatch.Builder() + .addColumn("my struct", arrowArray: ArrowArrayHolderImpl(try structBuilder.finish())) + .finish() + switch result { + case .success(let recordBatch): + return recordBatch + case .failure(let error): + throw error + } +} + final class IPCStreamReaderTests: XCTestCase { func testRBInMemoryToFromStream() throws { let schema = makeSchema() @@ -265,62 +310,20 @@ final class IPCStreamReaderTests: XCTestCase { } final class IPCFileReaderTests: XCTestCase { // swiftlint:disable:this type_body_length - func testFileReader_double() throws { - let fileURL = currentDirectory().appendingPathComponent("testdata_double.arrow") - let arrowReader = ArrowReader() - let result = arrowReader.fromFile(fileURL) - let recordBatches: [RecordBatch] - switch result { - case .success(let result): - recordBatches = result.batches - case .failure(let error): - throw error - } - - XCTAssertEqual(recordBatches.count, 1) - for recordBatch in recordBatches { - XCTAssertEqual(recordBatch.length, 5) - XCTAssertEqual(recordBatch.columns.count, 2) - XCTAssertEqual(recordBatch.schema.fields.count, 2) - XCTAssertEqual(recordBatch.schema.fields[0].name, "one") - XCTAssertEqual(recordBatch.schema.fields[0].type.info, ArrowType.ArrowDouble) - XCTAssertEqual(recordBatch.schema.fields[1].name, "two") - XCTAssertEqual(recordBatch.schema.fields[1].type.info, ArrowType.ArrowString) - for index in 0.. Date: Sat, 25 Jul 2026 22:35:23 +0200 Subject: [PATCH 7/7] chore!: Remove the Go test data generator data-generator/swift-datagen produced three Arrow files consumed by the file reader tests. Those tests now either read gold files from the arrow-testing corpus or build their data with the Swift array builders, so the generator has no remaining consumers. Removes the generator, its build step in ci/scripts/build.sh, the Go toolchain from ci/docker/ubuntu.dockerfile, its dependabot entry, the now-obsolete rat_exclude_files.txt rule for go.sum under two levels, the .gitignore rules for the files it produced, and the corresponding section of Sources/Arrow/README.md. swift test now requires only a Swift toolchain and the arrow-testing data, which ci/scripts/build.sh fetches when the submodule is not initialized. Verified end-to-end via docker compose build && docker compose run: 52 tests pass, 0 failures, with no Go toolchain in the image. CDataWGo is unaffected: it tests the C Data Interface against a real Go runtime and cannot be replaced by static test data. Part of #10 --- .github/dependabot.yml | 7 -- .gitignore | 4 +- Sources/Arrow/README.md | 5 -- ci/docker/ubuntu.dockerfile | 6 -- ci/scripts/build.sh | 14 ---- data-generator/swift-datagen/go.mod | 32 -------- data-generator/swift-datagen/go.sum | 36 --------- data-generator/swift-datagen/main.go | 116 --------------------------- dev/release/rat_exclude_files.txt | 1 - 9 files changed, 1 insertion(+), 220 deletions(-) delete mode 100644 data-generator/swift-datagen/go.mod delete mode 100644 data-generator/swift-datagen/go.sum delete mode 100644 data-generator/swift-datagen/main.go diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a582a2c..7128fd9 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -52,10 +52,3 @@ updates: commit-message: prefix: "chore: " open-pull-requests-limit: 10 - - package-ecosystem: "gomod" - directory: "/data-generator/swift-datagen/" - schedule: - interval: "daily" - commit-message: - prefix: "chore: " - open-pull-requests-limit: 10 diff --git a/.gitignore b/.gitignore index 7607d6e..62467d7 100644 --- a/.gitignore +++ b/.gitignore @@ -27,10 +27,8 @@ xcuserdata/ # Docker /.docker/ -# Generated test fixtures (see ci/scripts/build.sh) -/Tests/ArrowTests/testdata_*.arrow +# Written by the file writer tests /Tests/ArrowTests/testfilewriter_*.arrow -/data-generator/swift-datagen/testdata_*.arrow # Release Audit Tool /dev/release/apache-rat-*.jar diff --git a/Sources/Arrow/README.md b/Sources/Arrow/README.md index 3acded8..7e68955 100644 --- a/Sources/Arrow/README.md +++ b/Sources/Arrow/README.md @@ -48,9 +48,4 @@ An implementation of Arrow targeting Swift. - Fields - Schema -## Test data generation -Test data files for the reader tests are generated by an executable built in go whose source is included in the data-generator directory. -```sh -$ go build -o swift-datagen -``` diff --git a/ci/docker/ubuntu.dockerfile b/ci/docker/ubuntu.dockerfile index f7a0910..3929851 100644 --- a/ci/docker/ubuntu.dockerfile +++ b/ci/docker/ubuntu.dockerfile @@ -18,9 +18,3 @@ ARG SWIFT=5.10 ARG UBUNTU_CODE_NAME=noble FROM swift:${SWIFT}-${UBUNTU_CODE_NAME} - -# Go is needed for generating test data -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - golang-go && \ - apt-get clean diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 2045a57..d56129e 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -61,20 +61,6 @@ fi popd github_actions_group_end -github_actions_group_begin "Generate data" -data_gen_dir="${build_dir}/source/data-generator/swift-datagen" -if [ -d /cache ]; then - export GOCACHE="/cache/go-build" - export GOMODCACHE="/cache/go-mod" -fi -export GOPATH="${build_dir}" -pushd "${data_gen_dir}" -go get -d ./... -go run . -cp *.arrow ../../Tests/ArrowTests -popd -github_actions_group_end - github_actions_group_begin "Use -warnings-as-errors" pushd "${build_dir}/source/" sed 's/\/\/ build://g' Package.swift > Package.swift.build diff --git a/data-generator/swift-datagen/go.mod b/data-generator/swift-datagen/go.mod deleted file mode 100644 index 8c17e33..0000000 --- a/data-generator/swift-datagen/go.mod +++ /dev/null @@ -1,32 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -module swift-datagen/main - -go 1.25.0 - -require github.com/apache/arrow-go/v18 v18.7.0 - -require ( - github.com/goccy/go-json v0.10.6 // indirect - github.com/google/flatbuffers v25.12.19+incompatible // indirect - github.com/klauspost/compress v1.19.0 // indirect - github.com/klauspost/cpuid/v2 v2.4.0 // indirect - github.com/pierrec/lz4/v4 v4.1.27 // indirect - github.com/zeebo/xxh3 v1.1.0 // indirect - golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect - golang.org/x/sys v0.47.0 // indirect -) diff --git a/data-generator/swift-datagen/go.sum b/data-generator/swift-datagen/go.sum deleted file mode 100644 index 65f5fda..0000000 --- a/data-generator/swift-datagen/go.sum +++ /dev/null @@ -1,36 +0,0 @@ -github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= -github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/apache/arrow-go/v18 v18.7.0 h1:Vw/i+cJyebUofT7JlqFpe65LrmwxULn166jjwStM4HY= -github.com/apache/arrow-go/v18 v18.7.0/go.mod h1:PM6IigLJkdMwIpeHXnymo+xZ52f42a9EYiLtRel4p/A= -github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ= -github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= -github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= -github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= -github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= -github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= -github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk= -github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= -github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= -github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= -github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= -golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU= -golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= -gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/data-generator/swift-datagen/main.go b/data-generator/swift-datagen/main.go deleted file mode 100644 index dbe2422..0000000 --- a/data-generator/swift-datagen/main.go +++ /dev/null @@ -1,116 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "log" - "os" - - "github.com/apache/arrow-go/v18/arrow" - "github.com/apache/arrow-go/v18/arrow/array" - "github.com/apache/arrow-go/v18/arrow/ipc" - "github.com/apache/arrow-go/v18/arrow/memory" -) - -func writeBytes(rec arrow.Record, file_name string) { - file, err := os.Create(file_name) - defer file.Close() - if err != nil { - log.Fatal(err) - } - - rr, write_err := ipc.NewFileWriter(file, ipc.WithSchema(rec.Schema())) - if write_err != nil { - log.Fatal(write_err) - } - - rr.Write(rec) - rr.Close() -} - -func writeBoolData() { - alloc := memory.NewGoAllocator() - schema := arrow.NewSchema([]arrow.Field{ - {Name: "one", Type: arrow.FixedWidthTypes.Boolean}, - {Name: "two", Type: arrow.BinaryTypes.String}, - }, nil) - - b := array.NewRecordBuilder(alloc, schema) - defer b.Release() - - b.Field(0).(*array.BooleanBuilder).AppendValues([]bool{true, false}, nil) - b.Field(0).(*array.BooleanBuilder).AppendNull() - b.Field(0).(*array.BooleanBuilder).AppendValues([]bool{false, true}, nil) - b.Field(1).(*array.StringBuilder).AppendValues([]string{"zero", "one", "two", "three", "four"}, nil) - rec := b.NewRecord() - defer rec.Release() - - writeBytes(rec, "testdata_bool.arrow") -} - -func writeDoubleData() { - alloc := memory.NewGoAllocator() - schema := arrow.NewSchema([]arrow.Field{ - {Name: "one", Type: arrow.PrimitiveTypes.Float64}, - {Name: "two", Type: arrow.BinaryTypes.String}, - }, nil) - - b := array.NewRecordBuilder(alloc, schema) - defer b.Release() - - b.Field(0).(*array.Float64Builder).AppendValues([]float64{1.1, 2.2, 3.3, 4.4, 5.5}, nil) - b.Field(1).(*array.StringBuilder).AppendValues([]string{"zero"}, nil) - b.Field(1).(*array.StringBuilder).AppendNull() - b.Field(1).(*array.StringBuilder).AppendValues([]string{"two", "three", "four"}, nil) - rec := b.NewRecord() - defer rec.Release() - - writeBytes(rec, "testdata_double.arrow") -} - -func writeStructData() { - mem := memory.NewGoAllocator() - - fields := []arrow.Field{ - {Name: "my struct", Type: arrow.StructOf([]arrow.Field{ - {Name: "my string", Type: arrow.BinaryTypes.String}, - {Name: "my bool", Type: arrow.FixedWidthTypes.Boolean}, - }...)}, - } - - schema := arrow.NewSchema(fields, nil) - - bld := array.NewRecordBuilder(mem, schema) - defer bld.Release() - - sb := bld.Field(0).(*array.StructBuilder) - f1b := sb.FieldBuilder(0).(*array.StringBuilder) - f2b := sb.FieldBuilder(1).(*array.BooleanBuilder) - - sb.AppendValues([]bool{true, true, false}) - f1b.AppendValues([]string{"0", "1", ""}, []bool{true, true, false}) - f2b.AppendValues([]bool{false, true, false}, []bool{true, true, false}) - - rec := bld.NewRecord() - writeBytes(rec, "testdata_struct.arrow") -} - -func main() { - writeBoolData() - writeDoubleData() - writeStructData() -} diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 9ebde90..bcbfa6a 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -15,7 +15,6 @@ # specific language governing permissions and limitations # under the License. -*/*/go.sum */Package.resolved */go.sum .github/pull_request_template.md