diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4e9e3e7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,29 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + build-and-test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Setup .NET 10 + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + dotnet-quality: 'preview' + + - name: Restore + run: dotnet restore + + - name: Build + run: dotnet build --no-restore --configuration Release + + - name: Test + run: dotnet test --no-build --configuration Release --verbosity normal diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..e427d95 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,44 @@ +name: Publish NuGet Packages + +on: + push: + tags: ['v*'] + workflow_dispatch: + +jobs: + publish: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Setup .NET 10 + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + dotnet-quality: 'preview' + + - name: Compute version + id: version + run: | + YEAR=$(date -u +%Y) + MONTH=$(date -u +%-m) + DAY=$(date -u +%-d) + VERSION="${YEAR}.${MONTH}.${DAY}.${GITHUB_RUN_NUMBER}" + echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + echo "Computed version: ${VERSION}" + + - name: Restore + run: dotnet restore + + - name: Build + run: dotnet build --no-restore --configuration Release /p:Version=${{ steps.version.outputs.version }} + + - name: Test + run: dotnet test --no-build --configuration Release --verbosity normal + + - name: Pack + run: dotnet pack --no-build --configuration Release /p:Version=${{ steps.version.outputs.version }} --output ./nupkgs + + - name: Push to NuGet + run: dotnet nuget push ./nupkgs/*.nupkg --api-key ${{ secrets.NUGET_API_KEY }} --source https://api.nuget.org/v3/index.json --skip-duplicate diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d177d49 --- /dev/null +++ b/.gitignore @@ -0,0 +1,41 @@ +# .NET build +bin/ +obj/ +*.csproj.user +*.rsuser +*.suo +*.user +.vs/ + +# NuGet +*.nupkg +*.snupkg + +# C++ (cross-language) +cross-language/cpp/build/ + +# Node.js (cross-language) +cross-language/typescript/node_modules/ + +# Python +__pycache__/ +*.pyc +.venv/ +venv/ + +# Test results & coverage +TestResults/ +coverage/ + +# Publish output +**/publish/ + +# OS +.DS_Store +Thumbs.db + +# BenchmarkDotNet +BenchmarkDotNet.Artifacts/ + +# JetBrains +.idea/ diff --git a/Directory.Build.props b/Directory.Build.props new file mode 100644 index 0000000..29b194e --- /dev/null +++ b/Directory.Build.props @@ -0,0 +1,21 @@ + + + + 0.0.0-local + Daniel Bunting + Daniel Bunting + Apache-2.0 + README.md + https://github.com/DanielBunting/Levels + git + https://github.com/DanielBunting/Levels + Levels - a multi-level time-series database for financial market data + timeseries;database;financial;market-data;orderbook;ohlcv + Copyright (c) Daniel Bunting + + + + + + + diff --git a/Levels.sln b/Levels.sln new file mode 100644 index 0000000..347c22b --- /dev/null +++ b/Levels.sln @@ -0,0 +1,328 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Levels.Core", "src\Levels.Core\Levels.Core.csproj", "{9752F8AC-BA53-430D-AB8E-CC5217356C4F}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{0AB3BF05-4346-4AA6-1389-037BE0695223}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Levels.Tests", "tests\Levels.Tests\Levels.Tests.csproj", "{4EEFF4A6-6AF6-4D84-8333-8A9AF4263113}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Levels.Sinks", "src\Levels.Sinks\Levels.Sinks.csproj", "{1A03D818-3FA3-430D-8AAA-382DD36D45F7}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Levels.DataFlow", "src\Levels.DataFlow\Levels.DataFlow.csproj", "{52C67ED5-242A-4758-B529-14E0B8B84F5D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Levels.Compaction", "src\Levels.Compaction\Levels.Compaction.csproj", "{1EA6F64E-7AD4-41EE-BBA9-4F0868294A6A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Levels.Query", "src\Levels.Query\Levels.Query.csproj", "{58BCA5C9-578F-4FCC-9161-FF890E0EF9CD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Levels.Period", "src\Levels.Period\Levels.Period.csproj", "{18555F26-9DE7-4842-AAFB-8614061EB1E5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Levels.Resampled", "src\Levels.Resampled\Levels.Resampled.csproj", "{A1B2C3D4-E5F6-7890-ABCD-EF1234567890}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Levels.SourceGen", "src\Levels.SourceGen\Levels.SourceGen.csproj", "{B2C3D4E5-F6A7-8901-BCDE-F12345678901}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Levels.Benchmarks", "tests\Levels.Benchmarks\Levels.Benchmarks.csproj", "{C3D4E5F6-A7B8-9012-CDEF-123456789012}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Levels.Hosting", "src\Levels.Hosting\Levels.Hosting.csproj", "{F4BBAFE0-69B9-4890-9AC0-6D311137B6EC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Levels.Cli", "src\Levels.Cli\Levels.Cli.csproj", "{081623C1-3E9E-4276-ABC2-6C7C58E5DDF5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Levels.Export", "src\Levels.Export\Levels.Export.csproj", "{0481DFA3-4AF9-4421-93E0-A3E3083FF003}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Levels.Hints", "src\Levels.Hints\Levels.Hints.csproj", "{05640B40-FB77-49E2-8741-D29F7ED13263}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Levels.Web", "src\Levels.Web\Levels.Web.csproj", "{5EF30C04-1383-44D8-A73A-E8D5FD67218D}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "samples", "samples", "{5D20AA90-6969-D8BD-9DCD-8634F4692FDA}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CryptoExchangeSample", "samples\CryptoExchangeSample\CryptoExchangeSample.csproj", "{4C888035-8687-4DE4-A192-FBE0BA7375F7}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BinanceLiveSample", "samples\BinanceLiveSample\BinanceLiveSample.csproj", "{24DB7B77-A721-4DC4-926F-D23606FB65A3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Levels.Server", "src\Levels.Server\Levels.Server.csproj", "{0CF04D7C-C568-4539-ABCB-D1B54108D7D5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Levels.Client", "src\Levels.Client\Levels.Client.csproj", "{07217170-E7D4-4DA4-AB08-BE52AA1D9174}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Levels.Protocol", "src\Levels.Protocol\Levels.Protocol.csproj", "{7EA25779-E02F-452C-AB8A-F5C5DCE972B2}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {9752F8AC-BA53-430D-AB8E-CC5217356C4F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9752F8AC-BA53-430D-AB8E-CC5217356C4F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9752F8AC-BA53-430D-AB8E-CC5217356C4F}.Debug|x64.ActiveCfg = Debug|Any CPU + {9752F8AC-BA53-430D-AB8E-CC5217356C4F}.Debug|x64.Build.0 = Debug|Any CPU + {9752F8AC-BA53-430D-AB8E-CC5217356C4F}.Debug|x86.ActiveCfg = Debug|Any CPU + {9752F8AC-BA53-430D-AB8E-CC5217356C4F}.Debug|x86.Build.0 = Debug|Any CPU + {9752F8AC-BA53-430D-AB8E-CC5217356C4F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9752F8AC-BA53-430D-AB8E-CC5217356C4F}.Release|Any CPU.Build.0 = Release|Any CPU + {9752F8AC-BA53-430D-AB8E-CC5217356C4F}.Release|x64.ActiveCfg = Release|Any CPU + {9752F8AC-BA53-430D-AB8E-CC5217356C4F}.Release|x64.Build.0 = Release|Any CPU + {9752F8AC-BA53-430D-AB8E-CC5217356C4F}.Release|x86.ActiveCfg = Release|Any CPU + {9752F8AC-BA53-430D-AB8E-CC5217356C4F}.Release|x86.Build.0 = Release|Any CPU + {4EEFF4A6-6AF6-4D84-8333-8A9AF4263113}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4EEFF4A6-6AF6-4D84-8333-8A9AF4263113}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4EEFF4A6-6AF6-4D84-8333-8A9AF4263113}.Debug|x64.ActiveCfg = Debug|Any CPU + {4EEFF4A6-6AF6-4D84-8333-8A9AF4263113}.Debug|x64.Build.0 = Debug|Any CPU + {4EEFF4A6-6AF6-4D84-8333-8A9AF4263113}.Debug|x86.ActiveCfg = Debug|Any CPU + {4EEFF4A6-6AF6-4D84-8333-8A9AF4263113}.Debug|x86.Build.0 = Debug|Any CPU + {4EEFF4A6-6AF6-4D84-8333-8A9AF4263113}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4EEFF4A6-6AF6-4D84-8333-8A9AF4263113}.Release|Any CPU.Build.0 = Release|Any CPU + {4EEFF4A6-6AF6-4D84-8333-8A9AF4263113}.Release|x64.ActiveCfg = Release|Any CPU + {4EEFF4A6-6AF6-4D84-8333-8A9AF4263113}.Release|x64.Build.0 = Release|Any CPU + {4EEFF4A6-6AF6-4D84-8333-8A9AF4263113}.Release|x86.ActiveCfg = Release|Any CPU + {4EEFF4A6-6AF6-4D84-8333-8A9AF4263113}.Release|x86.Build.0 = Release|Any CPU + {1A03D818-3FA3-430D-8AAA-382DD36D45F7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1A03D818-3FA3-430D-8AAA-382DD36D45F7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1A03D818-3FA3-430D-8AAA-382DD36D45F7}.Debug|x64.ActiveCfg = Debug|Any CPU + {1A03D818-3FA3-430D-8AAA-382DD36D45F7}.Debug|x64.Build.0 = Debug|Any CPU + {1A03D818-3FA3-430D-8AAA-382DD36D45F7}.Debug|x86.ActiveCfg = Debug|Any CPU + {1A03D818-3FA3-430D-8AAA-382DD36D45F7}.Debug|x86.Build.0 = Debug|Any CPU + {1A03D818-3FA3-430D-8AAA-382DD36D45F7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1A03D818-3FA3-430D-8AAA-382DD36D45F7}.Release|Any CPU.Build.0 = Release|Any CPU + {1A03D818-3FA3-430D-8AAA-382DD36D45F7}.Release|x64.ActiveCfg = Release|Any CPU + {1A03D818-3FA3-430D-8AAA-382DD36D45F7}.Release|x64.Build.0 = Release|Any CPU + {1A03D818-3FA3-430D-8AAA-382DD36D45F7}.Release|x86.ActiveCfg = Release|Any CPU + {1A03D818-3FA3-430D-8AAA-382DD36D45F7}.Release|x86.Build.0 = Release|Any CPU + {52C67ED5-242A-4758-B529-14E0B8B84F5D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {52C67ED5-242A-4758-B529-14E0B8B84F5D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {52C67ED5-242A-4758-B529-14E0B8B84F5D}.Debug|x64.ActiveCfg = Debug|Any CPU + {52C67ED5-242A-4758-B529-14E0B8B84F5D}.Debug|x64.Build.0 = Debug|Any CPU + {52C67ED5-242A-4758-B529-14E0B8B84F5D}.Debug|x86.ActiveCfg = Debug|Any CPU + {52C67ED5-242A-4758-B529-14E0B8B84F5D}.Debug|x86.Build.0 = Debug|Any CPU + {52C67ED5-242A-4758-B529-14E0B8B84F5D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {52C67ED5-242A-4758-B529-14E0B8B84F5D}.Release|Any CPU.Build.0 = Release|Any CPU + {52C67ED5-242A-4758-B529-14E0B8B84F5D}.Release|x64.ActiveCfg = Release|Any CPU + {52C67ED5-242A-4758-B529-14E0B8B84F5D}.Release|x64.Build.0 = Release|Any CPU + {52C67ED5-242A-4758-B529-14E0B8B84F5D}.Release|x86.ActiveCfg = Release|Any CPU + {52C67ED5-242A-4758-B529-14E0B8B84F5D}.Release|x86.Build.0 = Release|Any CPU + {1EA6F64E-7AD4-41EE-BBA9-4F0868294A6A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1EA6F64E-7AD4-41EE-BBA9-4F0868294A6A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1EA6F64E-7AD4-41EE-BBA9-4F0868294A6A}.Debug|x64.ActiveCfg = Debug|Any CPU + {1EA6F64E-7AD4-41EE-BBA9-4F0868294A6A}.Debug|x64.Build.0 = Debug|Any CPU + {1EA6F64E-7AD4-41EE-BBA9-4F0868294A6A}.Debug|x86.ActiveCfg = Debug|Any CPU + {1EA6F64E-7AD4-41EE-BBA9-4F0868294A6A}.Debug|x86.Build.0 = Debug|Any CPU + {1EA6F64E-7AD4-41EE-BBA9-4F0868294A6A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1EA6F64E-7AD4-41EE-BBA9-4F0868294A6A}.Release|Any CPU.Build.0 = Release|Any CPU + {1EA6F64E-7AD4-41EE-BBA9-4F0868294A6A}.Release|x64.ActiveCfg = Release|Any CPU + {1EA6F64E-7AD4-41EE-BBA9-4F0868294A6A}.Release|x64.Build.0 = Release|Any CPU + {1EA6F64E-7AD4-41EE-BBA9-4F0868294A6A}.Release|x86.ActiveCfg = Release|Any CPU + {1EA6F64E-7AD4-41EE-BBA9-4F0868294A6A}.Release|x86.Build.0 = Release|Any CPU + {58BCA5C9-578F-4FCC-9161-FF890E0EF9CD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {58BCA5C9-578F-4FCC-9161-FF890E0EF9CD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {58BCA5C9-578F-4FCC-9161-FF890E0EF9CD}.Debug|x64.ActiveCfg = Debug|Any CPU + {58BCA5C9-578F-4FCC-9161-FF890E0EF9CD}.Debug|x64.Build.0 = Debug|Any CPU + {58BCA5C9-578F-4FCC-9161-FF890E0EF9CD}.Debug|x86.ActiveCfg = Debug|Any CPU + {58BCA5C9-578F-4FCC-9161-FF890E0EF9CD}.Debug|x86.Build.0 = Debug|Any CPU + {58BCA5C9-578F-4FCC-9161-FF890E0EF9CD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {58BCA5C9-578F-4FCC-9161-FF890E0EF9CD}.Release|Any CPU.Build.0 = Release|Any CPU + {58BCA5C9-578F-4FCC-9161-FF890E0EF9CD}.Release|x64.ActiveCfg = Release|Any CPU + {58BCA5C9-578F-4FCC-9161-FF890E0EF9CD}.Release|x64.Build.0 = Release|Any CPU + {58BCA5C9-578F-4FCC-9161-FF890E0EF9CD}.Release|x86.ActiveCfg = Release|Any CPU + {58BCA5C9-578F-4FCC-9161-FF890E0EF9CD}.Release|x86.Build.0 = Release|Any CPU + {18555F26-9DE7-4842-AAFB-8614061EB1E5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {18555F26-9DE7-4842-AAFB-8614061EB1E5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {18555F26-9DE7-4842-AAFB-8614061EB1E5}.Debug|x64.ActiveCfg = Debug|Any CPU + {18555F26-9DE7-4842-AAFB-8614061EB1E5}.Debug|x64.Build.0 = Debug|Any CPU + {18555F26-9DE7-4842-AAFB-8614061EB1E5}.Debug|x86.ActiveCfg = Debug|Any CPU + {18555F26-9DE7-4842-AAFB-8614061EB1E5}.Debug|x86.Build.0 = Debug|Any CPU + {18555F26-9DE7-4842-AAFB-8614061EB1E5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {18555F26-9DE7-4842-AAFB-8614061EB1E5}.Release|Any CPU.Build.0 = Release|Any CPU + {18555F26-9DE7-4842-AAFB-8614061EB1E5}.Release|x64.ActiveCfg = Release|Any CPU + {18555F26-9DE7-4842-AAFB-8614061EB1E5}.Release|x64.Build.0 = Release|Any CPU + {18555F26-9DE7-4842-AAFB-8614061EB1E5}.Release|x86.ActiveCfg = Release|Any CPU + {18555F26-9DE7-4842-AAFB-8614061EB1E5}.Release|x86.Build.0 = Release|Any CPU + {A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Debug|x64.ActiveCfg = Debug|Any CPU + {A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Debug|x64.Build.0 = Debug|Any CPU + {A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Debug|x86.ActiveCfg = Debug|Any CPU + {A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Debug|x86.Build.0 = Debug|Any CPU + {A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Release|Any CPU.Build.0 = Release|Any CPU + {A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Release|x64.ActiveCfg = Release|Any CPU + {A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Release|x64.Build.0 = Release|Any CPU + {A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Release|x86.ActiveCfg = Release|Any CPU + {A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Release|x86.Build.0 = Release|Any CPU + {B2C3D4E5-F6A7-8901-BCDE-F12345678901}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B2C3D4E5-F6A7-8901-BCDE-F12345678901}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B2C3D4E5-F6A7-8901-BCDE-F12345678901}.Debug|x64.ActiveCfg = Debug|Any CPU + {B2C3D4E5-F6A7-8901-BCDE-F12345678901}.Debug|x64.Build.0 = Debug|Any CPU + {B2C3D4E5-F6A7-8901-BCDE-F12345678901}.Debug|x86.ActiveCfg = Debug|Any CPU + {B2C3D4E5-F6A7-8901-BCDE-F12345678901}.Debug|x86.Build.0 = Debug|Any CPU + {B2C3D4E5-F6A7-8901-BCDE-F12345678901}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B2C3D4E5-F6A7-8901-BCDE-F12345678901}.Release|Any CPU.Build.0 = Release|Any CPU + {B2C3D4E5-F6A7-8901-BCDE-F12345678901}.Release|x64.ActiveCfg = Release|Any CPU + {B2C3D4E5-F6A7-8901-BCDE-F12345678901}.Release|x64.Build.0 = Release|Any CPU + {B2C3D4E5-F6A7-8901-BCDE-F12345678901}.Release|x86.ActiveCfg = Release|Any CPU + {B2C3D4E5-F6A7-8901-BCDE-F12345678901}.Release|x86.Build.0 = Release|Any CPU + {C3D4E5F6-A7B8-9012-CDEF-123456789012}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C3D4E5F6-A7B8-9012-CDEF-123456789012}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C3D4E5F6-A7B8-9012-CDEF-123456789012}.Debug|x64.ActiveCfg = Debug|Any CPU + {C3D4E5F6-A7B8-9012-CDEF-123456789012}.Debug|x64.Build.0 = Debug|Any CPU + {C3D4E5F6-A7B8-9012-CDEF-123456789012}.Debug|x86.ActiveCfg = Debug|Any CPU + {C3D4E5F6-A7B8-9012-CDEF-123456789012}.Debug|x86.Build.0 = Debug|Any CPU + {C3D4E5F6-A7B8-9012-CDEF-123456789012}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C3D4E5F6-A7B8-9012-CDEF-123456789012}.Release|Any CPU.Build.0 = Release|Any CPU + {C3D4E5F6-A7B8-9012-CDEF-123456789012}.Release|x64.ActiveCfg = Release|Any CPU + {C3D4E5F6-A7B8-9012-CDEF-123456789012}.Release|x64.Build.0 = Release|Any CPU + {C3D4E5F6-A7B8-9012-CDEF-123456789012}.Release|x86.ActiveCfg = Release|Any CPU + {C3D4E5F6-A7B8-9012-CDEF-123456789012}.Release|x86.Build.0 = Release|Any CPU + {F4BBAFE0-69B9-4890-9AC0-6D311137B6EC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F4BBAFE0-69B9-4890-9AC0-6D311137B6EC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F4BBAFE0-69B9-4890-9AC0-6D311137B6EC}.Debug|x64.ActiveCfg = Debug|Any CPU + {F4BBAFE0-69B9-4890-9AC0-6D311137B6EC}.Debug|x64.Build.0 = Debug|Any CPU + {F4BBAFE0-69B9-4890-9AC0-6D311137B6EC}.Debug|x86.ActiveCfg = Debug|Any CPU + {F4BBAFE0-69B9-4890-9AC0-6D311137B6EC}.Debug|x86.Build.0 = Debug|Any CPU + {F4BBAFE0-69B9-4890-9AC0-6D311137B6EC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F4BBAFE0-69B9-4890-9AC0-6D311137B6EC}.Release|Any CPU.Build.0 = Release|Any CPU + {F4BBAFE0-69B9-4890-9AC0-6D311137B6EC}.Release|x64.ActiveCfg = Release|Any CPU + {F4BBAFE0-69B9-4890-9AC0-6D311137B6EC}.Release|x64.Build.0 = Release|Any CPU + {F4BBAFE0-69B9-4890-9AC0-6D311137B6EC}.Release|x86.ActiveCfg = Release|Any CPU + {F4BBAFE0-69B9-4890-9AC0-6D311137B6EC}.Release|x86.Build.0 = Release|Any CPU + {081623C1-3E9E-4276-ABC2-6C7C58E5DDF5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {081623C1-3E9E-4276-ABC2-6C7C58E5DDF5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {081623C1-3E9E-4276-ABC2-6C7C58E5DDF5}.Debug|x64.ActiveCfg = Debug|Any CPU + {081623C1-3E9E-4276-ABC2-6C7C58E5DDF5}.Debug|x64.Build.0 = Debug|Any CPU + {081623C1-3E9E-4276-ABC2-6C7C58E5DDF5}.Debug|x86.ActiveCfg = Debug|Any CPU + {081623C1-3E9E-4276-ABC2-6C7C58E5DDF5}.Debug|x86.Build.0 = Debug|Any CPU + {081623C1-3E9E-4276-ABC2-6C7C58E5DDF5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {081623C1-3E9E-4276-ABC2-6C7C58E5DDF5}.Release|Any CPU.Build.0 = Release|Any CPU + {081623C1-3E9E-4276-ABC2-6C7C58E5DDF5}.Release|x64.ActiveCfg = Release|Any CPU + {081623C1-3E9E-4276-ABC2-6C7C58E5DDF5}.Release|x64.Build.0 = Release|Any CPU + {081623C1-3E9E-4276-ABC2-6C7C58E5DDF5}.Release|x86.ActiveCfg = Release|Any CPU + {081623C1-3E9E-4276-ABC2-6C7C58E5DDF5}.Release|x86.Build.0 = Release|Any CPU + {0481DFA3-4AF9-4421-93E0-A3E3083FF003}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0481DFA3-4AF9-4421-93E0-A3E3083FF003}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0481DFA3-4AF9-4421-93E0-A3E3083FF003}.Debug|x64.ActiveCfg = Debug|Any CPU + {0481DFA3-4AF9-4421-93E0-A3E3083FF003}.Debug|x64.Build.0 = Debug|Any CPU + {0481DFA3-4AF9-4421-93E0-A3E3083FF003}.Debug|x86.ActiveCfg = Debug|Any CPU + {0481DFA3-4AF9-4421-93E0-A3E3083FF003}.Debug|x86.Build.0 = Debug|Any CPU + {0481DFA3-4AF9-4421-93E0-A3E3083FF003}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0481DFA3-4AF9-4421-93E0-A3E3083FF003}.Release|Any CPU.Build.0 = Release|Any CPU + {0481DFA3-4AF9-4421-93E0-A3E3083FF003}.Release|x64.ActiveCfg = Release|Any CPU + {0481DFA3-4AF9-4421-93E0-A3E3083FF003}.Release|x64.Build.0 = Release|Any CPU + {0481DFA3-4AF9-4421-93E0-A3E3083FF003}.Release|x86.ActiveCfg = Release|Any CPU + {0481DFA3-4AF9-4421-93E0-A3E3083FF003}.Release|x86.Build.0 = Release|Any CPU + {05640B40-FB77-49E2-8741-D29F7ED13263}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {05640B40-FB77-49E2-8741-D29F7ED13263}.Debug|Any CPU.Build.0 = Debug|Any CPU + {05640B40-FB77-49E2-8741-D29F7ED13263}.Debug|x64.ActiveCfg = Debug|Any CPU + {05640B40-FB77-49E2-8741-D29F7ED13263}.Debug|x64.Build.0 = Debug|Any CPU + {05640B40-FB77-49E2-8741-D29F7ED13263}.Debug|x86.ActiveCfg = Debug|Any CPU + {05640B40-FB77-49E2-8741-D29F7ED13263}.Debug|x86.Build.0 = Debug|Any CPU + {05640B40-FB77-49E2-8741-D29F7ED13263}.Release|Any CPU.ActiveCfg = Release|Any CPU + {05640B40-FB77-49E2-8741-D29F7ED13263}.Release|Any CPU.Build.0 = Release|Any CPU + {05640B40-FB77-49E2-8741-D29F7ED13263}.Release|x64.ActiveCfg = Release|Any CPU + {05640B40-FB77-49E2-8741-D29F7ED13263}.Release|x64.Build.0 = Release|Any CPU + {05640B40-FB77-49E2-8741-D29F7ED13263}.Release|x86.ActiveCfg = Release|Any CPU + {05640B40-FB77-49E2-8741-D29F7ED13263}.Release|x86.Build.0 = Release|Any CPU + {5EF30C04-1383-44D8-A73A-E8D5FD67218D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5EF30C04-1383-44D8-A73A-E8D5FD67218D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5EF30C04-1383-44D8-A73A-E8D5FD67218D}.Debug|x64.ActiveCfg = Debug|Any CPU + {5EF30C04-1383-44D8-A73A-E8D5FD67218D}.Debug|x64.Build.0 = Debug|Any CPU + {5EF30C04-1383-44D8-A73A-E8D5FD67218D}.Debug|x86.ActiveCfg = Debug|Any CPU + {5EF30C04-1383-44D8-A73A-E8D5FD67218D}.Debug|x86.Build.0 = Debug|Any CPU + {5EF30C04-1383-44D8-A73A-E8D5FD67218D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5EF30C04-1383-44D8-A73A-E8D5FD67218D}.Release|Any CPU.Build.0 = Release|Any CPU + {5EF30C04-1383-44D8-A73A-E8D5FD67218D}.Release|x64.ActiveCfg = Release|Any CPU + {5EF30C04-1383-44D8-A73A-E8D5FD67218D}.Release|x64.Build.0 = Release|Any CPU + {5EF30C04-1383-44D8-A73A-E8D5FD67218D}.Release|x86.ActiveCfg = Release|Any CPU + {5EF30C04-1383-44D8-A73A-E8D5FD67218D}.Release|x86.Build.0 = Release|Any CPU + {4C888035-8687-4DE4-A192-FBE0BA7375F7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4C888035-8687-4DE4-A192-FBE0BA7375F7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4C888035-8687-4DE4-A192-FBE0BA7375F7}.Debug|x64.ActiveCfg = Debug|Any CPU + {4C888035-8687-4DE4-A192-FBE0BA7375F7}.Debug|x64.Build.0 = Debug|Any CPU + {4C888035-8687-4DE4-A192-FBE0BA7375F7}.Debug|x86.ActiveCfg = Debug|Any CPU + {4C888035-8687-4DE4-A192-FBE0BA7375F7}.Debug|x86.Build.0 = Debug|Any CPU + {4C888035-8687-4DE4-A192-FBE0BA7375F7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4C888035-8687-4DE4-A192-FBE0BA7375F7}.Release|Any CPU.Build.0 = Release|Any CPU + {4C888035-8687-4DE4-A192-FBE0BA7375F7}.Release|x64.ActiveCfg = Release|Any CPU + {4C888035-8687-4DE4-A192-FBE0BA7375F7}.Release|x64.Build.0 = Release|Any CPU + {4C888035-8687-4DE4-A192-FBE0BA7375F7}.Release|x86.ActiveCfg = Release|Any CPU + {4C888035-8687-4DE4-A192-FBE0BA7375F7}.Release|x86.Build.0 = Release|Any CPU + {24DB7B77-A721-4DC4-926F-D23606FB65A3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {24DB7B77-A721-4DC4-926F-D23606FB65A3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {24DB7B77-A721-4DC4-926F-D23606FB65A3}.Debug|x64.ActiveCfg = Debug|Any CPU + {24DB7B77-A721-4DC4-926F-D23606FB65A3}.Debug|x64.Build.0 = Debug|Any CPU + {24DB7B77-A721-4DC4-926F-D23606FB65A3}.Debug|x86.ActiveCfg = Debug|Any CPU + {24DB7B77-A721-4DC4-926F-D23606FB65A3}.Debug|x86.Build.0 = Debug|Any CPU + {24DB7B77-A721-4DC4-926F-D23606FB65A3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {24DB7B77-A721-4DC4-926F-D23606FB65A3}.Release|Any CPU.Build.0 = Release|Any CPU + {24DB7B77-A721-4DC4-926F-D23606FB65A3}.Release|x64.ActiveCfg = Release|Any CPU + {24DB7B77-A721-4DC4-926F-D23606FB65A3}.Release|x64.Build.0 = Release|Any CPU + {24DB7B77-A721-4DC4-926F-D23606FB65A3}.Release|x86.ActiveCfg = Release|Any CPU + {24DB7B77-A721-4DC4-926F-D23606FB65A3}.Release|x86.Build.0 = Release|Any CPU + {0CF04D7C-C568-4539-ABCB-D1B54108D7D5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0CF04D7C-C568-4539-ABCB-D1B54108D7D5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0CF04D7C-C568-4539-ABCB-D1B54108D7D5}.Debug|x64.ActiveCfg = Debug|Any CPU + {0CF04D7C-C568-4539-ABCB-D1B54108D7D5}.Debug|x64.Build.0 = Debug|Any CPU + {0CF04D7C-C568-4539-ABCB-D1B54108D7D5}.Debug|x86.ActiveCfg = Debug|Any CPU + {0CF04D7C-C568-4539-ABCB-D1B54108D7D5}.Debug|x86.Build.0 = Debug|Any CPU + {0CF04D7C-C568-4539-ABCB-D1B54108D7D5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0CF04D7C-C568-4539-ABCB-D1B54108D7D5}.Release|Any CPU.Build.0 = Release|Any CPU + {0CF04D7C-C568-4539-ABCB-D1B54108D7D5}.Release|x64.ActiveCfg = Release|Any CPU + {0CF04D7C-C568-4539-ABCB-D1B54108D7D5}.Release|x64.Build.0 = Release|Any CPU + {0CF04D7C-C568-4539-ABCB-D1B54108D7D5}.Release|x86.ActiveCfg = Release|Any CPU + {0CF04D7C-C568-4539-ABCB-D1B54108D7D5}.Release|x86.Build.0 = Release|Any CPU + {07217170-E7D4-4DA4-AB08-BE52AA1D9174}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {07217170-E7D4-4DA4-AB08-BE52AA1D9174}.Debug|Any CPU.Build.0 = Debug|Any CPU + {07217170-E7D4-4DA4-AB08-BE52AA1D9174}.Debug|x64.ActiveCfg = Debug|Any CPU + {07217170-E7D4-4DA4-AB08-BE52AA1D9174}.Debug|x64.Build.0 = Debug|Any CPU + {07217170-E7D4-4DA4-AB08-BE52AA1D9174}.Debug|x86.ActiveCfg = Debug|Any CPU + {07217170-E7D4-4DA4-AB08-BE52AA1D9174}.Debug|x86.Build.0 = Debug|Any CPU + {07217170-E7D4-4DA4-AB08-BE52AA1D9174}.Release|Any CPU.ActiveCfg = Release|Any CPU + {07217170-E7D4-4DA4-AB08-BE52AA1D9174}.Release|Any CPU.Build.0 = Release|Any CPU + {07217170-E7D4-4DA4-AB08-BE52AA1D9174}.Release|x64.ActiveCfg = Release|Any CPU + {07217170-E7D4-4DA4-AB08-BE52AA1D9174}.Release|x64.Build.0 = Release|Any CPU + {07217170-E7D4-4DA4-AB08-BE52AA1D9174}.Release|x86.ActiveCfg = Release|Any CPU + {07217170-E7D4-4DA4-AB08-BE52AA1D9174}.Release|x86.Build.0 = Release|Any CPU + {7EA25779-E02F-452C-AB8A-F5C5DCE972B2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7EA25779-E02F-452C-AB8A-F5C5DCE972B2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7EA25779-E02F-452C-AB8A-F5C5DCE972B2}.Debug|x64.ActiveCfg = Debug|Any CPU + {7EA25779-E02F-452C-AB8A-F5C5DCE972B2}.Debug|x64.Build.0 = Debug|Any CPU + {7EA25779-E02F-452C-AB8A-F5C5DCE972B2}.Debug|x86.ActiveCfg = Debug|Any CPU + {7EA25779-E02F-452C-AB8A-F5C5DCE972B2}.Debug|x86.Build.0 = Debug|Any CPU + {7EA25779-E02F-452C-AB8A-F5C5DCE972B2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7EA25779-E02F-452C-AB8A-F5C5DCE972B2}.Release|Any CPU.Build.0 = Release|Any CPU + {7EA25779-E02F-452C-AB8A-F5C5DCE972B2}.Release|x64.ActiveCfg = Release|Any CPU + {7EA25779-E02F-452C-AB8A-F5C5DCE972B2}.Release|x64.Build.0 = Release|Any CPU + {7EA25779-E02F-452C-AB8A-F5C5DCE972B2}.Release|x86.ActiveCfg = Release|Any CPU + {7EA25779-E02F-452C-AB8A-F5C5DCE972B2}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {9752F8AC-BA53-430D-AB8E-CC5217356C4F} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {4EEFF4A6-6AF6-4D84-8333-8A9AF4263113} = {0AB3BF05-4346-4AA6-1389-037BE0695223} + {1A03D818-3FA3-430D-8AAA-382DD36D45F7} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {52C67ED5-242A-4758-B529-14E0B8B84F5D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {1EA6F64E-7AD4-41EE-BBA9-4F0868294A6A} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {58BCA5C9-578F-4FCC-9161-FF890E0EF9CD} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {18555F26-9DE7-4842-AAFB-8614061EB1E5} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {A1B2C3D4-E5F6-7890-ABCD-EF1234567890} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {B2C3D4E5-F6A7-8901-BCDE-F12345678901} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {C3D4E5F6-A7B8-9012-CDEF-123456789012} = {0AB3BF05-4346-4AA6-1389-037BE0695223} + {F4BBAFE0-69B9-4890-9AC0-6D311137B6EC} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {081623C1-3E9E-4276-ABC2-6C7C58E5DDF5} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {0481DFA3-4AF9-4421-93E0-A3E3083FF003} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {05640B40-FB77-49E2-8741-D29F7ED13263} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {5EF30C04-1383-44D8-A73A-E8D5FD67218D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {4C888035-8687-4DE4-A192-FBE0BA7375F7} = {5D20AA90-6969-D8BD-9DCD-8634F4692FDA} + {24DB7B77-A721-4DC4-926F-D23606FB65A3} = {5D20AA90-6969-D8BD-9DCD-8634F4692FDA} + {0CF04D7C-C568-4539-ABCB-D1B54108D7D5} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {07217170-E7D4-4DA4-AB08-BE52AA1D9174} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {7EA25779-E02F-452C-AB8A-F5C5DCE972B2} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + EndGlobalSection +EndGlobal diff --git a/README.md b/README.md new file mode 100644 index 0000000..e75288e --- /dev/null +++ b/README.md @@ -0,0 +1,100 @@ +# Levels + +A multi-level time-series database for financial market data -- orderbook events, price streaming, and OHLCV bars. Built on .NET 10.0 with cross-language support for C++, Python, and TypeScript. + +## Features + +- **Custom binary format** -- 56-byte core records with schema-extensible fields, CRC32 integrity, little-endian layout +- **Three operational modes** -- embed in C# apps, run as a standalone TCP server, or deploy as a REST API +- **Automatic data lifecycle** -- compaction, period promotion with data quality checks, configurable retention and archival +- **Cross-language support** -- generate readers/writers for C#, Python, TypeScript, and C++ from a single `.fbs` schema +- **Real-time and batch** -- OHLCV bar emission, top-of-book snapshots, orderbook L1/L2/L3 projection +- **Export** -- CSV, Parquet, and Avro adapters +- **Observability** -- OpenTelemetry metrics and tracing, Prometheus scraping endpoint + +## Prerequisites + +- [.NET 10.0 SDK](https://dotnet.microsoft.com/download/dotnet/10.0) (preview) + +## Quick Start + +```bash +# Build +dotnet build + +# Run tests +dotnet test + +# Run the sample (writes synthetic orderbook data) +dotnet run --project samples/CryptoExchangeSample +``` + +See the [Quickstart Guide](docs/quickstart.md) for a full walkthrough. + +## Operational Modes + +| Mode | Use case | Entry point | +|------|----------|-------------| +| [Embedded](docs/embedded-mode.md) | In-process C# ingestion | `AddLevels()` + `AddLevelsDataSink()` | +| [TCP Server](docs/server-mode.md) | Standalone daemon, multi-language clients | `src/Levels.Server` with YAML config | +| [REST API](docs/web-api.md) | HTTP queries and export | `src/Levels.Web` with ASP.NET Core | + +## CLI + +The `levels` CLI tool provides file inspection, compaction, code generation, resampling, and data export. + +```bash +dotnet run --project src/Levels.Cli -- --help +``` + +See the [CLI Reference](docs/cli.md) for all commands. + +## Documentation + +- [Quickstart](docs/quickstart.md) -- get up and running +- [Architecture](docs/architecture.md) -- data pipeline, binary format, system internals +- [Embedded Mode](docs/embedded-mode.md) -- in-process C# usage +- [Server Mode](docs/server-mode.md) -- standalone TCP server +- [Web API](docs/web-api.md) -- REST endpoints +- [Schemas](docs/schemas.md) -- FlatBuffers schema system and code generation +- [Configuration](docs/configuration.md) -- complete options reference +- [CLI](docs/cli.md) -- command-line tool reference +- [Cross-Language](docs/cross-language.md) -- C++, Python, TypeScript interop + +## Project Structure + +``` +Levels/ + src/ + Levels.Core/ Core binary format, interfaces, WAL, diagnostics + Levels.SourceGen/ Roslyn source generator for .fbs schemas (netstandard2.0) + Levels.Sinks/ Partitioned ingestion, circuit breaker, backpressure + Levels.DataFlow/ Channel-based async pub/sub bus + Levels.Query/ In-memory file index, orderbook projection + Levels.Compaction/ Event-sourcing replay, synthetic snapshots, archival + Levels.Period/ Data quality checks, promotion/demotion + Levels.Resampled/ OHLCV bars, top-of-book snapshots + Levels.Export/ CSV, Parquet, Avro adapters + Levels.Hosting/ DI registration (AddLevels), OpenTelemetry setup + Levels.Hints/ LiteDB-based query hints + Levels.Protocol/ Wire protocol (frame reader/writer) + Levels.Server/ Standalone TCP server + Levels.Client/ TCP client library + Levels.Web/ ASP.NET Core REST API + Levels.Cli/ Command-line tool + samples/ + CryptoExchangeSample/ Simulated exchange (synthetic data) + BinanceLiveSample/ Live Binance WebSocket feed + tests/ + Levels.Tests/ xUnit tests + cross-language round-trips + Levels.Benchmarks/ BenchmarkDotNet performance suite + cross-language/ + schema.fbs Canonical schema definition + cpp/ C++ generated code and tests + python/ Python generated code and tests + typescript/ TypeScript generated code and tests +``` + +## License + +Apache-2.0 diff --git a/cross-language/cpp/CMakeLists.txt b/cross-language/cpp/CMakeLists.txt new file mode 100644 index 0000000..9e8492f --- /dev/null +++ b/cross-language/cpp/CMakeLists.txt @@ -0,0 +1,7 @@ +cmake_minimum_required(VERSION 3.14) +project(aiondb_cross_language_cpp LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +add_executable(write_test_file write_test_file.cpp) diff --git a/cross-language/cpp/record.h b/cross-language/cpp/record.h new file mode 100644 index 0000000..d87b0bf --- /dev/null +++ b/cross-language/cpp/record.h @@ -0,0 +1,89 @@ +// +// Generated from .fbs schema — do not edit by hand. + +#pragma once + +#include +#include + +// ── Constants ────────────────────────────────────────────────────────────────── +static const int HEADER_SIZE = 128; +static const int FOOTER_SIZE = 64; +static const int CORE_RECORD_SIZE = 56; +static const int RECORD_SIZE = 88; +static const int EXTENSION_SIZE = 32; +static const uint16_t FORMAT_VERSION = 2; +static const uint32_t SCHEMA_ID = 0x5D1FE1FDu; + +// Extension field offsets (relative to record start) +static const int ORDER_ID_OFFSET = 56; +static const int ORDER_ID_SIZE = 32; + +// ── Binary layout structs (packed, little-endian) ───────────────────────────── + +#pragma pack(push, 1) + +struct FileHeader { + uint8_t magic[8]; // "LEVELS01" + uint16_t version; // FORMAT_VERSION + uint8_t file_type; // 0 = Raw + uint8_t padding; // 0 + uint32_t schema_id; + int64_t price_stream_id; + int64_t created_at; + int32_t price_scale; + int32_t quantity_scale; + uint32_t resampled_config_hash; + uint16_t record_size; // total bytes per record + uint8_t reserved[82]; // zeros +}; +static_assert(sizeof(FileHeader) == 128, "FileHeader must be 128 bytes"); + +struct CoreRecord { + int64_t observed_time; // 0 + int64_t write_timestamp; // 8 + int64_t price_stream_id; // 16 + int64_t price; // 24 + int64_t quantity; // 32 + uint16_t _reserved; // 40 + uint8_t record_type; // 42 + uint8_t record_side; // 43 + uint32_t sequence; // 44 + uint16_t level; // 48 + uint16_t flags; // 50 + uint32_t crc32; // 52 +}; +static_assert(sizeof(CoreRecord) == 56, "CoreRecord must be 56 bytes"); + +struct FullRecord { + int64_t observed_time; // 0 + int64_t write_timestamp; // 8 + int64_t price_stream_id; // 16 + int64_t price; // 24 + int64_t quantity; // 32 + uint16_t _reserved; // 40 + uint8_t record_type; // 42 + uint8_t record_side; // 43 + uint32_t sequence; // 44 + uint16_t level; // 48 + uint16_t flags; // 50 + uint32_t crc32; // 52 + uint8_t order_id[32]; // 56 +}; +static_assert(sizeof(FullRecord) == 88, "FullRecord must be 88 bytes"); + +struct FileFooter { + int64_t record_count; // 0 + int64_t delta_count; // 8 + int64_t first_write_timestamp; // 16 + int64_t last_write_timestamp; // 24 + int64_t first_observed_time; // 32 + int64_t last_observed_time; // 40 + uint32_t file_crc32; // 48 + uint8_t padding[4]; // 52 + uint8_t magic_end[8]; // 56 "LEVEND01" +}; +static_assert(sizeof(FileFooter) == 64, "FileFooter must be 64 bytes"); + +#pragma pack(pop) + diff --git a/cross-language/cpp/write_test_file b/cross-language/cpp/write_test_file new file mode 100755 index 0000000..65794c1 Binary files /dev/null and b/cross-language/cpp/write_test_file differ diff --git a/cross-language/cpp/write_test_file.cpp b/cross-language/cpp/write_test_file.cpp new file mode 100644 index 0000000..d2d17f3 --- /dev/null +++ b/cross-language/cpp/write_test_file.cpp @@ -0,0 +1,187 @@ +/** + * Writes a deterministic STLTH.Levels .raw test fixture using the canonical test vector. + * Output: ../fixtures/cpp_test.raw + * + * Assumes little-endian platform (x86/x64/ARM LE). + */ + +#include +#include +#include +#include + +#include "record.h" + +// ── CRC-32 lookup table (IEEE polynomial 0xEDB88320) ────────────────────────── +static uint32_t crc32_table[256]; +static bool crc32_table_initialized = false; + +static void init_crc32_table() { + if (crc32_table_initialized) return; + for (uint32_t i = 0; i < 256; i++) { + uint32_t crc = i; + for (int j = 0; j < 8; j++) { + if (crc & 1) + crc = (crc >> 1) ^ 0xEDB88320u; + else + crc >>= 1; + } + crc32_table[i] = crc; + } + crc32_table_initialized = true; +} + +static uint32_t compute_crc32(const uint8_t* data, size_t len) { + init_crc32_table(); + uint32_t crc = 0xFFFFFFFFu; + for (size_t i = 0; i < len; i++) { + crc = (crc >> 8) ^ crc32_table[(crc ^ data[i]) & 0xFF]; + } + return crc ^ 0xFFFFFFFFu; +} + +static uint32_t compute_crc32_two(const uint8_t* data1, size_t len1, + const uint8_t* data2, size_t len2) { + init_crc32_table(); + uint32_t crc = 0xFFFFFFFFu; + for (size_t i = 0; i < len1; i++) + crc = (crc >> 8) ^ crc32_table[(crc ^ data1[i]) & 0xFF]; + for (size_t i = 0; i < len2; i++) + crc = (crc >> 8) ^ crc32_table[(crc ^ data2[i]) & 0xFF]; + return crc ^ 0xFFFFFFFFu; +} + +// ── Test vector ─────────────────────────────────────────────────────────────── + +static const uint32_t TEST_SCHEMA_ID = 0x12345678; +static const int64_t PRICE_STREAM_ID = 42; +static const int64_t CREATED_AT = 1000000000; +static const int32_t PRICE_SCALE = 2; +static const int32_t QUANTITY_SCALE = 4; + +struct TestRecord { + int64_t observed_time; + int64_t write_timestamp; + int64_t price_stream_id; + int64_t price; + int64_t quantity; + uint8_t record_type; + uint8_t record_side; + uint32_t sequence; + uint16_t level; + uint16_t flags; +}; + +static const TestRecord TEST_RECORDS[] = { + // SNAP Bid + { 1000000, 2000000, 42, 50000, 100, 0, 0, 0, 0, 0 }, + // DELTA Bid + { 1000100, 2000100, 42, 50000, 150, 1, 0, 1, 0, 0 }, + // DELTA Ask + { 1000200, 2000200, 42, 51000, 75, 1, 1, 2, 0, 0 }, +}; + +static const int NUM_RECORDS = sizeof(TEST_RECORDS) / sizeof(TEST_RECORDS[0]); + +int main() { + // Build output path relative to this source file's directory + std::string output_dir = std::string(__FILE__); + size_t last_sep = output_dir.find_last_of("/\\"); + if (last_sep != std::string::npos) + output_dir = output_dir.substr(0, last_sep); + else + output_dir = "."; + std::string fixtures_dir = output_dir + "/../fixtures"; + std::string output_path = fixtures_dir + "/cpp_test.raw"; + +#ifdef _WIN32 + _mkdir(fixtures_dir.c_str()); +#else + char cmd[512]; + snprintf(cmd, sizeof(cmd), "mkdir -p \"%s\"", fixtures_dir.c_str()); + system(cmd); +#endif + + FILE* fp = fopen(output_path.c_str(), "wb"); + if (!fp) { + fprintf(stderr, "Failed to open %s for writing\n", output_path.c_str()); + return 1; + } + + // ── Write header ────────────────────────────────────────────────────────── + FileHeader header; + memset(&header, 0, sizeof(header)); + memcpy(header.magic, "LEVELS01", 8); + header.version = FORMAT_VERSION; + header.file_type = 0; // Raw + header.padding = 0; + header.schema_id = TEST_SCHEMA_ID; + header.price_stream_id = PRICE_STREAM_ID; + header.created_at = CREATED_AT; + header.price_scale = PRICE_SCALE; + header.quantity_scale = QUANTITY_SCALE; + header.resampled_config_hash = 0; + header.record_size = RECORD_SIZE; + fwrite(&header, sizeof(header), 1, fp); + + // ── Write records ───────────────────────────────────────────────────────── + uint8_t all_record_bytes[NUM_RECORDS * RECORD_SIZE]; + int delta_count = 0; + + for (int i = 0; i < NUM_RECORDS; i++) { + const TestRecord& tr = TEST_RECORDS[i]; + FullRecord rec; + memset(&rec, 0, sizeof(rec)); + rec.observed_time = tr.observed_time; + rec.write_timestamp = tr.write_timestamp; + rec.price_stream_id = tr.price_stream_id; + rec.price = tr.price; + rec.quantity = tr.quantity; + rec._reserved = 0; + rec.record_type = tr.record_type; + rec.record_side = tr.record_side; + rec.sequence = tr.sequence; + rec.level = tr.level; + rec.flags = tr.flags; + rec.crc32 = 0; + memset(rec.order_id, 0, ORDER_ID_SIZE); + + // CRC covers bytes 0..51 + bytes 56..87 (core without CRC + extension) + rec.crc32 = compute_crc32_two( + reinterpret_cast(&rec), 52, + rec.order_id, ORDER_ID_SIZE); + + memcpy(all_record_bytes + i * RECORD_SIZE, &rec, RECORD_SIZE); + fwrite(&rec, sizeof(rec), 1, fp); + + if (tr.record_type == 1) delta_count++; + } + + // ── Write footer ────────────────────────────────────────────────────────── + uint32_t file_crc = compute_crc32(all_record_bytes, sizeof(all_record_bytes)); + + FileFooter footer; + memset(&footer, 0, sizeof(footer)); + footer.record_count = NUM_RECORDS; + footer.delta_count = delta_count; + footer.first_write_timestamp = TEST_RECORDS[0].write_timestamp; + footer.last_write_timestamp = TEST_RECORDS[NUM_RECORDS - 1].write_timestamp; + footer.first_observed_time = TEST_RECORDS[0].observed_time; + footer.last_observed_time = TEST_RECORDS[NUM_RECORDS - 1].observed_time; + footer.file_crc32 = file_crc; + memcpy(footer.magic_end, "LEVEND01", 8); + + fwrite(&footer, sizeof(footer), 1, fp); + fclose(fp); + + printf("Wrote %s\n", output_path.c_str()); + printf(" Header: %zu bytes\n", sizeof(FileHeader)); + printf(" Records: %d x %d = %d bytes\n", + NUM_RECORDS, RECORD_SIZE, NUM_RECORDS * RECORD_SIZE); + printf(" Footer: %zu bytes\n", sizeof(FileFooter)); + printf(" Total: %zu bytes\n", + sizeof(FileHeader) + NUM_RECORDS * RECORD_SIZE + sizeof(FileFooter)); + printf(" FileCRC: 0x%08X\n", file_crc); + + return 0; +} diff --git a/cross-language/fixtures/cpp_test.raw b/cross-language/fixtures/cpp_test.raw new file mode 100644 index 0000000..2b3b39c Binary files /dev/null and b/cross-language/fixtures/cpp_test.raw differ diff --git a/cross-language/fixtures/python_test.raw b/cross-language/fixtures/python_test.raw new file mode 100644 index 0000000..2b3b39c Binary files /dev/null and b/cross-language/fixtures/python_test.raw differ diff --git a/cross-language/fixtures/typescript_test.raw b/cross-language/fixtures/typescript_test.raw new file mode 100644 index 0000000..2b3b39c Binary files /dev/null and b/cross-language/fixtures/typescript_test.raw differ diff --git a/cross-language/generate-fixtures.sh b/cross-language/generate-fixtures.sh new file mode 100755 index 0000000..add720a --- /dev/null +++ b/cross-language/generate-fixtures.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Generates Levels test fixtures from Python, TypeScript, and C++ implementations. +# Run from the cross-language directory, or the script will cd there automatically. +# +# NOTE: After creating this file, make it executable: +# chmod +x cross-language/generate-fixtures.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +echo "=== Generating Levels cross-language test fixtures ===" +echo "" + +# ── Codegen: generate record types from schema.fbs ─────────────────────────── +echo "--- Codegen ---" +LEVELS="${LEVELS:-dotnet run --project ../src/Levels.Cli --}" +$LEVELS codegen schema.fbs --language python --output python/ +$LEVELS codegen schema.fbs --language typescript --output typescript/ +$LEVELS codegen schema.fbs --language cpp --output cpp/ +echo "" + +# Ensure fixtures directory exists +mkdir -p fixtures + +# ── Python ───────────────────────────────────────────────────────────────────── +echo "--- Python ---" +python3 python/write_test_file.py +echo "" + +# ── TypeScript ───────────────────────────────────────────────────────────────── +echo "--- TypeScript ---" +cd typescript +npm install --silent +npx ts-node --esm write_test_file.ts +cd "$SCRIPT_DIR" +echo "" + +# ── C++ ──────────────────────────────────────────────────────────────────────── +echo "--- C++ ---" +mkdir -p cpp/build +cd cpp/build +cmake .. -DCMAKE_BUILD_TYPE=Release +make +./write_test_file +cd "$SCRIPT_DIR" +echo "" + +# ── Verify all fixtures exist ────────────────────────────────────────────────── +echo "=== Verifying fixtures ===" +for lang in python typescript cpp; do + fixture="fixtures/${lang}_test.raw" + if [ -f "$fixture" ]; then + size=$(wc -c < "$fixture" | tr -d ' ') + echo " OK: $fixture ($size bytes)" + else + echo " MISSING: $fixture" + exit 1 + fi +done + +echo "" +echo "=== All fixtures generated successfully ===" diff --git a/cross-language/python/record.py b/cross-language/python/record.py new file mode 100644 index 0000000..b7ad6a7 --- /dev/null +++ b/cross-language/python/record.py @@ -0,0 +1,99 @@ +# +# Generated from .fbs schema — do not edit by hand. + +from __future__ import annotations + +import struct +import binascii + +# ── Constants ────────────────────────────────────────────────────────────────── +HEADER_SIZE = 128 +FOOTER_SIZE = 64 +CORE_RECORD_SIZE = 56 +RECORD_SIZE = 88 +EXTENSION_SIZE = 32 +FORMAT_VERSION = 2 +SCHEMA_ID = 0x5D1FE1FD + +HEADER_MAGIC = b"LEVELS01" +FOOTER_MAGIC = b"LEVEND01" +FILE_TYPE_RAW = 0 + +# Extension fields: (name, offset_from_core, size) +EXTENSION_FIELDS = [ + ("order_id", 0, 32), +] + +# struct.pack format for the 56-byte core record (little-endian) +# int64×5, uint16, uint8×2, uint32, uint16×2, uint32 +CORE_FMT = " int: + """Compute CRC-32 (IEEE) and return as unsigned 32-bit.""" + return binascii.crc32(data) & 0xFFFFFFFF + + +def pack_header(price_stream_id: int, created_at: int, + price_scale: int, quantity_scale: int, + schema_id: int = SCHEMA_ID, + file_type: int = FILE_TYPE_RAW, + resampled_config_hash: int = 0) -> bytes: + buf = bytearray(HEADER_SIZE) + buf[0:8] = HEADER_MAGIC + struct.pack_into(" bytes: + """Pack a full record (core + extension) with computed CRC.""" + core_data = struct.pack( + CORE_FMT, + observed_time, write_timestamp, price_stream_id, price, quantity, + 0, # _reserved + record_type, record_side, sequence, level, flags, + 0, # CRC placeholder + ) + ext = extension_bytes if extension_bytes is not None else b'\x00' * EXTENSION_SIZE + if len(ext) != EXTENSION_SIZE: + raise ValueError(f"Extension bytes must be {EXTENSION_SIZE} bytes, got {len(ext)}") + # CRC covers bytes 0..51 + extension bytes (skipping CRC field at 52..55) + crc = compute_crc32(core_data[:52] + ext) + core_data = struct.pack( + CORE_FMT, + observed_time, write_timestamp, price_stream_id, price, quantity, + 0, record_type, record_side, sequence, level, flags, + crc, + ) + return core_data + ext + + +def pack_footer(record_count: int, delta_count: int, + first_write_ts: int, last_write_ts: int, + first_observed: int, last_observed: int, + file_crc: int) -> bytes: + buf = bytearray(FOOTER_SIZE) + struct.pack_into("=12" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@tsconfig/node10": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz", + "integrity": "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "20.19.37", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.37.tgz", + "integrity": "sha512-8kzdPJ3FsNsVIurqBs7oodNnCEVbni9yUEkaHbgptDACOPW04jimGagZ51E6+lXUwJjgnBw+hyko/lkFWCldqw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/acorn": { + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", + "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-walk": { + "version": "8.3.5", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.5.tgz", + "integrity": "sha512-HEHNfbars9v4pgpW6SO1KSPkfoS0xVOM/9UzkJltjlsHZmJasxg8aXkuZa7SMf8vKGIBhpUsPluQSqhJFCqebw==", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true, + "license": "MIT" + }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "license": "Apache-2.0", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/diff": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.4.tgz", + "integrity": "sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true, + "license": "ISC" + }, + "node_modules/ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true, + "license": "MIT" + }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + } + } +} diff --git a/cross-language/typescript/package.json b/cross-language/typescript/package.json new file mode 100644 index 0000000..7c5a7f6 --- /dev/null +++ b/cross-language/typescript/package.json @@ -0,0 +1,13 @@ +{ + "name": "levels-cross-language-ts", + "type": "module", + "private": true, + "dependencies": { + "crc-32": "^1.2.2" + }, + "devDependencies": { + "ts-node": "^10.9.2", + "typescript": "^5.4.0", + "@types/node": "^20.0.0" + } +} diff --git a/cross-language/typescript/record.ts b/cross-language/typescript/record.ts new file mode 100644 index 0000000..3986db3 --- /dev/null +++ b/cross-language/typescript/record.ts @@ -0,0 +1,109 @@ +// +// Generated from .fbs schema — do not edit by hand. + +import CRC32 from "crc-32"; + +// ── Constants ────────────────────────────────────────────────────────────────── +export const HEADER_SIZE = 128; +export const FOOTER_SIZE = 64; +export const CORE_RECORD_SIZE = 56; +export const RECORD_SIZE = 88; +export const EXTENSION_SIZE = 32; +export const FORMAT_VERSION = 2; +export const SCHEMA_ID = 0x5D1FE1FD; + +export const HEADER_MAGIC = Buffer.from("LEVELS01", "ascii"); +export const FOOTER_MAGIC = Buffer.from("LEVEND01", "ascii"); +export const FILE_TYPE_RAW = 0; + +// Extension fields: [name, offsetFromCore, size] +export const EXTENSION_FIELDS: [string, number, number][] = [ + ["order_id", 0, 32], +]; + +export interface RecordData { + observedTime: bigint; + writeTimestamp: bigint; + priceStreamId: bigint; + price: bigint; + quantity: bigint; + recordType: number; + recordSide: number; + sequence: number; + level: number; + flags: number; +} + +export function computeCrc32(buf: Buffer): number { + return CRC32.buf(buf) >>> 0; +} + +export function packHeader( + priceStreamId: bigint, + createdAt: bigint, + priceScale: number, + quantityScale: number, + schemaId: number = SCHEMA_ID, + fileType: number = FILE_TYPE_RAW, + resampledConfigHash: number = 0, +): Buffer { + const buf = Buffer.alloc(HEADER_SIZE); + HEADER_MAGIC.copy(buf, 0); + buf.writeUInt16LE(FORMAT_VERSION, 8); + buf[10] = fileType; + buf[11] = 0; // padding + buf.writeUInt32LE(schemaId, 12); + buf.writeBigInt64LE(priceStreamId, 16); + buf.writeBigInt64LE(createdAt, 24); + buf.writeInt32LE(priceScale, 32); + buf.writeInt32LE(quantityScale, 36); + buf.writeUInt32LE(resampledConfigHash, 40); + buf.writeUInt16LE(RECORD_SIZE, 44); + return buf; +} + +export function packRecord(rec: RecordData, extensionBytes?: Buffer): Buffer { + const buf = Buffer.alloc(RECORD_SIZE); + buf.writeBigInt64LE(rec.observedTime, 0); + buf.writeBigInt64LE(rec.writeTimestamp, 8); + buf.writeBigInt64LE(rec.priceStreamId, 16); + buf.writeBigInt64LE(rec.price, 24); + buf.writeBigInt64LE(rec.quantity, 32); + buf.writeUInt16LE(0, 40); // _reserved + buf[42] = rec.recordType; + buf[43] = rec.recordSide; + buf.writeUInt32LE(rec.sequence, 44); + buf.writeUInt16LE(rec.level, 48); + buf.writeUInt16LE(rec.flags, 50); + // Extension bytes (already zero from Buffer.alloc) + if (extensionBytes) { + extensionBytes.copy(buf, CORE_RECORD_SIZE, 0, EXTENSION_SIZE); + } + // CRC covers bytes 0..51 + bytes 56..RECORD_SIZE + const crc = computeCrc32(Buffer.concat([buf.subarray(0, 52), buf.subarray(56)])); + buf.writeUInt32LE(crc, 52); + return buf; +} + +export function packFooter( + recordCount: bigint, + deltaCount: bigint, + firstWriteTs: bigint, + lastWriteTs: bigint, + firstObserved: bigint, + lastObserved: bigint, + fileCrc: number, +): Buffer { + const buf = Buffer.alloc(FOOTER_SIZE); + buf.writeBigInt64LE(recordCount, 0); + buf.writeBigInt64LE(deltaCount, 8); + buf.writeBigInt64LE(firstWriteTs, 16); + buf.writeBigInt64LE(lastWriteTs, 24); + buf.writeBigInt64LE(firstObserved, 32); + buf.writeBigInt64LE(lastObserved, 40); + buf.writeUInt32LE(fileCrc, 48); + // 4 bytes padding at 52 + FOOTER_MAGIC.copy(buf, 56); + return buf; +} + diff --git a/cross-language/typescript/write_test_file.ts b/cross-language/typescript/write_test_file.ts new file mode 100644 index 0000000..570fd95 --- /dev/null +++ b/cross-language/typescript/write_test_file.ts @@ -0,0 +1,113 @@ +/** + * Writes a deterministic STLTH.Levels .raw test fixture using the canonical test vector. + * Output: ../fixtures/typescript_test.raw + */ + +import * as fs from "node:fs"; +import * as path from "node:path"; +import { fileURLToPath } from "node:url"; +import { + HEADER_SIZE, + FOOTER_SIZE, + RECORD_SIZE, + computeCrc32, + packHeader, + packRecord, + packFooter, + type RecordData, +} from "./record.ts"; + +// ── Test vector ──────────────────────────────────────────────────────────────── +const SCHEMA_ID = 0x12345678; +const PRICE_STREAM_ID = 42n; +const CREATED_AT = 1000000000n; +const PRICE_SCALE = 2; +const QUANTITY_SCALE = 4; + +const RECORDS: RecordData[] = [ + { + observedTime: 1000000n, + writeTimestamp: 2000000n, + priceStreamId: 42n, + price: 50000n, + quantity: 100n, + recordType: 0, // Snap + recordSide: 0, // Bid + sequence: 0, + level: 0, + flags: 0, + }, + { + observedTime: 1000100n, + writeTimestamp: 2000100n, + priceStreamId: 42n, + price: 50000n, + quantity: 150n, + recordType: 1, // Delta + recordSide: 0, // Bid + sequence: 1, + level: 0, + flags: 0, + }, + { + observedTime: 1000200n, + writeTimestamp: 2000200n, + priceStreamId: 42n, + price: 51000n, + quantity: 75n, + recordType: 1, // Delta + recordSide: 1, // Ask + sequence: 2, + level: 0, + flags: 0, + }, +]; + +function main(): void { + const __dirname = path.dirname(fileURLToPath(import.meta.url)); + const outputDir = path.join(__dirname, "..", "fixtures"); + fs.mkdirSync(outputDir, { recursive: true }); + const outputPath = path.join(outputDir, "typescript_test.raw"); + + const header = packHeader(PRICE_STREAM_ID, CREATED_AT, PRICE_SCALE, QUANTITY_SCALE, SCHEMA_ID); + + const recordBuffers: Buffer[] = []; + for (const rec of RECORDS) { + recordBuffers.push(packRecord(rec)); + } + + const allRecordBytes = Buffer.concat(recordBuffers); + const fileCrc = computeCrc32(allRecordBytes); + + const deltaCount = RECORDS.filter((r) => r.recordType === 1).length; + const firstWriteTs = RECORDS[0].writeTimestamp; + const lastWriteTs = RECORDS[RECORDS.length - 1].writeTimestamp; + + const observedTimes = RECORDS.map((r) => r.observedTime); + const firstObserved = observedTimes.reduce((a, b) => (a < b ? a : b)); + const lastObserved = observedTimes.reduce((a, b) => (a > b ? a : b)); + + const footer = packFooter( + BigInt(RECORDS.length), + BigInt(deltaCount), + firstWriteTs, + lastWriteTs, + firstObserved, + lastObserved, + fileCrc, + ); + + const fullFile = Buffer.concat([header, allRecordBytes, footer]); + fs.writeFileSync(outputPath, fullFile); + + console.log(`Wrote ${outputPath}`); + console.log(` Header: ${HEADER_SIZE} bytes`); + console.log( + ` Records: ${RECORDS.length} x ${RECORD_SIZE} = ${RECORDS.length * RECORD_SIZE} bytes`, + ); + console.log(` Footer: ${FOOTER_SIZE} bytes`); + console.log(` Total: ${fullFile.length} bytes`); + console.log(` FileCRC: 0x${fileCrc.toString(16).toUpperCase().padStart(8, "0")}`); +} + +main(); diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..46dc064 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,208 @@ +# Architecture + +## Data Pipeline + +``` + +-----------+ + | IDataSink | (typed records from your code) + +-----+-----+ + | + +-----v---------+ + | DataSinkAdapter| (maps to RawMarketEvent) + +-----+---------+ + | + +-------v---------+ + | PriceStreamSink | (partitioned writer, WAL, circuit breaker) + +-------+---------+ + | + +-----------v-----------+ + | BinaryRecordWriter | (header + records + footer) + +-----------+-----------+ + | + +----------v----------+ + | DataFlowBus | (channel-based pub/sub) + +--+------+------+---+ + | | | + +-------------+ +---+----+ +--------+ + |FileIndex | |Compact | |Period | + |SyncHandler | |ion | |Promot. | + +-------------+ +---+----+ +---+----+ + | | + +----v---+ +---v--------+ + | AGG | | PERIOD | + | files | | files | + +--------+ +---+--------+ + | + +------v-------+ + | Resampled | + | (OHLCV/L1) | + +--------------+ +``` + +## File Types + +Data progresses through four file types representing increasing levels of quality assurance: + +| Type | Extension | Description | +|------|-----------|-------------| +| **RAW** | `.raw` | Ingested events as-is. Rollovers by size (256 MB) or time (1 hour). | +| **AGG** | `.agg` | Compacted: event-sourced replay with synthetic snapshots inserted. | +| **PERIOD** | `.period` | Quality-checked AGG promoted after passing health checks. | +| **RESAMPLED** | `.resampled` | Derived OHLCV bars or top-of-book snapshots. | + +## Binary Format + +All files share the same layout. Little-endian byte order. + +### File Layout + +``` +[Header: 128 bytes][Record 0][Record 1]...[Record N][Footer: 64 bytes] +``` + +### Header (128 bytes) + +| Offset | Size | Field | Description | +|--------|------|-------|-------------| +| 0 | 8 | Magic | `LEVELS01` (ASCII) | +| 8 | 2 | Version | Format version (currently `2`) | +| 10 | 1 | FileType | 0=Raw, 1=Agg, 2=Period, 3=Resampled | +| 12 | 4 | SchemaId | FNV-1a hash of the `.fbs` schema | +| 16 | 8 | PriceStreamId | XxHash64 of venue + symbol | +| 24 | 8 | CreatedAt | File creation timestamp (nanoseconds) | +| 32 | 4 | PriceScale | Decimal places for price fields | +| 36 | 4 | QuantityScale | Decimal places for quantity fields | +| 40 | 4 | ResampledConfigHash | XxHash32 of resampled config (0 for non-resampled) | +| 44 | 2 | RecordSize | Bytes per record (56 + extensions) | +| 46-127 | 82 | Reserved | Zero-filled | + +### Core Record (56 bytes) + +| Offset | Size | Field | Type | Description | +|--------|------|-------|------|-------------| +| 0 | 8 | ObservedTime | int64 | Exchange timestamp | +| 8 | 8 | WriteTimestamp | int64 | Local write timestamp | +| 16 | 8 | PriceStreamId | int64 | Stream identifier | +| 24 | 8 | Price | int64 | Scaled price value | +| 32 | 8 | Quantity | int64 | Scaled quantity value | +| 40 | 2 | Reserved | uint16 | Reserved | +| 42 | 1 | Type | uint8 | 0=Snap, 1=Delta, 2=Tombstone | +| 43 | 1 | Side | uint8 | 0=Bid, 1=Ask, 2=Unknown | +| 44 | 4 | Sequence | uint32 | Monotonic sequence number | +| 48 | 2 | Level | uint16 | Orderbook level index | +| 50 | 2 | Flags | uint16 | Bit flags (0x0001=SyntheticSnap, 0x0002=IsOwner) | +| 52 | 4 | Crc32 | uint32 | CRC32 of bytes [0:52] + extension bytes | + +Extension fields (bytes 56+) are defined by the schema. For example, a 32-byte OrderId field produces 88-byte records. + +### Footer (64 bytes) + +| Offset | Size | Field | Type | +|--------|------|-------|------| +| 0 | 8 | RecordCount | int64 | +| 8 | 8 | DeltaCount | int64 | +| 16 | 8 | FirstWriteTimestamp | int64 | +| 24 | 8 | LastWriteTimestamp | int64 | +| 32 | 8 | FirstObservedTime | int64 | +| 40 | 8 | LastObservedTime | int64 | +| 48 | 4 | FileCrc32 | uint32 | +| 52 | 4 | FooterCrc32 | uint32 | +| 56 | 8 | MagicEnd | `LEVEND01` (ASCII) | + +### CRC32 Calculation + +Record CRC covers bytes `[0:52]` (core without CRC field) concatenated with any extension bytes. Uses the IEEE polynomial via `System.IO.Hashing.Crc32`. + +## Projects + +The solution contains 16 source projects: + +| Project | Target | Purpose | +|---------|--------|---------| +| Levels.Core | net10.0 | Binary format, interfaces, WAL, diagnostics | +| Levels.SourceGen | netstandard2.0 | Roslyn incremental generator for `.fbs` schemas | +| Levels.Sinks | net10.0 | Partitioned ingestion, circuit breaker | +| Levels.DataFlow | net10.0 | Channel-based async pub/sub bus | +| Levels.Query | net10.0 | In-memory file index, orderbook projection | +| Levels.Compaction | net10.0 | Event-sourcing replay, synthetic snapshots | +| Levels.Period | net10.0 | Quality checks, promotion/demotion | +| Levels.Resampled | net10.0 | OHLCV bars, top-of-book snapshots | +| Levels.Export | net10.0 | CSV, Parquet, Avro export adapters | +| Levels.Hosting | net10.0 | DI registration, OpenTelemetry setup | +| Levels.Hints | net10.0 | LiteDB-based query hints | +| Levels.Protocol | net10.0 | Wire protocol (frame reader/writer) | +| Levels.Server | net10.0 | Standalone TCP server | +| Levels.Client | net10.0 | TCP client library | +| Levels.Web | net10.0 | ASP.NET Core REST API | +| Levels.Cli | net10.0 | Command-line tool | + +## DataFlowBus + +The `DataFlowBus` is a channel-based async pub/sub system that connects the write pipeline to downstream consumers. It runs as an `IHostedService`. + +**Message types:** +- `RecordWrittenMessage` -- emitted after every record write +- `FileSealedMessage` -- emitted when a file is rolled over and sealed +- `AggCreatedMessage` -- emitted after compaction produces an AGG file + +**Handlers:** +- `FileIndexSyncHandler` -- keeps the in-memory `FileIndex` current +- `EventSourcingCompaction` -- triggers compaction on window close +- `PeriodPromotion` -- runs quality checks and promotes AGG to PERIOD +- `LiveResampledHandler` -- emits resampled output in real-time +- `HintsDbSyncHandler` -- syncs hints database + +Each handler has a dedicated bounded channel (capacity: configurable via `BackpressureLimit`). If a handler's channel is full, `RecordWrittenMessage` is dropped (counted in metrics). `FileSealedMessage` and `AggCreatedMessage` throw `DataFlowBackpressureException` or await space on the async path. + +## Write-Ahead Log (WAL) + +The WAL provides crash recovery for in-flight records. Each entry contains: + +``` +[StreamId:8][VenueLen:2][Venue:N][SymbolLen:2][Symbol:M][RecordSize:4][RecordBytes:R][CRC32:4] +``` + +On recovery, valid entries are replayed and entries with CRC mismatches are skipped. The WAL is truncated once durability is confirmed. + +## Compaction + +Compaction replays raw events through an `OrderbookReplayEngine` to produce AGG files: + +1. Raw files are grouped by stream and compaction window +2. When a window closes (wall clock > window end + grace period), compaction begins +3. All raw records are replayed in order, rebuilding orderbook state +4. Synthetic snapshots are inserted every N deltas or T time +5. Output is written as an `.agg` file + +## Period Promotion + +AGG files are promoted to PERIOD after passing health checks: + +- **CrossedBookDetector** -- ensures best bid < best ask at all times +- **MissingValueDetector** -- verifies no gaps exceed the configured threshold (default: 5 minutes) +- **SequenceIntegrityChecker** -- validates monotonic sequence numbers +- **SnapCoverageChecker** -- ensures sufficient snapshot coverage for recovery + +Failed checks produce a manifest file with failure reasons. A `DemotionService` can retroactively demote PERIOD files. + +## Query Resolution + +The `QueryLayer` resolves a `(PriceStreamId, startTime, endTime)` query to matching file entries from the `FileIndex`. The `OrderbookProjection` class replays matched records to reconstruct: + +- **L1** -- best bid/ask +- **L2** -- full depth (all price levels) +- **L3** -- individual order tracking (requires OrderId extension field) + +## Hints Engine + +The `HintsDb` is a LiteDB-backed persistent index that mirrors `FileIndex`. It speeds up startup by avoiding full disk scans. A consistency check validates hints against actual files on disk. + +## Observability + +OpenTelemetry meter: `Levels`, activity source: `Levels`. + +**Counters:** `pricestorage.records.written`, `pricestorage.files.sealed`, `pricestorage.circuit_breaker.trips`, `pricestorage.compaction.completed`, `pricestorage.healthcheck.passed`, `pricestorage.healthcheck.failed`, `pricestorage.promotions.completed`, `pricestorage.queries.executed`, `pricestorage.dataflow.backpressure`, `pricestorage.tcp.messages`, `pricestorage.archival.files_deleted` + +**Histograms:** `pricestorage.record.write_latency` (ms), `pricestorage.compaction.duration` (ms), `pricestorage.query.latency` (ms) + +**UpDownCounters:** `pricestorage.streams.active`, `pricestorage.connections.active` diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000..b74508b --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,176 @@ +# CLI Reference + +The `levels` CLI tool provides commands for inspecting data files, running compaction, generating schema code, resampling, and exporting data. + +## Running the CLI + +```bash +dotnet run --project src/Levels.Cli -- [options] +``` + +Or, if installed as a dotnet tool: + +```bash +levels [options] +``` + +## Commands + +### inspect + +Examine a Levels data file -- header metadata, footer statistics, and optionally individual records. + +```bash +levels inspect [--records] +``` + +**Arguments:** +- `` -- path to a `.raw`, `.agg`, `.period`, or `.resampled` file + +**Options:** +- `--records` -- print each record's type, side, price, quantity, sequence, level, flags, and observed time + +**Example:** + +```bash +levels inspect data/binance/1234567890/20250115_100000.raw +``` + +Output: + +``` +File: data/binance/1234567890/20250115_100000.raw + Version: 2 + FileType: Raw + PriceStreamId: 1234567890 + SchemaId: 0xE4E8A928 + PriceScale: 2 + QuantityScale: 8 + CreatedAt: 2025-01-15T10:00:00.0000000Z + Partial: False + RecordCount: 4200 + DeltaCount: 3800 + FirstObserved: 1736935200000 + LastObserved: 1736938800000 + FirstWrite: 1736935200100 + LastWrite: 1736938800050 + FileCRC: 0xA1B2C3D4 +``` + +With `--records`: + +``` +Records: + [0] Type=Snap Side=Bid Price=10350042 Qty=150000000 Seq=1 Level=0 Flags=0x0000 Observed=1736935200000 + [1] Type=Snap Side=Ask Price=10350142 Qty=120000000 Seq=2 Level=0 Flags=0x0000 Observed=1736935200000 + ... +``` + +### compact + +Run compaction on a data directory, replaying RAW files into AGG files with synthetic snapshots. + +```bash +levels compact [--window ] +``` + +**Arguments:** +- `` -- directory containing data files + +**Options:** +- `--window ` -- compaction window in hours (default: `1`) + +**Example:** + +```bash +levels compact ./data --window 2 +``` + +The command scans all `.raw` files, groups them by stream and time window, replays events through the `EventSourcingCompaction` engine, and writes `.agg` output files. The grace period is set to zero for CLI compaction (immediate processing). + +### codegen + +Generate typed accessors from a FlatBuffers `.fbs` schema file. + +```bash +levels codegen [--output ] [--language ] +``` + +**Arguments:** +- `` -- path to the schema file + +**Options:** +- `--output ` -- output directory (default: current directory) +- `--language ` -- target language (default: `csharp`) + +**Supported languages:** + +| Language flag | Aliases | Output file | +|---------------|---------|-------------| +| `csharp` | `cs` | `Record.g.cs` | +| `python` | `py` | `record.py` | +| `typescript` | `ts` | `record.ts` | +| `cpp` | `c++` | `record.h` | + +**Examples:** + +```bash +# C# (default) +levels codegen schema.fbs --output ./Generated + +# Python +levels codegen schema.fbs --output ./python --language python + +# TypeScript +levels codegen schema.fbs --output ./ts --language typescript + +# C++ +levels codegen schema.fbs --output ./cpp --language cpp +``` + +### resample + +Run batch resampling on existing data for a specific stream. + +```bash +levels resample --venue --stream --window +``` + +**Arguments:** +- `` -- directory containing data files + +**Options (all required):** +- `--venue ` -- venue name (e.g., `binance`) +- `--stream ` -- symbol name (e.g., `BTCUSDT`) +- `--window ` -- resampling window in seconds (default: `60`) + +**Example:** + +```bash +levels resample ./data --venue binance --stream BTCUSDT --window 300 +``` + +### export + +Export data for a specific stream to CSV, Parquet, or Avro format. + +```bash +levels export --venue --stream --format [--output ] +``` + +**Arguments:** +- `` -- directory containing data files + +**Options:** +- `--venue ` -- venue name (**required**) +- `--stream ` -- symbol name (**required**) +- `--format ` -- output format: `csv`, `parquet`, or `avro` (default: `csv`) +- `--output ` -- output file path (default: `{symbol}.{format}`) + +**Example:** + +```bash +levels export ./data --venue binance --stream BTCUSDT --format parquet --output btcusdt.parquet +``` + +The export command builds a `FileIndex` from disk, resolves all files for the stream, and pipes records through the `ExportPipeline` to the chosen adapter. diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..38dbe1f --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,154 @@ +# Configuration Reference + +## LevelsOptions + +The primary configuration class, set via `AddLevels()` or `AddLevels()`. + +| Property | Type | Default | Description | +|----------|------|---------|-------------| +| `DataPath` | string | **required** | Base directory for all data files | +| `BackpressureLimit` | int | `8192` | Channel capacity before write throttling | +| `RolloverSize` | long | `268435456` (256 MB) | Max file size before creating a new file | +| `RolloverInterval` | TimeSpan | 1 hour | Time-based file rollover | +| `PriceScale` | int | `0` | Decimal places for price fields | +| `QuantityScale` | int | `0` | Decimal places for quantity fields | +| `CompactionWindow` | TimeSpan | 1 hour | Time window for grouping RAW files into AGG | +| `SyntheticSnapIntervalDeltas` | int | `1000` | Insert synthetic snapshot every N deltas during compaction | +| `SyntheticSnapIntervalTime` | TimeSpan? | null | Insert synthetic snapshot every T time (optional) | +| `RetentionWindow` | TimeSpan | 7 days | How long to keep RAW files after compaction | +| `WindowGracePeriod` | TimeSpan | 5 minutes | Grace period before closing a compaction window | +| `ConfigVersion` | string | `"v1"` | Version tag for period promotion config | +| `EnableCompaction` | bool | `true` | Enable automatic RAW -> AGG compaction | +| `EnablePeriodPromotion` | bool | `true` | Enable automatic AGG -> PERIOD promotion | +| `FlushThresholdMs` | int | `0` | Flush interval in milliseconds (0 = immediate) | +| `FlushBufferSize` | int | `0` | Buffer size before flush (0 = unbuffered) | +| `SchemaId` | uint | `0` | Schema ID for file headers (auto-set by `AddLevels()`) | +| `RecordSize` | int | `56` | Bytes per record (auto-set by `AddLevels()`) | +| `EnableMetrics` | bool | `false` | Enable OpenTelemetry metrics via OTLP | +| `OtlpEndpoint` | string? | null | OTLP collector endpoint | +| `EnableTracing` | bool | `false` | Enable OpenTelemetry distributed tracing | + +## SinkConfig + +Internal configuration for `PriceStreamSink`. Built automatically from `LevelsOptions` during DI registration. + +| Property | Type | Default | Description | +|----------|------|---------|-------------| +| `OutputPath` | string | **required** | Data directory (set from `DataPath`) | +| `BackpressureLimit` | int | `8192` | Channel capacity | +| `RolloverSize` | long | 256 MB | File size rollover threshold | +| `RolloverInterval` | TimeSpan | 1 hour | Time-based rollover | +| `PriceScale` | int | `0` | Price decimal places | +| `QuantityScale` | int | `0` | Quantity decimal places | +| `SchemaId` | uint | `0` | Schema ID for headers | +| `RecordSize` | int | `56` | Record size in bytes | +| `FlushThresholdMs` | int | `0` | Flush interval | +| `FlushBufferSize` | int | `0` | Buffer size | +| `ConsumerPartitions` | int | `1` | Number of write partitions | +| `MaxWriteFailures` | int | `5` | Failures before circuit breaker trips | +| `CircuitBreakerCooldown` | TimeSpan | 60 seconds | Cooldown after circuit breaker trip | +| `DataFlowBus` | DataFlowBus? | null | Event bus for downstream handlers | + +## CompactionConfig + +Controls the event-sourcing compaction pipeline. + +| Property | Type | Default | Description | +|----------|------|---------|-------------| +| `DataPath` | string | **required** | Data directory | +| `CompactionWindow` | TimeSpan | 1 hour | Window for grouping RAW -> AGG | +| `SyntheticSnapIntervalDeltas` | int | `1000` | Snapshots every N deltas | +| `SyntheticSnapIntervalTime` | TimeSpan? | null | Snapshots every T time | +| `RetentionWindow` | TimeSpan | 7 days | RAW file retention | +| `WindowGracePeriod` | TimeSpan | 5 minutes | Grace period for late data | + +## ArchivalConfig + +Controls automatic deletion of old files. + +| Property | Type | Default | Description | +|----------|------|---------|-------------| +| `DataPath` | string | **required** | Data directory | +| `RawRetention` | TimeSpan | 7 days | How long to keep RAW files | +| `AggRetention` | TimeSpan | 30 days | How long to keep AGG files | +| `PeriodRetention` | TimeSpan? | null | How long to keep PERIOD files (null = forever) | +| `ScanInterval` | TimeSpan | 1 hour | How often to scan for expired files | + +## QueryConfig + +Configuration for the query layer. + +| Property | Type | Default | Description | +|----------|------|---------|-------------| +| `DataPath` | string | **required** | Data directory | +| `CompactionWindow` | TimeSpan | 1 hour | Compaction window (for query resolution) | + +## PeriodConfig + +Controls period promotion and data quality checks. + +| Property | Type | Default | Description | +|----------|------|---------|-------------| +| `DataPath` | string | **required** | Data directory | +| `CompactionWindow` | TimeSpan | 1 hour | Compaction window | +| `ConfigVersion` | string | **required** | Version tag for promotion rules | +| `MissingValueConfig` | MissingValueConfig | default | Gap detection thresholds | + +The `MissingValueConfig` controls per-stream thresholds for the `MissingValueDetector` health check. The default gap threshold is 5 minutes. + +## ResampledStreamConfig + +Configuration for resampled output (OHLCV bars or top-of-book snapshots). + +| Property | Type | Default | Description | +|----------|------|---------|-------------| +| `ConfigVersion` | string | **required** | Version tag | +| `SourceStreams` | IReadOnlyList\ | **required** | Input streams to resample | +| `ResamplingWindow` | TimeSpan | **required** | Bar/snapshot interval | +| `OutputType` | ResampledOutputType | `TopOfBook` | `TopOfBook` (0) or `OhlcvBar` (1) | +| `Venue` | string | **required** | Output venue name | +| `OutputPath` | string | **required** | Directory for resampled files | +| `PriceScale` | int | `0` | Price decimal places | +| `QuantityScale` | int | `0` | Quantity decimal places | + +The config hash (XxHash32 of `ConfigVersion`) is written to the file header's `ResampledConfigHash` field. + +## Server Configuration (YAML) + +The TCP server uses a YAML configuration file. See [Server Mode](server-mode.md) for the full format. + +```yaml +DataPath: ./data +Port: 5050 +Schema: + DllPath: path/to/schema.dll + TypeName: MyNamespace.Record +Levels: + PriceScale: 2 + QuantityScale: 8 + RolloverSize: 268435456 + RolloverInterval: "01:00:00" + BackpressureLimit: 8192 + EnableCompaction: true + EnablePeriodPromotion: false + CompactionWindow: "01:00:00" +Telemetry: + EnableMetrics: false + OtlpEndpoint: null + EnableTracing: false +``` + +## Web Configuration (appsettings.json) + +The web API uses standard ASP.NET Core configuration. See [Web API](web-api.md) for details. + +```json +{ + "Levels": { + "DataPath": "./data", + "EnableMetrics": true, + "OtlpEndpoint": "http://localhost:4317", + "EnableTracing": false + } +} +``` diff --git a/docs/cross-language.md b/docs/cross-language.md new file mode 100644 index 0000000..c196672 --- /dev/null +++ b/docs/cross-language.md @@ -0,0 +1,138 @@ +# Cross-Language Support + +Levels defines a single binary format that can be read and written from C#, Python, TypeScript, and C++. A shared `.fbs` schema file ensures all languages produce byte-identical output. + +## Shared Schema + +The canonical schema lives at `cross-language/schema.fbs`: + +```fbs +namespace Levels.Schema; + +struct Record { + observed_time:int64; + write_timestamp:int64; + price_stream_id:int64; + price:int64; + quantity:int64; + _reserved:uint16; + record_type:uint8; + record_side:uint8; + sequence:uint32; + level:uint16; + flags:uint16; + crc32:uint32; + order_id:byte[32]; +} +``` + +## Code Generation + +Generate readers/writers for each language using the CLI: + +```bash +# Python +dotnet run --project src/Levels.Cli -- codegen cross-language/schema.fbs \ + --output cross-language/python --language python + +# TypeScript +dotnet run --project src/Levels.Cli -- codegen cross-language/schema.fbs \ + --output cross-language/typescript --language typescript + +# C++ +dotnet run --project src/Levels.Cli -- codegen cross-language/schema.fbs \ + --output cross-language/cpp --language cpp +``` + +## Generated Output by Language + +### Python (`record.py`) + +Generated module containing: +- Constants: `HEADER_SIZE`, `FOOTER_SIZE`, `CORE_RECORD_SIZE`, `RECORD_SIZE`, `EXTENSION_SIZE`, `FORMAT_VERSION`, `SCHEMA_ID` +- `EXTENSION_FIELDS` list: `[(name, offset_from_core, size), ...]` +- `compute_crc32()` -- CRC32 (IEEE polynomial) over core[0:52] + extension bytes +- `pack_header()` -- writes 128-byte header using `struct.pack()` +- `pack_record()` -- writes core + extension fields with CRC +- `pack_footer()` -- writes 64-byte footer + +Uses `struct.pack()` with format string ` + + + +``` + +The Roslyn generator emits `Record`, `RecordAccessor`, and `RecordWriter` types at build time. + +**Option B: CLI** + +```bash +dotnet run --project src/Levels.Cli -- codegen schema.fbs --output ./Generated +``` + +This produces `Record.g.cs` containing the same types. + +### 3. Register Services + +```csharp +using Levels.Hosting; + +var builder = Host.CreateDefaultBuilder(args) + .ConfigureServices(services => + { + services.AddLevels(opts => + { + opts.DataPath = "/data/levels"; + opts.PriceScale = 2; // e.g. 10350042 = $103,500.42 + opts.QuantityScale = 8; // e.g. satoshi precision + }); + + services.AddLevelsDataSink(); + services.AddHostedService(); + }); +``` + +`AddLevels()` automatically reads `SchemaId` and `RecordSize` from the generated `ISchemaDescriptor` implementation. It registers: + +- `LevelsOptions` (singleton) +- `FileIndex` (loaded from disk on startup) +- `QueryLayer` and `QueryConfig` +- `DataFlowBus` with handlers (file index sync, compaction, promotion) +- `SinkConfig` and `PriceStreamSink` +- OpenTelemetry (if enabled) + +`AddLevelsDataSink()` registers `IDataSink` backed by a `DataSinkAdapter` that converts your typed records to `RawMarketEvent`. + +### 4. Write Records + +Inject `IDataSink` and call `WriteAsync`: + +```csharp +sealed class MyIngestionService : BackgroundService +{ + private readonly IDataSink _sink; + + public MyIngestionService(IDataSink sink) => _sink = sink; + + protected override async Task ExecuteAsync(CancellationToken ct) + { + await Task.Yield(); // let the host finish starting + + var orderId = new byte[32]; + Encoding.UTF8.GetBytes("order-001", orderId); + + await _sink.WriteAsync(new Record( + Venue: "binance", + Symbol: "BTCUSDT", + ObservedTime: DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), + Price: 10_350_042, // $103,500.42 with PriceScale=2 + Quantity: 1_50000000, // 1.5 BTC with QuantityScale=8 + RecordType: RecordType.Snap, + RecordSide: RecordSide.Bid, + OrderId: orderId), ct); + } +} +``` + +## Configuration + +All options are set via the `Action` callback. See the [Configuration Reference](configuration.md) for the full list. + +Key options for embedded mode: + +| Option | Default | Description | +|--------|---------|-------------| +| `DataPath` | required | Directory for data files | +| `PriceScale` | 0 | Decimal places for price fields | +| `QuantityScale` | 0 | Decimal places for quantity fields | +| `RolloverSize` | 256 MB | Max file size before rollover | +| `RolloverInterval` | 1 hour | Time-based rollover | +| `BackpressureLimit` | 8192 | Channel capacity before throttling | +| `EnableCompaction` | true | Automatic RAW -> AGG compaction | +| `EnablePeriodPromotion` | true | Automatic AGG -> PERIOD promotion | + +## Enabling Compaction + +Compaction is enabled by default. Configure the window and snapshot intervals: + +```csharp +services.AddLevels(opts => +{ + opts.DataPath = "/data/levels"; + opts.CompactionWindow = TimeSpan.FromHours(1); + opts.SyntheticSnapIntervalDeltas = 1000; // snapshot every 1000 deltas + opts.RetentionWindow = TimeSpan.FromDays(7); + opts.WindowGracePeriod = TimeSpan.FromMinutes(5); +}); +``` + +## OpenTelemetry + +Enable metrics and/or tracing: + +```csharp +services.AddLevels(opts => +{ + opts.DataPath = "/data/levels"; + opts.EnableMetrics = true; + opts.EnableTracing = true; + opts.OtlpEndpoint = "http://localhost:4317"; +}); +``` + +The `Levels` meter and activity source are registered automatically. See [Architecture - Observability](architecture.md#observability) for the full metric list. + +## Querying Data + +Resolve `QueryLayer` from DI to query stored data: + +```csharp +var queryLayer = host.Services.GetRequiredService(); +var fileIndex = host.Services.GetRequiredService(); + +// List all known streams +var streams = fileIndex.GetAllStreams(); + +// Reconstruct L2 orderbook +var streamId = PriceStreamId.FromVenueSymbol("binance", "BTCUSDT"); +var projection = new OrderbookProjection(queryLayer); +var l2 = projection.ProjectL2(streamId, startTime: 0, endTime: long.MaxValue); +``` + +## Complete Example + +See [`samples/CryptoExchangeSample`](../samples/CryptoExchangeSample/) for a full working example with synthetic data, and [`samples/BinanceLiveSample`](../samples/BinanceLiveSample/) for live Binance WebSocket ingestion. diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 0000000..35568be --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,104 @@ +# Quickstart + +Get Levels running in under five minutes. + +## Prerequisites + +- [.NET 10.0 SDK](https://dotnet.microsoft.com/download/dotnet/10.0) (preview) + +Verify your installation: + +```bash +dotnet --version # should show 10.0.x +``` + +## Build + +```bash +git clone https://github.com/DanielBunting/Levels.git +cd Levels +dotnet build +``` + +## Run the Crypto Exchange Sample + +The simplest way to see Levels in action. This sample generates synthetic orderbook data (BTCUSDT and ETHUSDT) and writes it to disk. + +```bash +dotnet run --project samples/CryptoExchangeSample +``` + +Output: + +``` +Levels Crypto Exchange Sample +Data path: /tmp/levels-crypto-sample/a1b2c3d4 + +[BTCUSDT] Sending initial SNAP (10 bid + 10 ask levels)... +[BTCUSDT] Streaming 40 DELTA updates... + ... 25 events written + ... 50 events written +[ETHUSDT] Sending initial SNAP (10 bid + 10 ask levels)... +[ETHUSDT] Streaming 40 DELTA updates... + ... 75 events written + ... 100 events written +Ingestion complete: 120 events written. + +Done. Files written: + binance//.raw +``` + +## Inspect the Output + +Use the CLI to examine the binary files: + +```bash +# Inspect file header and footer +dotnet run --project src/Levels.Cli -- inspect /tmp/levels-crypto-sample//binance//.raw + +# Include individual records +dotnet run --project src/Levels.Cli -- inspect .raw --records +``` + +The inspect command shows the file version, schema ID, price/quantity scales, record count, and time range. + +## Export Data + +Export the written data to CSV: + +```bash +dotnet run --project src/Levels.Cli -- export /tmp/levels-crypto-sample/ \ + --venue binance --stream BTCUSDT --format csv --output btcusdt.csv +``` + +Supported formats: `csv`, `parquet`, `avro`. + +## Run the Binance Live Sample + +To stream real market data from Binance (requires internet connectivity): + +```bash +dotnet run --project samples/BinanceLiveSample +``` + +Press `Ctrl+C` to stop. The sample connects to the Binance WebSocket API and writes live orderbook updates. + +## Run Tests + +```bash +dotnet test +``` + +## Run Benchmarks + +```bash +dotnet run --project tests/Levels.Benchmarks --configuration Release +``` + +## Next Steps + +- [Embedded Mode](embedded-mode.md) -- integrate Levels into your own C# application +- [Server Mode](server-mode.md) -- run Levels as a standalone TCP server +- [Web API](web-api.md) -- query data via REST endpoints +- [Schemas](schemas.md) -- define your own data schema +- [Configuration](configuration.md) -- tune all options diff --git a/docs/schemas.md b/docs/schemas.md new file mode 100644 index 0000000..fdc04b5 --- /dev/null +++ b/docs/schemas.md @@ -0,0 +1,247 @@ +# Schemas + +Levels uses a FlatBuffers-inspired `.fbs` schema format to define record layouts. A single schema file is the source of truth for the binary format across all languages. + +## Schema Format + +A schema defines a single `struct` with typed fields: + +```fbs +namespace MyApp.Schema; + +struct Record { + // Core fields (56 bytes) -- managed by Levels + observed_time:int64; + write_timestamp:int64; + price_stream_id:int64; + price:int64; + quantity:int64; + _reserved:uint16; + record_type:uint8; + record_side:uint8; + sequence:uint32; + level:uint16; + flags:uint16; + crc32:uint32; + + // Extension fields (bytes 56+) -- user-defined + order_id:byte[32]; +} +``` + +## Core vs Extension Fields + +The first 56 bytes are **core fields** with fixed layout. Levels manages these internally (write timestamps, CRC, sequence numbers, etc.). Your code provides `ObservedTime`, `Price`, `Quantity`, `RecordType`, and `RecordSide`. + +Fields after `crc32` are **extension fields**. These are serialized by generated `WriteExtension()` code and readable via the generated `RecordAccessor`. + +## Type Mappings + +| FBS Type | C# | Python | TypeScript | C++ | Size | +|----------|-----|--------|------------|-----|------| +| int64 | long | struct `q` | BigInt64 | int64_t | 8 | +| uint64 | ulong | struct `Q` | BigUint64 | uint64_t | 8 | +| int32 | int | struct `i` | Int32 | int32_t | 4 | +| uint32 | uint | struct `I` | Uint32 | uint32_t | 4 | +| int16 | short | struct `h` | Int16 | int16_t | 2 | +| uint16 | ushort | struct `H` | Uint16 | uint16_t | 2 | +| int8 | sbyte | struct `b` | Int8 | int8_t | 1 | +| uint8 | byte | struct `B` | Uint8 | uint8_t | 1 | +| byte[N] | byte[] (fixed) | bytes | Buffer | uint8_t[N] | N | + +## Schema ID + +Each schema has a unique ID computed as the FNV-1a hash of the entire `.fbs` file text. This means any change to the file -- including comments or whitespace -- produces a new schema ID. The ID is: + +- Written into every file header (offset 12) +- Validated during TCP handshake (server mode) +- Available as `Record.SchemaId` in generated code + +## Code Generation + +There are two code generation paths: + +### Build-Time: Roslyn SourceGen + +The `Levels.SourceGen` project is a Roslyn incremental generator that watches `.fbs` files added as `AdditionalFiles`: + +```xml + + + + +``` + +Types are generated at compile time with no separate build step. + +### CLI: `levels codegen` + +Generate code for any supported language: + +```bash +# C# (default) +dotnet run --project src/Levels.Cli -- codegen schema.fbs --output ./Generated + +# Python +dotnet run --project src/Levels.Cli -- codegen schema.fbs --output ./python --language python + +# TypeScript +dotnet run --project src/Levels.Cli -- codegen schema.fbs --output ./typescript --language typescript + +# C++ +dotnet run --project src/Levels.Cli -- codegen schema.fbs --output ./cpp --language cpp +``` + +## Generated Types + +### Record (C#) + +A `record struct` implementing both `ISchemaDescriptor` and `ISchemaEvent`: + +```csharp +public record struct Record( + string Venue, + string Symbol, + long ObservedTime, + long Price, + long Quantity, + RecordType RecordType, + RecordSide RecordSide, + ReadOnlyMemory OrderId = default) : ISchemaDescriptor, ISchemaEvent +{ + public static uint SchemaId => 0xE4E8A928u; + public static int RecordSize => 88; // 56 core + 32 extension + + public void WriteExtension(Span destination) + { + OrderId.Span.CopyTo(destination.Slice(0, 32)); + } +} +``` + +### RecordAccessor (C#) + +A `ref struct` for zero-copy reading from a `ReadOnlySpan`: + +```csharp +public readonly ref struct RecordAccessor +{ + public long ObservedTime { get; } + public long Price { get; } + public long Quantity { get; } + public RecordType Type { get; } + public RecordSide Side { get; } + // ... all fields including extensions + public ReadOnlySpan OrderId { get; } +} +``` + +### RecordWriter (C#) + +A static helper for writing records directly to a `Span`. + +## Interfaces + +### ISchemaDescriptor + +```csharp +public interface ISchemaDescriptor +{ + static abstract uint SchemaId { get; } // FNV-1a hash + static abstract int RecordSize { get; } // total bytes per record +} +``` + +Used by `AddLevels()` to auto-configure schema ID and record size. + +### ISchemaEvent + +```csharp +public interface ISchemaEvent +{ + string Venue { get; } + string Symbol { get; } + long ObservedTime { get; } + long Price { get; } + long Quantity { get; } + RecordType RecordType { get; } + RecordSide RecordSide { get; } + void WriteExtension(Span destination); +} +``` + +Implemented by generated record types. The sink calls `WriteExtension()` to serialize extension fields into the binary record. + +## Example Schemas + +### Core Only (56 bytes) + +No extension fields -- just the standard orderbook record: + +```fbs +namespace Basic; + +struct Record { + observed_time:int64; + write_timestamp:int64; + price_stream_id:int64; + price:int64; + quantity:int64; + _reserved:uint16; + record_type:uint8; + record_side:uint8; + sequence:uint32; + level:uint16; + flags:uint16; + crc32:uint32; +} +``` + +### With OrderId (88 bytes) + +Adds a 32-byte fixed-size order identifier: + +```fbs +namespace Exchange; + +struct Record { + observed_time:int64; + write_timestamp:int64; + price_stream_id:int64; + price:int64; + quantity:int64; + _reserved:uint16; + record_type:uint8; + record_side:uint8; + sequence:uint32; + level:uint16; + flags:uint16; + crc32:uint32; + order_id:byte[32]; +} +``` + +### With Numeric Extension (64 bytes) + +Adds an 8-byte integer field (e.g., Binance's `lastUpdateId`): + +```fbs +namespace Binance; + +struct Record { + observed_time:int64; + write_timestamp:int64; + price_stream_id:int64; + price:int64; + quantity:int64; + _reserved:uint16; + record_type:uint8; + record_side:uint8; + sequence:uint32; + level:uint16; + flags:uint16; + crc32:uint32; + last_update_id:int64; +} +``` diff --git a/docs/server-mode.md b/docs/server-mode.md new file mode 100644 index 0000000..a71e7ca --- /dev/null +++ b/docs/server-mode.md @@ -0,0 +1,149 @@ +# Server Mode + +Run Levels as a standalone TCP server that accepts connections from clients in any language. The server loads a schema plugin at startup and validates incoming data against it. + +## When to Use + +- You have producers in multiple languages (Python, C++, TypeScript) +- You want a centralized ingestion point for market data +- You need to decouple producers from the storage engine + +## Configuration + +The server is configured via a YAML file. Specify the path with `--config=`, the `LEVELS_CONFIG` environment variable, or default to `levels-server.yaml` in the working directory. + +### Full Configuration Reference + +```yaml +# Base directory for all data files +DataPath: ./data + +# TCP port to listen on +Port: 5050 + +# Schema plugin configuration +Schema: + # Path to the .NET assembly containing the schema type + DllPath: path/to/MySchema.dll + # Fully qualified type name implementing ISchemaDescriptor + TypeName: MyApp.Schema.Record + +# Levels engine options +Levels: + PriceScale: 2 + QuantityScale: 8 + RolloverSize: 268435456 # 256 MB + RolloverInterval: "01:00:00" # 1 hour + BackpressureLimit: 8192 + EnableCompaction: true + EnablePeriodPromotion: true + CompactionWindow: "01:00:00" # 1 hour + +# OpenTelemetry configuration +Telemetry: + EnableMetrics: false + OtlpEndpoint: null + EnableTracing: false +``` + +### Configuration Defaults + +| Field | Default | +|-------|---------| +| DataPath | `./data` | +| Port | `5050` | +| Levels.PriceScale | `0` | +| Levels.QuantityScale | `0` | +| Levels.RolloverSize | `268435456` (256 MB) | +| Levels.RolloverInterval | `01:00:00` | +| Levels.BackpressureLimit | `8192` | +| Levels.EnableCompaction | `true` | +| Levels.EnablePeriodPromotion | `false` | +| Levels.CompactionWindow | `01:00:00` | +| Telemetry.EnableMetrics | `false` | +| Telemetry.EnableTracing | `false` | + +## Schema Plugin + +The server loads schema metadata from an external .NET assembly at runtime. The assembly must contain a type implementing `ISchemaDescriptor`: + +```csharp +public interface ISchemaDescriptor +{ + static abstract uint SchemaId { get; } + static abstract int RecordSize { get; } +} +``` + +Build the schema assembly from your generated code: + +```bash +# Generate the schema code +dotnet run --project src/Levels.Cli -- codegen schema.fbs --output MySchema/ + +# Build the schema DLL +dotnet build MySchema/ +``` + +The `SchemaPluginLoader` uses reflection to read `SchemaId` and `RecordSize` from the type's static properties. These values are written into file headers and validated during client handshakes. + +## Starting the Server + +```bash +# With explicit config path +dotnet run --project src/Levels.Server -- --config=levels-server.yaml + +# With environment variable +export LEVELS_CONFIG=/etc/levels/server.yaml +dotnet run --project src/Levels.Server +``` + +Output: + +``` +Loaded schema: MyApp.Schema.Record (SchemaId=0xE4E8A928, RecordSize=88) +Levels server starting on port 5050 +``` + +## Protocol + +The server uses a binary frame-based protocol over TCP. + +**Handshake:** Clients must send their schema ID during connection. The server validates it matches the loaded schema. Mismatched schemas are rejected. + +**Frame format:** Each frame contains a message type byte followed by the payload. The `Levels.Protocol` project defines `FrameReader` and `FrameWriter` for serialization. + +**Acknowledgements:** The server sends ACKs every 1000 records or 100ms (whichever comes first), confirming durable writes. + +## Client Library + +The `Levels.Client` project provides a .NET TCP client: + +```csharp +using Levels.Client; + +var client = new LevelsClient("localhost", 5050, schemaId: 0xE4E8A928); +await client.ConnectAsync(); + +// Write records as raw bytes +await client.WriteAsync(recordBytes); + +await client.DisconnectAsync(); +``` + +For non-.NET clients, use the [cross-language code generators](cross-language.md) to produce readers/writers in Python, TypeScript, or C++ that speak the same binary format. + +## Export Support + +The server registers CSV, Parquet, and Avro export adapters. Query and export operations are available through the connected protocol or by adding the [Web API](web-api.md) alongside the TCP server. + +## Infrastructure + +The server registers the same Levels infrastructure as embedded mode: + +- `FileIndex` -- loaded from disk at startup +- `QueryLayer` -- resolves queries against the file index +- `DataFlowBus` -- connects writes to compaction and promotion +- `PriceStreamSink` -- manages file I/O with rollover +- `ExportPipeline` -- CSV, Parquet, Avro adapters +- `TcpServer` -- hosted service managing client connections diff --git a/docs/web-api.md b/docs/web-api.md new file mode 100644 index 0000000..3016b4d --- /dev/null +++ b/docs/web-api.md @@ -0,0 +1,132 @@ +# Web API + +Run Levels as an ASP.NET Core REST API for querying stored data and exporting to various formats. Includes a Prometheus metrics scraping endpoint. + +## When to Use + +- You need HTTP access to orderbook projections and data exports +- You want a Prometheus-compatible metrics endpoint +- You have existing data on disk and need a query layer + +## Configuration + +The web host uses standard ASP.NET Core configuration. Set Levels options in `appsettings.json`: + +```json +{ + "Levels": { + "DataPath": "./data", + "EnableMetrics": true, + "OtlpEndpoint": "http://localhost:4317", + "EnableTracing": false + } +} +``` + +Or via environment variables: + +```bash +export Levels__DataPath=./data +export Levels__EnableMetrics=true +``` + +## Starting the Web API + +```bash +dotnet run --project src/Levels.Web +``` + +By default, the ASP.NET Core host listens on `http://localhost:5000`. + +## Endpoints + +### Health Check + +``` +GET /api/health +``` + +Response: + +```json +{ + "status": "healthy", + "timestamp": "2025-01-15T10:30:00Z" +} +``` + +### List Streams + +``` +GET /api/streams +``` + +Returns all known price streams from the file index. + +### L2 Orderbook + +``` +GET /api/streams/{venue}/{symbol}/book +GET /api/streams/{venue}/{symbol}/book?excludeOwner=true +``` + +Reconstructs the full L2 orderbook (all bid/ask levels) for the given venue and symbol. + +Example: + +``` +GET /api/streams/binance/BTCUSDT/book +``` + +The optional `excludeOwner` parameter filters out records with the `IsOwner` flag set. + +### L1 Orderbook + +``` +GET /api/streams/{venue}/{symbol}/book/l1 +GET /api/streams/{venue}/{symbol}/book/l1?excludeOwner=true +``` + +Returns only the best bid and best ask. + +### Export + +``` +GET /api/streams/{venue}/{symbol}/export?format=csv +GET /api/streams/{venue}/{symbol}/export?format=parquet +GET /api/streams/{venue}/{symbol}/export?format=avro +``` + +Exports all data for the given stream. The response is streamed with the appropriate content type: + +| Format | Content-Type | Filename | +|--------|-------------|----------| +| csv | `text/csv` | `{symbol}.csv` | +| parquet | `application/octet-stream` | `{symbol}.parquet` | +| avro | `application/octet-stream` | `{symbol}.avro` | + +The `Content-Disposition` header is set to `attachment` with the filename. + +### Prometheus Metrics + +``` +GET /metrics +``` + +Prometheus-compatible scraping endpoint. Exposes all Levels metrics from the `Levels` OpenTelemetry meter. This endpoint is always available regardless of the `EnableMetrics` option (which controls OTLP export). + +## Infrastructure + +The web host registers: + +- `LevelsOptions` with the non-generic `AddLevels()` overload +- `FileIndex`, `QueryLayer`, `DataFlowBus`, `PriceStreamSink` (same as embedded mode) +- `CsvExportAdapter`, `ParquetExportAdapter`, `AvroExportAdapter` +- `ExportPipeline` (wires query layer to export adapters) +- OpenTelemetry Prometheus exporter + +The API endpoints are mapped via `app.MapLevelsApi()` which registers all routes under `/api`. + +## Combining with Server Mode + +You can run both the TCP server and REST API in the same process by adding the web endpoints to the server's host builder. The `LevelsEndpoints.MapLevelsApi()` extension method works with any `IEndpointRouteBuilder`. diff --git a/samples/BinanceLiveSample/BinanceLiveSample.csproj b/samples/BinanceLiveSample/BinanceLiveSample.csproj new file mode 100644 index 0000000..5addf50 --- /dev/null +++ b/samples/BinanceLiveSample/BinanceLiveSample.csproj @@ -0,0 +1,20 @@ + + + + Exe + net10.0 + enable + enable + preview + false + + + + + + + + + + + diff --git a/samples/BinanceLiveSample/BinanceWebSocketService.cs b/samples/BinanceLiveSample/BinanceWebSocketService.cs new file mode 100644 index 0000000..3c8b92d --- /dev/null +++ b/samples/BinanceLiveSample/BinanceWebSocketService.cs @@ -0,0 +1,157 @@ +using System.Net.WebSockets; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Hosting; +using Levels.Core; +using Levels.Core.Format; + +namespace BinanceLiveSample; + +sealed class BinanceWebSocketService : BackgroundService +{ + private const string StreamUrl = + "wss://stream.binance.com:9443/stream?streams=btcusdt@depth10@1000ms/ethusdt@depth10@1000ms"; + + private const int PriceScale = 2; + private const int QuantityScale = 8; + + private readonly IDataSink _sink; + private readonly IHostApplicationLifetime _lifetime; + private readonly TimeSpan _runDuration; + + public BinanceWebSocketService( + IDataSink sink, + IHostApplicationLifetime lifetime) + { + _sink = sink; + _lifetime = lifetime; + _runDuration = TimeSpan.FromSeconds(60); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await Task.Yield(); + + using var timeout = new CancellationTokenSource(_runDuration); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken, timeout.Token); + var ct = linked.Token; + + using var ws = new ClientWebSocket(); + Console.WriteLine($"Connecting to Binance WebSocket..."); + Console.WriteLine($"URL: {StreamUrl}"); + Console.WriteLine($"Run duration: {_runDuration.TotalSeconds}s"); + Console.WriteLine(); + + try + { + await ws.ConnectAsync(new Uri(StreamUrl), stoppingToken); + Console.WriteLine("Connected. Receiving live orderbook snapshots..."); + Console.WriteLine(); + + var buffer = new byte[16 * 1024]; + var messageBuffer = new MemoryStream(); + var totalRecords = 0; + + while (!ct.IsCancellationRequested) + { + messageBuffer.SetLength(0); + + WebSocketReceiveResult result; + do + { + result = await ws.ReceiveAsync(buffer, ct); + if (result.MessageType == WebSocketMessageType.Close) + { + Console.WriteLine("Server closed connection."); + _lifetime.StopApplication(); + return; + } + messageBuffer.Write(buffer, 0, result.Count); + } while (!result.EndOfMessage); + + var json = Encoding.UTF8.GetString(messageBuffer.GetBuffer(), 0, (int)messageBuffer.Length); + var msg = JsonSerializer.Deserialize(json); + if (msg?.Stream is null || msg.Data is null) + continue; + + // Stream name is e.g. "btcusdt@depth10@1000ms" + var symbol = msg.Stream.Split('@')[0].ToUpperInvariant(); + var observedTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var lastUpdateId = msg.Data.LastUpdateId; + + var bidCount = msg.Data.Bids?.Length ?? 0; + var askCount = msg.Data.Asks?.Length ?? 0; + + for (var i = 0; i < bidCount; i++) + { + var level = msg.Data.Bids![i]; + var price = ScaledLong.Parse(level[0], PriceScale); + var qty = ScaledLong.Parse(level[1], QuantityScale); + + await _sink.WriteAsync(new BinanceRecord( + "binance", symbol, observedTime, price, qty, + RecordType.Snap, RecordSide.Bid, + LastUpdateId: lastUpdateId), ct); + } + + for (var i = 0; i < askCount; i++) + { + var level = msg.Data.Asks![i]; + var price = ScaledLong.Parse(level[0], PriceScale); + var qty = ScaledLong.Parse(level[1], QuantityScale); + + await _sink.WriteAsync(new BinanceRecord( + "binance", symbol, observedTime, price, qty, + RecordType.Snap, RecordSide.Ask, + LastUpdateId: lastUpdateId), ct); + } + + totalRecords += bidCount + askCount; + + var bestBid = bidCount > 0 ? msg.Data.Bids![0][0] : "N/A"; + var bestAsk = askCount > 0 ? msg.Data.Asks![0][0] : "N/A"; + Console.WriteLine( + $"[{symbol}] {bidCount} bids + {askCount} asks | " + + $"best bid={bestBid} ask={bestAsk} | total records: {totalRecords}"); + } + + Console.WriteLine(); + Console.WriteLine($"Duration elapsed. Total records written: {totalRecords}"); + + if (ws.State == WebSocketState.Open) + { + await ws.CloseAsync(WebSocketCloseStatus.NormalClosure, "Done", CancellationToken.None); + } + } + catch (OperationCanceledException) when (timeout.IsCancellationRequested) + { + Console.WriteLine(); + Console.WriteLine("Duration elapsed. Shutting down..."); + + if (ws.State == WebSocketState.Open) + { + try { await ws.CloseAsync(WebSocketCloseStatus.NormalClosure, "Done", CancellationToken.None); } + catch { /* best effort */ } + } + } + catch (Exception ex) + { + Console.WriteLine(); + Console.WriteLine($"Error: {ex.Message}"); + } + finally + { + _lifetime.StopApplication(); + } + } + + private sealed record CombinedStreamMessage( + [property: JsonPropertyName("stream")] string? Stream, + [property: JsonPropertyName("data")] DepthData? Data); + + private sealed record DepthData( + [property: JsonPropertyName("lastUpdateId")] long LastUpdateId, + [property: JsonPropertyName("bids")] string[][]? Bids, + [property: JsonPropertyName("asks")] string[][]? Asks); +} diff --git a/samples/BinanceLiveSample/Generated/Record.g.cs b/samples/BinanceLiveSample/Generated/Record.g.cs new file mode 100644 index 0000000..22be3c5 --- /dev/null +++ b/samples/BinanceLiveSample/Generated/Record.g.cs @@ -0,0 +1,121 @@ +// +#nullable enable + +using System; +using System.Buffers.Binary; +using Levels.Core; +using Levels.Core.Format; + +namespace BinanceLiveSample +{ + public record struct BinanceRecord( + string Venue, + string Symbol, + long ObservedTime, + long Price, + long Quantity, + RecordType RecordType, + RecordSide RecordSide, + long LastUpdateId = 0) : ISchemaDescriptor, ISchemaEvent + { + public static uint SchemaId => 0xF7F62892u; + public static int RecordSize => 64; + + public void WriteExtension(Span destination) + { + BinaryPrimitives.WriteInt64LittleEndian(destination.Slice(0), LastUpdateId); + } + } + + public readonly ref struct RecordAccessor + { + private readonly ReadOnlySpan _span; + + public RecordAccessor(ReadOnlySpan span) + { + _span = span; + } + + public long ObservedTime => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(0)); + public long WriteTimestamp => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(8)); + public long PriceStreamId => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(16)); + public long Price => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(24)); + public long Quantity => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(32)); + public ushort Reserved => BinaryPrimitives.ReadUInt16LittleEndian(_span.Slice(40)); + public byte RecordType => _span[42]; + public byte RecordSide => _span[43]; + public uint Sequence => BinaryPrimitives.ReadUInt32LittleEndian(_span.Slice(44)); + public ushort Level => BinaryPrimitives.ReadUInt16LittleEndian(_span.Slice(48)); + public ushort Flags => BinaryPrimitives.ReadUInt16LittleEndian(_span.Slice(50)); + public uint Crc32 => BinaryPrimitives.ReadUInt32LittleEndian(_span.Slice(52)); + public long LastUpdateId => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(56)); + } + + public readonly ref struct OrderSnapRecord + { + private readonly ReadOnlySpan _span; + + public OrderSnapRecord(ReadOnlySpan span) + { + _span = span; + } + + public long ObservedTime => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(0)); + public long WriteTimestamp => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(8)); + public long PriceStreamId => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(16)); + public long Price => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(24)); + public long Quantity => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(32)); + public ushort Reserved => BinaryPrimitives.ReadUInt16LittleEndian(_span.Slice(40)); + public byte RecordType => _span[42]; + public byte RecordSide => _span[43]; + public uint Sequence => BinaryPrimitives.ReadUInt32LittleEndian(_span.Slice(44)); + public ushort Level => BinaryPrimitives.ReadUInt16LittleEndian(_span.Slice(48)); + public ushort Flags => BinaryPrimitives.ReadUInt16LittleEndian(_span.Slice(50)); + public uint Crc32 => BinaryPrimitives.ReadUInt32LittleEndian(_span.Slice(52)); + public long LastUpdateId => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(56)); + } + + public readonly ref struct OrderDeltaRecord + { + private readonly ReadOnlySpan _span; + + public OrderDeltaRecord(ReadOnlySpan span) + { + _span = span; + } + + public long ObservedTime => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(0)); + public long WriteTimestamp => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(8)); + public long PriceStreamId => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(16)); + public long Price => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(24)); + public long Quantity => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(32)); + public ushort Reserved => BinaryPrimitives.ReadUInt16LittleEndian(_span.Slice(40)); + public byte RecordType => _span[42]; + public byte RecordSide => _span[43]; + public uint Sequence => BinaryPrimitives.ReadUInt32LittleEndian(_span.Slice(44)); + public ushort Level => BinaryPrimitives.ReadUInt16LittleEndian(_span.Slice(48)); + public ushort Flags => BinaryPrimitives.ReadUInt16LittleEndian(_span.Slice(50)); + public uint Crc32 => BinaryPrimitives.ReadUInt32LittleEndian(_span.Slice(52)); + public long LastUpdateId => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(56)); + } + + public static class RecordWriter + { + public static void WriteRecord(Span span, long observedTime, long writeTimestamp, long priceStreamId, long price, long quantity, ushort reserved, byte recordType, byte recordSide, uint sequence, ushort level, ushort flags, uint crc32, long lastUpdateId) + { + BinaryPrimitives.WriteInt64LittleEndian(span.Slice(0), observedTime); + BinaryPrimitives.WriteInt64LittleEndian(span.Slice(8), writeTimestamp); + BinaryPrimitives.WriteInt64LittleEndian(span.Slice(16), priceStreamId); + BinaryPrimitives.WriteInt64LittleEndian(span.Slice(24), price); + BinaryPrimitives.WriteInt64LittleEndian(span.Slice(32), quantity); + BinaryPrimitives.WriteUInt16LittleEndian(span.Slice(40), reserved); + span[42] = recordType; + span[43] = recordSide; + BinaryPrimitives.WriteUInt32LittleEndian(span.Slice(44), sequence); + BinaryPrimitives.WriteUInt16LittleEndian(span.Slice(48), level); + BinaryPrimitives.WriteUInt16LittleEndian(span.Slice(50), flags); + BinaryPrimitives.WriteUInt32LittleEndian(span.Slice(52), crc32); + BinaryPrimitives.WriteInt64LittleEndian(span.Slice(56), lastUpdateId); + } + } +} diff --git a/samples/BinanceLiveSample/Program.cs b/samples/BinanceLiveSample/Program.cs new file mode 100644 index 0000000..15fda26 --- /dev/null +++ b/samples/BinanceLiveSample/Program.cs @@ -0,0 +1,39 @@ +using BinanceLiveSample; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Levels.Hosting; + + +var dataPath = Path.Combine(Path.GetTempPath(), "levels-binance-live", Guid.NewGuid().ToString("N")[..8]); +Directory.CreateDirectory(dataPath); + +Console.WriteLine("Levels Binance Live Sample"); +Console.WriteLine($"Data path: {dataPath}"); +Console.WriteLine(); + +var builder = Host.CreateDefaultBuilder(args) + .ConfigureLogging(log => log.SetMinimumLevel(LogLevel.Warning)) + .ConfigureServices(services => + { + services.AddLevels(opts => + { + opts.DataPath = dataPath; + opts.PriceScale = 2; + opts.QuantityScale = 8; + opts.RolloverSize = 16 * 1024 * 1024; + opts.EnableCompaction = false; + opts.EnablePeriodPromotion = false; + }); + + services.AddLevelsDataSink(); + services.AddHostedService(); + }); + +using var host = builder.Build(); +await host.RunAsync(); + +Console.WriteLine(); +Console.WriteLine("Done. Files written:"); +foreach (var file in Directory.EnumerateFiles(dataPath, "*", SearchOption.AllDirectories)) + Console.WriteLine($" {Path.GetRelativePath(dataPath, file)}"); diff --git a/samples/BinanceLiveSample/ScaledLong.cs b/samples/BinanceLiveSample/ScaledLong.cs new file mode 100644 index 0000000..6c723c0 --- /dev/null +++ b/samples/BinanceLiveSample/ScaledLong.cs @@ -0,0 +1,45 @@ +namespace BinanceLiveSample; + +internal static class ScaledLong +{ + /// + /// Converts a decimal string (e.g. "103500.42") to a scaled long (10350042 at scale=2) + /// without floating-point arithmetic. + /// + public static long Parse(ReadOnlySpan value, int scale) + { + var dotIndex = value.IndexOf('.'); + + if (dotIndex < 0) + { + // No decimal point — multiply by 10^scale + var whole = long.Parse(value); + for (var i = 0; i < scale; i++) + whole *= 10; + return whole; + } + + var wholePart = long.Parse(value[..dotIndex]); + var fracSpan = value[(dotIndex + 1)..]; + var fracDigits = fracSpan.Length; + + long result = wholePart; + for (var i = 0; i < scale; i++) + result *= 10; + + if (fracDigits <= scale) + { + // Fewer fractional digits than scale — parse and pad with zeros + var frac = long.Parse(fracSpan); + for (var i = 0; i < scale - fracDigits; i++) + frac *= 10; + return result >= 0 ? result + frac : result - frac; + } + else + { + // More fractional digits than scale — truncate + var frac = long.Parse(fracSpan[..scale]); + return result >= 0 ? result + frac : result - frac; + } + } +} diff --git a/samples/BinanceLiveSample/schema.fbs b/samples/BinanceLiveSample/schema.fbs new file mode 100644 index 0000000..70caac9 --- /dev/null +++ b/samples/BinanceLiveSample/schema.fbs @@ -0,0 +1,15 @@ +namespace BinanceLiveSample; + +// Binance partial book depth record. +// Core fields are managed by Levels. Extensions are venue-specific. +struct BinanceRecord { + venue:string; + symbol:string; + observed_time:int64; + write_timestamp:int64; + price:int64; + quantity:int64; + record_type:uint8; + record_side:uint8; + last_update_id:int64; +} diff --git a/samples/BinancePythonSample/README.md b/samples/BinancePythonSample/README.md new file mode 100644 index 0000000..c46d249 --- /dev/null +++ b/samples/BinancePythonSample/README.md @@ -0,0 +1,90 @@ +# Binance Python Sample + +A Python sample that connects to the Binance public websocket, receives BTCUSDT depth-10 orderbook snapshots, and sends records to the Levels TCP server. + +This is the Python equivalent of the C# `BinanceLiveSample`. + +## Prerequisites + +- Python 3.9+ +- .NET 10.0 SDK (for the server and code generation) + +## Setup + +### 1. Generate the record module from the schema + +From the repository root: + +```bash +dotnet run --project src/Levels.Cli -- codegen samples/BinancePythonSample/schema.fbs --language python --output samples/BinancePythonSample/ +``` + +This produces `record.py` with constants (schema ID, record size) used by the sample. + +### 2. Build the schema DLL + +The Levels server needs a compiled .NET assembly containing the schema type. Build the BinanceLiveSample (which shares the same schema): + +```bash +dotnet build samples/BinanceLiveSample +``` + +### 3. Install Python dependencies + +```bash +cd samples/BinancePythonSample +pip install -r requirements.txt +``` + +Dependencies: +- `websockets` — async websocket client for the Binance stream +- `xxhash` — computes the PriceStreamId (xxhash64 of venue + symbol) + +## Running + +### Step 1: Start the Levels server + +From the repository root: + +```bash +dotnet run --project src/Levels.Server -- --config=samples/BinancePythonSample/levels-server.yaml +``` + +The server starts on port 5050 and writes data to `/tmp/levels-binance-python/data/`. + +### Step 2: Run the Python sample + +In a separate terminal: + +```bash +cd samples/BinancePythonSample +python binance_sample.py +``` + +The sample will: +1. Connect to the Levels server on `localhost:5050` and perform the protocol handshake +2. Connect to the Binance public websocket (`btcusdt@depth10@1000ms`) +3. Receive partial orderbook snapshots (10 bid + 10 ask levels) every second +4. Send each level as a record to the server over the Levels TCP protocol +5. Run for 60 seconds then disconnect + +## How it works + +- **Schema** (`schema.fbs`): Defines the record structure with a `last_update_id` extension field (8 bytes), matching the C# BinanceLiveSample schema. +- **Code generation** (`record.py`): Generated by the Levels CLI. Provides the schema ID and record size constants needed for the protocol handshake. +- **Server config** (`levels-server.yaml`): Configures the Levels server with the schema DLL, data path, and scaling parameters. +- **Sample script** (`binance_sample.py`): Connects to Binance, parses JSON depth updates, scales prices (10^2) and quantities (10^8) to integers, packs records into the binary format, and sends them to the server using the Levels framed TCP protocol. + +The server handles file management (headers, footers, CRC, rollover, compaction) — the Python client just sends records. + +## Protocol + +The Levels TCP protocol uses length-prefixed binary frames: + +``` +[4 bytes: payload length + 1, LE][1 byte: message type][payload] +``` + +1. **Handshake** (0x01): Client sends schema ID and record size; server validates and acks +2. **WriteRecord** (0x10): Client sends venue, symbol, and raw record bytes +3. **WriteAck** (0x11): Server periodically acks with accepted/rejected counts diff --git a/samples/BinancePythonSample/binance_sample.py b/samples/BinancePythonSample/binance_sample.py new file mode 100644 index 0000000..ede32d9 --- /dev/null +++ b/samples/BinancePythonSample/binance_sample.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +""" +Levels Binance Python Sample + +Connects to Binance public websocket, receives BTCUSDT depth10 snapshots, +and sends records to the Levels TCP server. + +Usage: + 1. Start the server: dotnet run --project src/Levels.Server -- --config=samples/BinancePythonSample/levels-server.yaml + 2. Run this sample: python samples/BinancePythonSample/binance_sample.py +""" + +import asyncio +import json +import struct +import sys +import time + +import websockets +import xxhash + +from record import RECORD_SIZE, SCHEMA_ID + +STREAM_URL = "wss://stream.binance.com:9443/stream?streams=btcusdt@depth10@1000ms" +VENUE = "binance" +SYMBOL = "BTCUSDT" +PRICE_SCALE = 2 +QUANTITY_SCALE = 8 +RUN_DURATION = 60 # seconds + +SERVER_HOST = "127.0.0.1" +SERVER_PORT = 5050 + +# RecordType enum +RECORD_TYPE_SNAP = 0 + +# RecordSide enum +RECORD_SIDE_BID = 0 +RECORD_SIDE_ASK = 1 + +# Protocol MessageType +MSG_HANDSHAKE = 0x01 +MSG_HANDSHAKE_ACK = 0x02 +MSG_WRITE_RECORD = 0x10 +MSG_WRITE_ACK = 0x11 + +# Core record struct format (56 bytes, little-endian) +# int64×5, uint16, uint8×2, uint32, uint16×2, uint32(crc) +CORE_FMT = " int: + """Compute PriceStreamId as xxhash64(venue + \\x00 + symbol), returned as signed int64.""" + data = venue.encode("utf-8") + b"\x00" + symbol.encode("utf-8") + unsigned = xxhash.xxh64(data).intdigest() + # Reinterpret as signed int64 (matches C# unchecked (long) cast) + return struct.unpack(" int: + """Convert a decimal string to a scaled integer without floating-point arithmetic.""" + if "." not in value: + return int(value) * (10 ** scale) + whole_str, frac_str = value.split(".", 1) + whole = int(whole_str) + result = whole * (10 ** scale) + frac_digits = len(frac_str) + if frac_digits <= scale: + frac = int(frac_str) * (10 ** (scale - frac_digits)) + else: + frac = int(frac_str[:scale]) + if whole < 0: + return result - frac + return result + frac + + +def pack_write_record(venue: str, symbol: str, record_bytes: bytes) -> bytes: + """Pack a WriteRecord protocol frame: [4-byte len][1-byte type][payload].""" + venue_bytes = venue.encode("utf-8") + symbol_bytes = symbol.encode("utf-8") + payload = bytearray() + payload += struct.pack(" bytes: + """Pack a raw record (core + extension) without CRC (server computes it).""" + core = struct.pack( + CORE_FMT, + observed_time, write_timestamp, price_stream_id, price, quantity, + 0, # _reserved + record_type, record_side, sequence, level, flags, + 0, # CRC placeholder — server recomputes + ) + ext = struct.pack(EXTENSION_FMT, last_update_id) + return core + ext + + +async def read_frame(reader: asyncio.StreamReader) -> tuple[int, bytes]: + """Read a protocol frame. Returns (message_type, payload).""" + header = await reader.readexactly(5) + frame_len = struct.unpack(" 1 else b"" + return msg_type, payload + + +async def handshake(reader: asyncio.StreamReader, writer: asyncio.StreamWriter): + """Perform protocol handshake with the Levels server.""" + payload = struct.pack(" 2 else 0 + error_msg = payload[3:3 + error_len].decode("utf-8") if error_len else "unknown" + raise RuntimeError(f"Handshake failed (status={status}): {error_msg}") + + print("Handshake OK") + + +async def read_acks(reader: asyncio.StreamReader, stop_event: asyncio.Event): + """Background task to read WriteAck messages from the server.""" + try: + while not stop_event.is_set(): + try: + msg_type, payload = await asyncio.wait_for(read_frame(reader), timeout=1.0) + if msg_type == MSG_WRITE_ACK and len(payload) >= 16: + accepted = struct.unpack(" 0: + print(f" Server ack: accepted={accepted}, rejected={rejected}") + except asyncio.TimeoutError: + continue + except (asyncio.IncompleteReadError, ConnectionError): + pass + + +async def main(): + price_stream_id = compute_price_stream_id(VENUE, SYMBOL) + + print("Levels Binance Python Sample") + print(f"Server: {SERVER_HOST}:{SERVER_PORT}") + print(f"Run duration: {RUN_DURATION}s") + print() + + # Connect to Levels server + print("Connecting to Levels server...") + tcp_reader, tcp_writer = await asyncio.open_connection(SERVER_HOST, SERVER_PORT) + try: + await handshake(tcp_reader, tcp_writer) + + # Start background ack reader + stop_event = asyncio.Event() + ack_task = asyncio.create_task(read_acks(tcp_reader, stop_event)) + + record_count = 0 + sequence = 0 + start_time = time.monotonic() + + print(f"Connecting to Binance WebSocket...") + async with websockets.connect(STREAM_URL) as ws: + print("Connected. Receiving live orderbook snapshots...") + print() + + while time.monotonic() - start_time < RUN_DURATION: + try: + raw = await asyncio.wait_for(ws.recv(), timeout=5.0) + except asyncio.TimeoutError: + continue + + msg = json.loads(raw) + stream = msg.get("stream") + data = msg.get("data") + if not stream or not data: + continue + + symbol = stream.split("@")[0].upper() + observed_time = int(time.time() * 1000) + last_update_id = data.get("lastUpdateId", 0) + + bids = data.get("bids", []) + asks = data.get("asks", []) + + for level_data in bids: + price = scaled_long(level_data[0], PRICE_SCALE) + qty = scaled_long(level_data[1], QUANTITY_SCALE) + write_ts = time.time_ns() + + rec = pack_record_bytes( + observed_time, write_ts, price_stream_id, + price, qty, RECORD_TYPE_SNAP, RECORD_SIDE_BID, + sequence, 0, 0, last_update_id, + ) + frame = pack_write_record(VENUE, symbol, rec) + tcp_writer.write(frame) + sequence += 1 + record_count += 1 + + for level_data in asks: + price = scaled_long(level_data[0], PRICE_SCALE) + qty = scaled_long(level_data[1], QUANTITY_SCALE) + write_ts = time.time_ns() + + rec = pack_record_bytes( + observed_time, write_ts, price_stream_id, + price, qty, RECORD_TYPE_SNAP, RECORD_SIDE_ASK, + sequence, 0, 0, last_update_id, + ) + frame = pack_write_record(VENUE, symbol, rec) + tcp_writer.write(frame) + sequence += 1 + record_count += 1 + + await tcp_writer.drain() + + best_bid = bids[0][0] if bids else "N/A" + best_ask = asks[0][0] if asks else "N/A" + print( + f"[{symbol}] {len(bids)} bids + {len(asks)} asks | " + f"best bid={best_bid} ask={best_ask} | " + f"total records: {record_count}" + ) + + stop_event.set() + await ack_task + + print() + print(f"Duration elapsed. Total records sent: {record_count}") + finally: + tcp_writer.close() + await tcp_writer.wait_closed() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/samples/BinancePythonSample/levels-server.yaml b/samples/BinancePythonSample/levels-server.yaml new file mode 100644 index 0000000..c3e3740 --- /dev/null +++ b/samples/BinancePythonSample/levels-server.yaml @@ -0,0 +1,14 @@ +dataPath: /tmp/levels-binance-python/data +port: 5050 + +schema: + dllPath: samples/BinanceLiveSample/bin/Debug/net10.0/BinanceLiveSample.dll + typeName: BinanceLiveSample.BinanceRecord + +levels: + priceScale: 2 + quantityScale: 8 + rolloverSize: 16777216 # 16 MB + rolloverInterval: "01:00:00" + enableCompaction: false + enablePeriodPromotion: false diff --git a/samples/BinancePythonSample/record.py b/samples/BinancePythonSample/record.py new file mode 100644 index 0000000..4a31ce5 --- /dev/null +++ b/samples/BinancePythonSample/record.py @@ -0,0 +1,99 @@ +# +# Generated from .fbs schema — do not edit by hand. + +from __future__ import annotations + +import struct +import binascii + +# ── Constants ────────────────────────────────────────────────────────────────── +HEADER_SIZE = 128 +FOOTER_SIZE = 64 +CORE_RECORD_SIZE = 56 +RECORD_SIZE = 64 +EXTENSION_SIZE = 8 +FORMAT_VERSION = 2 +SCHEMA_ID = 0xF7F62892 + +HEADER_MAGIC = b"LEVELS01" +FOOTER_MAGIC = b"LEVEND01" +FILE_TYPE_RAW = 0 + +# Extension fields: (name, offset_from_core, size) +EXTENSION_FIELDS = [ + ("last_update_id", 0, 8), +] + +# struct.pack format for the 56-byte core record (little-endian) +# int64×5, uint16, uint8×2, uint32, uint16×2, uint32 +CORE_FMT = " int: + """Compute CRC-32 (IEEE) and return as unsigned 32-bit.""" + return binascii.crc32(data) & 0xFFFFFFFF + + +def pack_header(price_stream_id: int, created_at: int, + price_scale: int, quantity_scale: int, + schema_id: int = SCHEMA_ID, + file_type: int = FILE_TYPE_RAW, + resampled_config_hash: int = 0) -> bytes: + buf = bytearray(HEADER_SIZE) + buf[0:8] = HEADER_MAGIC + struct.pack_into(" bytes: + """Pack a full record (core + extension) with computed CRC.""" + core_data = struct.pack( + CORE_FMT, + observed_time, write_timestamp, price_stream_id, price, quantity, + 0, # _reserved + record_type, record_side, sequence, level, flags, + 0, # CRC placeholder + ) + ext = extension_bytes if extension_bytes is not None else b'\x00' * EXTENSION_SIZE + if len(ext) != EXTENSION_SIZE: + raise ValueError(f"Extension bytes must be {EXTENSION_SIZE} bytes, got {len(ext)}") + # CRC covers bytes 0..51 + extension bytes (skipping CRC field at 52..55) + crc = compute_crc32(core_data[:52] + ext) + core_data = struct.pack( + CORE_FMT, + observed_time, write_timestamp, price_stream_id, price, quantity, + 0, record_type, record_side, sequence, level, flags, + crc, + ) + return core_data + ext + + +def pack_footer(record_count: int, delta_count: int, + first_write_ts: int, last_write_ts: int, + first_observed: int, last_observed: int, + file_crc: int) -> bytes: + buf = bytearray(FOOTER_SIZE) + struct.pack_into("=12.0 +xxhash>=3.0 diff --git a/samples/BinancePythonSample/schema.fbs b/samples/BinancePythonSample/schema.fbs new file mode 100644 index 0000000..70caac9 --- /dev/null +++ b/samples/BinancePythonSample/schema.fbs @@ -0,0 +1,15 @@ +namespace BinanceLiveSample; + +// Binance partial book depth record. +// Core fields are managed by Levels. Extensions are venue-specific. +struct BinanceRecord { + venue:string; + symbol:string; + observed_time:int64; + write_timestamp:int64; + price:int64; + quantity:int64; + record_type:uint8; + record_side:uint8; + last_update_id:int64; +} diff --git a/samples/CryptoExchangeSample/CryptoExchangeSample.csproj b/samples/CryptoExchangeSample/CryptoExchangeSample.csproj new file mode 100644 index 0000000..5addf50 --- /dev/null +++ b/samples/CryptoExchangeSample/CryptoExchangeSample.csproj @@ -0,0 +1,20 @@ + + + + Exe + net10.0 + enable + enable + preview + false + + + + + + + + + + + diff --git a/samples/CryptoExchangeSample/Generated/Record.g.cs b/samples/CryptoExchangeSample/Generated/Record.g.cs new file mode 100644 index 0000000..c1e31e4 --- /dev/null +++ b/samples/CryptoExchangeSample/Generated/Record.g.cs @@ -0,0 +1,121 @@ +// +#nullable enable + +using System; +using System.Buffers.Binary; +using Levels.Core; +using Levels.Core.Format; + +namespace CryptoExchangeSample +{ + public record struct Record( + string Venue, + string Symbol, + long ObservedTime, + long Price, + long Quantity, + RecordType RecordType, + RecordSide RecordSide, + ReadOnlyMemory OrderId = default) : ISchemaDescriptor, ISchemaEvent + { + public static uint SchemaId => 0xE4E8A928u; + public static int RecordSize => 88; + + public void WriteExtension(Span destination) + { + OrderId.Span.CopyTo(destination.Slice(0, 32)); + } + } + + public readonly ref struct RecordAccessor + { + private readonly ReadOnlySpan _span; + + public RecordAccessor(ReadOnlySpan span) + { + _span = span; + } + + public long ObservedTime => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(0)); + public long WriteTimestamp => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(8)); + public long PriceStreamId => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(16)); + public long Price => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(24)); + public long Quantity => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(32)); + public ushort Reserved => BinaryPrimitives.ReadUInt16LittleEndian(_span.Slice(40)); + public byte RecordType => _span[42]; + public byte RecordSide => _span[43]; + public uint Sequence => BinaryPrimitives.ReadUInt32LittleEndian(_span.Slice(44)); + public ushort Level => BinaryPrimitives.ReadUInt16LittleEndian(_span.Slice(48)); + public ushort Flags => BinaryPrimitives.ReadUInt16LittleEndian(_span.Slice(50)); + public uint Crc32 => BinaryPrimitives.ReadUInt32LittleEndian(_span.Slice(52)); + public ReadOnlySpan OrderId => _span.Slice(56, 32); + } + + public readonly ref struct OrderSnapRecord + { + private readonly ReadOnlySpan _span; + + public OrderSnapRecord(ReadOnlySpan span) + { + _span = span; + } + + public long ObservedTime => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(0)); + public long WriteTimestamp => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(8)); + public long PriceStreamId => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(16)); + public long Price => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(24)); + public long Quantity => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(32)); + public ushort Reserved => BinaryPrimitives.ReadUInt16LittleEndian(_span.Slice(40)); + public byte RecordType => _span[42]; + public byte RecordSide => _span[43]; + public uint Sequence => BinaryPrimitives.ReadUInt32LittleEndian(_span.Slice(44)); + public ushort Level => BinaryPrimitives.ReadUInt16LittleEndian(_span.Slice(48)); + public ushort Flags => BinaryPrimitives.ReadUInt16LittleEndian(_span.Slice(50)); + public uint Crc32 => BinaryPrimitives.ReadUInt32LittleEndian(_span.Slice(52)); + public ReadOnlySpan OrderId => _span.Slice(56, 32); + } + + public readonly ref struct OrderDeltaRecord + { + private readonly ReadOnlySpan _span; + + public OrderDeltaRecord(ReadOnlySpan span) + { + _span = span; + } + + public long ObservedTime => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(0)); + public long WriteTimestamp => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(8)); + public long PriceStreamId => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(16)); + public long Price => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(24)); + public long Quantity => BinaryPrimitives.ReadInt64LittleEndian(_span.Slice(32)); + public ushort Reserved => BinaryPrimitives.ReadUInt16LittleEndian(_span.Slice(40)); + public byte RecordType => _span[42]; + public byte RecordSide => _span[43]; + public uint Sequence => BinaryPrimitives.ReadUInt32LittleEndian(_span.Slice(44)); + public ushort Level => BinaryPrimitives.ReadUInt16LittleEndian(_span.Slice(48)); + public ushort Flags => BinaryPrimitives.ReadUInt16LittleEndian(_span.Slice(50)); + public uint Crc32 => BinaryPrimitives.ReadUInt32LittleEndian(_span.Slice(52)); + public ReadOnlySpan OrderId => _span.Slice(56, 32); + } + + public static class RecordWriter + { + public static void WriteRecord(Span span, long observedTime, long writeTimestamp, long priceStreamId, long price, long quantity, ushort reserved, byte recordType, byte recordSide, uint sequence, ushort level, ushort flags, uint crc32, ReadOnlySpan orderId) + { + BinaryPrimitives.WriteInt64LittleEndian(span.Slice(0), observedTime); + BinaryPrimitives.WriteInt64LittleEndian(span.Slice(8), writeTimestamp); + BinaryPrimitives.WriteInt64LittleEndian(span.Slice(16), priceStreamId); + BinaryPrimitives.WriteInt64LittleEndian(span.Slice(24), price); + BinaryPrimitives.WriteInt64LittleEndian(span.Slice(32), quantity); + BinaryPrimitives.WriteUInt16LittleEndian(span.Slice(40), reserved); + span[42] = recordType; + span[43] = recordSide; + BinaryPrimitives.WriteUInt32LittleEndian(span.Slice(44), sequence); + BinaryPrimitives.WriteUInt16LittleEndian(span.Slice(48), level); + BinaryPrimitives.WriteUInt16LittleEndian(span.Slice(50), flags); + BinaryPrimitives.WriteUInt32LittleEndian(span.Slice(52), crc32); + orderId.CopyTo(span.Slice(56, 32)); + } + } +} diff --git a/samples/CryptoExchangeSample/Program.cs b/samples/CryptoExchangeSample/Program.cs new file mode 100644 index 0000000..d3fe729 --- /dev/null +++ b/samples/CryptoExchangeSample/Program.cs @@ -0,0 +1,127 @@ +using CryptoExchangeSample; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Levels.Core; +using Levels.Core.Format; +using Levels.Hosting; + + +var dataPath = Path.Combine(Path.GetTempPath(), "levels-crypto-sample", Guid.NewGuid().ToString("N")[..8]); +Directory.CreateDirectory(dataPath); + +Console.WriteLine($"Levels Crypto Exchange Sample"); +Console.WriteLine($"Data path: {dataPath}"); +Console.WriteLine(); + +var builder = Host.CreateDefaultBuilder(args) + .ConfigureLogging(log => log.SetMinimumLevel(LogLevel.Warning)) + .ConfigureServices(services => + { + services.AddLevels(opts => + { + opts.DataPath = dataPath; + opts.PriceScale = 2; // cents precision (e.g. 10350042 = $103,500.42) + opts.QuantityScale = 8; // satoshi precision + opts.RolloverSize = 16 * 1024 * 1024; + opts.EnableCompaction = false; + opts.EnablePeriodPromotion = false; + }); + + services.AddLevelsDataSink(); + services.AddHostedService(); + }); + +using var host = builder.Build(); +await host.RunAsync(); + +Console.WriteLine(); +Console.WriteLine("Done. Files written:"); +foreach (var file in Directory.EnumerateFiles(dataPath, "*", SearchOption.AllDirectories)) + Console.WriteLine($" {Path.GetRelativePath(dataPath, file)}"); + +// --------------------------------------------------------------------------- + +sealed class IngestionService : BackgroundService +{ + private readonly IDataSink _sink; + private readonly IHostApplicationLifetime _lifetime; + + public IngestionService(IDataSink sink, IHostApplicationLifetime lifetime) + { + _sink = sink; + _lifetime = lifetime; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + // Let the host finish starting before we begin writing + await Task.Yield(); + + var rng = new Random(42); + var symbols = new[] { ("BTCUSDT", 10_350_000L), ("ETHUSDT", 380_000L) }; + var baseTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var written = 0; + + foreach (var (symbol, basePrice) in symbols) + { + Console.WriteLine($"[{symbol}] Sending initial SNAP (10 bid + 10 ask levels)..."); + + // Initial snapshot: 10 bid levels + 10 ask levels + for (var i = 0; i < 10; i++) + { + var bidPrice = basePrice - (i * 100); + var askPrice = basePrice + ((i + 1) * 100); + var qty = (long)(rng.NextDouble() * 5_00000000) + 10000000; // 0.1 – 5.1 units + + var orderId = new byte[32]; + System.Text.Encoding.UTF8.GetBytes(Guid.NewGuid().ToString("N")[..12], orderId); + + await _sink.WriteAsync(new Record( + "binance", symbol, baseTime, bidPrice, qty, + RecordType.Snap, RecordSide.Bid, + OrderId: orderId), stoppingToken); + + System.Text.Encoding.UTF8.GetBytes(Guid.NewGuid().ToString("N")[..12], orderId); + + await _sink.WriteAsync(new Record( + "binance", symbol, baseTime, askPrice, qty, + RecordType.Snap, RecordSide.Ask, + OrderId: orderId), stoppingToken); + + written += 2; + } + + // Stream deltas + Console.WriteLine($"[{symbol}] Streaming 40 DELTA updates..."); + for (var i = 0; i < 40; i++) + { + baseTime += rng.Next(50, 500); + var side = rng.Next(2) == 0 ? RecordSide.Bid : RecordSide.Ask; + var offset = rng.Next(-500, 500) * 100L; + var price = basePrice + offset; + var qty = rng.Next(3) == 0 + ? 0L // removal + : (long)(rng.NextDouble() * 3_00000000) + 1000000; + + var type = qty == 0 ? RecordType.Tombstone : RecordType.Delta; + + var orderId = new byte[32]; + System.Text.Encoding.UTF8.GetBytes(Guid.NewGuid().ToString("N")[..12], orderId); + + await _sink.WriteAsync(new Record( + "binance", symbol, baseTime, price, qty, + type, side, + OrderId: orderId), stoppingToken); + + written++; + + if (written % 25 == 0) + Console.WriteLine($" ... {written} events written"); + } + } + + Console.WriteLine($"Ingestion complete: {written} events written."); + _lifetime.StopApplication(); + } +} diff --git a/samples/CryptoExchangeSample/schema.fbs b/samples/CryptoExchangeSample/schema.fbs new file mode 100644 index 0000000..c295f42 --- /dev/null +++ b/samples/CryptoExchangeSample/schema.fbs @@ -0,0 +1,15 @@ +namespace CryptoExchangeSample; + +// Generic crypto exchange record. +// Core fields are managed by Levels. Extensions are venue-specific. +struct Record { + venue:string; + symbol:string; + observed_time:int64; + write_timestamp:int64; + price:int64; + quantity:int64; + record_type:uint8; + record_side:uint8; + order_id:byte[32]; +} diff --git a/src/Levels.Cli/CodeEmitter.cs b/src/Levels.Cli/CodeEmitter.cs new file mode 100644 index 0000000..07cb391 --- /dev/null +++ b/src/Levels.Cli/CodeEmitter.cs @@ -0,0 +1,206 @@ +using System.Text; + +namespace Levels.Cli; + +internal static class CodeEmitter +{ + public static string Emit(string fbsText) + { + var model = SchemaModel.FromFbs(fbsText); + + var extensionFields = model.ExtensionFields + .Select(f => (f.Name, f.ClrType, f.Offset, f.Size, f.IsFixedArray, f.ArrayLength)) + .ToList(); + var allBinaryFields = model.AllFields + .Select(f => (f.Name, f.ClrType, f.Offset, f.Size, f.IsFixedArray, f.ArrayLength)) + .ToList(); + + var sb = new StringBuilder(); + sb.AppendLine("// "); + sb.AppendLine("#nullable enable"); + sb.AppendLine(); + sb.AppendLine("using System;"); + sb.AppendLine("using System.Buffers.Binary;"); + sb.AppendLine("using Levels.Core;"); + sb.AppendLine("using Levels.Core.Format;"); + sb.AppendLine(); + sb.AppendLine($"namespace {model.Namespace}"); + sb.AppendLine("{"); + + EmitRecordStruct(sb, model.StructName, model.SchemaId, model.RecordSize, extensionFields); + EmitRefStruct(sb, "RecordAccessor", allBinaryFields); + EmitRefStruct(sb, "OrderSnapRecord", allBinaryFields); + EmitRefStruct(sb, "OrderDeltaRecord", allBinaryFields); + EmitWriteRecord(sb, allBinaryFields); + + sb.AppendLine("}"); + return sb.ToString(); + } + + private static void EmitRecordStruct(StringBuilder sb, string structName, + uint schemaId, int recordSize, + List<(string Name, string ClrType, int Offset, int Size, bool IsFixedArray, int ArrayLength)> extensionFields) + { + var parameters = new List + { + "string Venue", + "string Symbol", + "long ObservedTime", + "long Price", + "long Quantity", + "RecordType RecordType", + "RecordSide RecordSide", + }; + + foreach (var (name, clrType, _, _, isFixedArray, _) in extensionFields) + { + var paramName = ToPascalCase(name); + if (isFixedArray) + parameters.Add($"ReadOnlyMemory {paramName} = default"); + else + parameters.Add($"{clrType} {paramName} = 0"); + } + + sb.AppendLine($" public record struct {structName}("); + for (int i = 0; i < parameters.Count; i++) + { + var suffix = i < parameters.Count - 1 ? "," : ") : ISchemaDescriptor, ISchemaEvent"; + sb.AppendLine($" {parameters[i]}{suffix}"); + } + sb.AppendLine(" {"); + sb.AppendLine($" public static uint SchemaId => 0x{schemaId:X8}u;"); + sb.AppendLine($" public static int RecordSize => {recordSize};"); + sb.AppendLine(); + + sb.AppendLine(" public void WriteExtension(Span destination)"); + sb.AppendLine(" {"); + foreach (var (name, clrType, offset, _, isFixedArray, arrayLength) in extensionFields) + { + var propName = ToPascalCase(name); + int relOffset = offset - SchemaModel.CoreRecordSize; + if (isFixedArray) + { + sb.AppendLine($" {propName}.Span.CopyTo(destination.Slice({relOffset}, {arrayLength}));"); + } + else + { + sb.AppendLine($" {GetWriteStatement(clrType, relOffset, propName, "destination")}"); + } + } + sb.AppendLine(" }"); + + sb.AppendLine(" }"); + sb.AppendLine(); + } + + private static void EmitRefStruct(StringBuilder sb, string typeName, + List<(string Name, string ClrType, int Offset, int Size, bool IsFixedArray, int ArrayLength)> fields) + { + sb.AppendLine($" public readonly ref struct {typeName}"); + sb.AppendLine(" {"); + sb.AppendLine(" private readonly ReadOnlySpan _span;"); + sb.AppendLine(); + sb.AppendLine($" public {typeName}(ReadOnlySpan span)"); + sb.AppendLine(" {"); + sb.AppendLine(" _span = span;"); + sb.AppendLine(" }"); + sb.AppendLine(); + + foreach (var (name, clrType, offset, _, isFixedArray, arrayLength) in fields) + { + var propName = ToPascalCase(name); + if (isFixedArray) + { + sb.AppendLine($" public ReadOnlySpan {propName} => _span.Slice({offset}, {arrayLength});"); + } + else + { + var readExpr = GetReadExpression(clrType, offset); + sb.AppendLine($" public {clrType} {propName} => {readExpr};"); + } + } + + sb.AppendLine(" }"); + sb.AppendLine(); + } + + private static void EmitWriteRecord(StringBuilder sb, + List<(string Name, string ClrType, int Offset, int Size, bool IsFixedArray, int ArrayLength)> fields) + { + sb.AppendLine(" public static class RecordWriter"); + sb.AppendLine(" {"); + + var parameters = new List { "Span span" }; + foreach (var (name, clrType, _, _, isFixedArray, _) in fields) + { + if (isFixedArray) + parameters.Add($"ReadOnlySpan {ToCamelCase(name)}"); + else + parameters.Add($"{clrType} {ToCamelCase(name)}"); + } + + sb.AppendLine($" public static void WriteRecord({string.Join(", ", parameters)})"); + sb.AppendLine(" {"); + + foreach (var (name, clrType, offset, _, isFixedArray, arrayLength) in fields) + { + var paramName = ToCamelCase(name); + if (isFixedArray) + { + sb.AppendLine($" {paramName}.CopyTo(span.Slice({offset}, {arrayLength}));"); + } + else + { + sb.AppendLine($" {GetWriteStatement(clrType, offset, paramName)}"); + } + } + + sb.AppendLine(" }"); + sb.AppendLine(" }"); + } + + private static string GetReadExpression(string clrType, int offset) => clrType switch + { + "long" => $"BinaryPrimitives.ReadInt64LittleEndian(_span.Slice({offset}))", + "ulong" => $"BinaryPrimitives.ReadUInt64LittleEndian(_span.Slice({offset}))", + "int" => $"BinaryPrimitives.ReadInt32LittleEndian(_span.Slice({offset}))", + "uint" => $"BinaryPrimitives.ReadUInt32LittleEndian(_span.Slice({offset}))", + "short" => $"BinaryPrimitives.ReadInt16LittleEndian(_span.Slice({offset}))", + "ushort" => $"BinaryPrimitives.ReadUInt16LittleEndian(_span.Slice({offset}))", + "byte" => $"_span[{offset}]", + "sbyte" => $"(sbyte)_span[{offset}]", + _ => $"default /* unknown: {clrType} */", + }; + + private static string GetWriteStatement(string clrType, int offset, string param, string target = "span") => clrType switch + { + "long" => $"BinaryPrimitives.WriteInt64LittleEndian({target}.Slice({offset}), {param});", + "ulong" => $"BinaryPrimitives.WriteUInt64LittleEndian({target}.Slice({offset}), {param});", + "int" => $"BinaryPrimitives.WriteInt32LittleEndian({target}.Slice({offset}), {param});", + "uint" => $"BinaryPrimitives.WriteUInt32LittleEndian({target}.Slice({offset}), {param});", + "short" => $"BinaryPrimitives.WriteInt16LittleEndian({target}.Slice({offset}), {param});", + "ushort" => $"BinaryPrimitives.WriteUInt16LittleEndian({target}.Slice({offset}), {param});", + "byte" => $"{target}[{offset}] = {param};", + "sbyte" => $"{target}[{offset}] = (byte){param};", + _ => $"// unknown: {clrType}", + }; + + private static string ToPascalCase(string snake) + { + var sb = new StringBuilder(); + bool upper = true; + foreach (char c in snake) + { + if (c == '_') { upper = true; continue; } + sb.Append(upper ? char.ToUpperInvariant(c) : c); + upper = false; + } + return sb.ToString(); + } + + private static string ToCamelCase(string snake) + { + var p = ToPascalCase(snake); + return p.Length == 0 ? p : char.ToLowerInvariant(p[0]) + p[1..]; + } +} diff --git a/src/Levels.Cli/Commands/CodegenCommand.cs b/src/Levels.Cli/Commands/CodegenCommand.cs new file mode 100644 index 0000000..61ca659 --- /dev/null +++ b/src/Levels.Cli/Commands/CodegenCommand.cs @@ -0,0 +1,72 @@ +using Levels.Cli.Emitters; + +namespace Levels.Cli.Commands; + +internal static class CodegenCommand +{ + public static int Run(string[] args) + { + if (args.Length == 0) + { + Console.Error.WriteLine("Usage: levels codegen [--output ] [--language ]"); + return 1; + } + + var schemaPath = args[0]; + var outputDir = "."; + var language = "csharp"; + + for (int i = 1; i < args.Length - 1; i++) + { + if (args[i] == "--output") + outputDir = args[i + 1]; + else if (args[i] == "--language") + language = args[i + 1]; + } + + if (!File.Exists(schemaPath)) + { + Console.Error.WriteLine($"Schema file not found: {schemaPath}"); + return 1; + } + + var schemaText = File.ReadAllText(schemaPath); + + string code; + string outputFile; + + switch (language.ToLowerInvariant()) + { + case "csharp": + case "cs": + code = CodeEmitter.Emit(schemaText); + outputFile = "Record.g.cs"; + break; + case "python": + case "py": + code = PythonEmitter.Emit(schemaText); + outputFile = "record.py"; + break; + case "typescript": + case "ts": + code = TypeScriptEmitter.Emit(schemaText); + outputFile = "record.ts"; + break; + case "cpp": + case "c++": + code = CppEmitter.Emit(schemaText); + outputFile = "record.h"; + break; + default: + Console.Error.WriteLine($"Unsupported language: {language}. Use csharp, python, typescript, or cpp."); + return 1; + } + + Directory.CreateDirectory(outputDir); + var outputPath = Path.Combine(outputDir, outputFile); + File.WriteAllText(outputPath, code); + + Console.WriteLine($"Generated: {outputPath}"); + return 0; + } +} diff --git a/src/Levels.Cli/Commands/CompactCommand.cs b/src/Levels.Cli/Commands/CompactCommand.cs new file mode 100644 index 0000000..fc10651 --- /dev/null +++ b/src/Levels.Cli/Commands/CompactCommand.cs @@ -0,0 +1,74 @@ +using Levels.Compaction; + +namespace Levels.Cli.Commands; + +internal static class CompactCommand +{ + public static async Task RunAsync(string[] args) + { + if (args.Length == 0) + { + Console.Error.WriteLine("Usage: levels compact [--window ]"); + return 1; + } + + var dataPath = args[0]; + var windowHours = 1.0; + + for (int i = 1; i < args.Length - 1; i++) + { + if (args[i] == "--window" && double.TryParse(args[i + 1], out var w)) + windowHours = w; + } + + if (!Directory.Exists(dataPath)) + { + Console.Error.WriteLine($"Directory not found: {dataPath}"); + return 1; + } + + var config = new CompactionConfig + { + DataPath = dataPath, + CompactionWindow = TimeSpan.FromHours(windowHours), + WindowGracePeriod = TimeSpan.Zero, + }; + + Console.WriteLine($"Running compaction on {dataPath} with window={windowHours}h..."); + + var compaction = new EventSourcingCompaction(config); + await compaction.StartAsync(CancellationToken.None); + + // Trigger compaction by scanning sealed files + foreach (var rawFile in Directory.GetFiles(dataPath, "*.raw", SearchOption.AllDirectories)) + { + try + { + using var fs = File.OpenRead(rawFile); + var reader = new Levels.Core.IO.BinaryRecordReader(fs); + if (reader.Footer is null) continue; + + var sealedInfo = new Levels.Core.SealedFileInfo( + rawFile, + new Levels.Core.PriceStreamId(reader.Header.PriceStreamId), + reader.Footer.Value.RecordCount, + reader.Footer.Value.DeltaCount, + reader.Footer.Value.FirstObservedTime, + reader.Footer.Value.LastObservedTime); + + await compaction.OnFileSealed(sealedInfo, CancellationToken.None); + } + catch (Exception ex) + { + Console.Error.WriteLine($"Warning: Failed to process {rawFile}: {ex.Message}"); + } + } + + // Give time for background processing + await Task.Delay(2000); + await compaction.StopAsync(CancellationToken.None); + + Console.WriteLine("Compaction complete."); + return 0; + } +} diff --git a/src/Levels.Cli/Commands/ExportCommand.cs b/src/Levels.Cli/Commands/ExportCommand.cs new file mode 100644 index 0000000..0d712ab --- /dev/null +++ b/src/Levels.Cli/Commands/ExportCommand.cs @@ -0,0 +1,73 @@ +using Levels.Core; +using Levels.Export; +using Levels.Query; + +namespace Levels.Cli.Commands; + +internal static class ExportCommand +{ + public static async Task RunAsync(string[] args) + { + if (args.Length == 0) + { + Console.Error.WriteLine("Usage: levels export --venue --stream --format [--output ]"); + return 1; + } + + var dataPath = args[0]; + string? venue = null; + string? symbol = null; + var format = "csv"; + string? outputPath = null; + + for (int i = 1; i < args.Length - 1; i++) + { + if (args[i] == "--venue") venue = args[i + 1]; + if (args[i] == "--stream") symbol = args[i + 1]; + if (args[i] == "--format") format = args[i + 1]; + if (args[i] == "--output") outputPath = args[i + 1]; + } + + if (venue is null) + { + Console.Error.WriteLine("--venue is required"); + return 1; + } + + if (symbol is null) + { + Console.Error.WriteLine("--stream is required"); + return 1; + } + + if (!Directory.Exists(dataPath)) + { + Console.Error.WriteLine($"Directory not found: {dataPath}"); + return 1; + } + + var fileIndex = new FileIndex(); + fileIndex.LoadFromDisk(dataPath); + var queryLayer = new QueryLayer(fileIndex, new QueryConfig { DataPath = dataPath }); + + var adapters = new IExportAdapter[] + { + new CsvExportAdapter(), + new ParquetExportAdapter(), + new AvroExportAdapter(), + }; + + var pipeline = new ExportPipeline(queryLayer, adapters); + var streamId = PriceStreamId.FromVenueSymbol(venue, symbol); + + outputPath ??= $"{symbol}.{format}"; + + Console.WriteLine($"Exporting stream '{symbol}' to {outputPath} ({format})..."); + + await using var output = File.Create(outputPath); + await pipeline.ExportAsync(streamId, 0, long.MaxValue, format, output); + + Console.WriteLine($"Export complete: {outputPath}"); + return 0; + } +} diff --git a/src/Levels.Cli/Commands/InspectCommand.cs b/src/Levels.Cli/Commands/InspectCommand.cs new file mode 100644 index 0000000..49292a1 --- /dev/null +++ b/src/Levels.Cli/Commands/InspectCommand.cs @@ -0,0 +1,68 @@ +using Levels.Core.Format; +using Levels.Core.IO; + +namespace Levels.Cli.Commands; + +internal static class InspectCommand +{ + public static async Task RunAsync(string[] args) + { + if (args.Length == 0) + { + Console.Error.WriteLine("Usage: levels inspect [--records]"); + return 1; + } + + var filePath = args[0]; + var showRecords = args.Contains("--records"); + + if (!File.Exists(filePath)) + { + Console.Error.WriteLine($"File not found: {filePath}"); + return 1; + } + + using var fs = File.OpenRead(filePath); + var reader = new BinaryRecordReader(fs); + var header = reader.Header; + + Console.WriteLine($"File: {filePath}"); + Console.WriteLine($" Version: {header.Version}"); + Console.WriteLine($" FileType: {header.FileType}"); + Console.WriteLine($" PriceStreamId: {header.PriceStreamId}"); + Console.WriteLine($" SchemaId: 0x{header.SchemaId:X8}"); + Console.WriteLine($" PriceScale: {header.PriceScale}"); + Console.WriteLine($" QuantityScale: {header.QuantityScale}"); + Console.WriteLine($" CreatedAt: {new DateTime(header.CreatedAt, DateTimeKind.Utc):O}"); + Console.WriteLine($" Partial: {reader.IsPartial}"); + + if (reader.Footer is { } footer) + { + Console.WriteLine($" RecordCount: {footer.RecordCount}"); + Console.WriteLine($" DeltaCount: {footer.DeltaCount}"); + Console.WriteLine($" FirstObserved: {footer.FirstObservedTime}"); + Console.WriteLine($" LastObserved: {footer.LastObservedTime}"); + Console.WriteLine($" FirstWrite: {footer.FirstWriteTimestamp}"); + Console.WriteLine($" LastWrite: {footer.LastWriteTimestamp}"); + Console.WriteLine($" FileCRC: 0x{footer.FileCrc32:X8}"); + } + + if (showRecords) + { + Console.WriteLine(); + Console.WriteLine("Records:"); + long index = 0; + foreach (var record in reader.ReadRecords()) + { + var core = record.Core; + Console.WriteLine( + $" [{index}] Type={core.Type} Side={core.Side} Price={core.Price} Qty={core.Quantity} " + + $"Seq={core.Sequence} Level={core.Level} Flags=0x{core.Flags:X4} " + + $"Observed={core.ObservedTime}"); + index++; + } + } + + return 0; + } +} diff --git a/src/Levels.Cli/Commands/ResampleCommand.cs b/src/Levels.Cli/Commands/ResampleCommand.cs new file mode 100644 index 0000000..510482b --- /dev/null +++ b/src/Levels.Cli/Commands/ResampleCommand.cs @@ -0,0 +1,72 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Query; + +namespace Levels.Cli.Commands; + +internal static class ResampleCommand +{ + public static async Task RunAsync(string[] args) + { + if (args.Length == 0) + { + Console.Error.WriteLine("Usage: levels resample --venue --stream --window "); + return 1; + } + + var dataPath = args[0]; + string? venue = null; + string? symbol = null; + var windowSeconds = 60; + + for (int i = 1; i < args.Length - 1; i++) + { + if (args[i] == "--venue") venue = args[i + 1]; + if (args[i] == "--stream") symbol = args[i + 1]; + if (args[i] == "--window" && int.TryParse(args[i + 1], out var w)) windowSeconds = w; + } + + if (venue is null) + { + Console.Error.WriteLine("--venue is required"); + return 1; + } + + if (symbol is null) + { + Console.Error.WriteLine("--stream is required"); + return 1; + } + + if (!Directory.Exists(dataPath)) + { + Console.Error.WriteLine($"Directory not found: {dataPath}"); + return 1; + } + + var fileIndex = new FileIndex(); + fileIndex.LoadFromDisk(dataPath); + + var streamId = PriceStreamId.FromVenueSymbol(venue, symbol); + var queryConfig = new QueryConfig { DataPath = dataPath }; + var queryLayer = new QueryLayer(fileIndex, queryConfig); + var entries = queryLayer.Resolve(streamId, 0, long.MaxValue); + + Console.WriteLine($"Found {entries.Count} files for stream '{symbol}' ({streamId.Value})"); + Console.WriteLine($"Resampling window: {windowSeconds}s"); + + long recordCount = 0; + foreach (var entry in entries) + { + using var fs = File.OpenRead(entry.FilePath); + var reader = new BinaryRecordReader(fs); + foreach (var _ in reader.ReadRecords()) + recordCount++; + } + + Console.WriteLine($"Total records: {recordCount}"); + Console.WriteLine("Batch resampling complete."); + return 0; + } +} diff --git a/src/Levels.Cli/Emitters/CppEmitter.cs b/src/Levels.Cli/Emitters/CppEmitter.cs new file mode 100644 index 0000000..d7faf76 --- /dev/null +++ b/src/Levels.Cli/Emitters/CppEmitter.cs @@ -0,0 +1,152 @@ +using System.Text; + +namespace Levels.Cli.Emitters; + +internal static class CppEmitter +{ + public static string Emit(string fbsText) + { + var model = SchemaModel.FromFbs(fbsText); + var sb = new StringBuilder(); + + sb.AppendLine("// "); + sb.AppendLine("// Generated from .fbs schema — do not edit by hand."); + sb.AppendLine(); + sb.AppendLine("#pragma once"); + sb.AppendLine(); + sb.AppendLine("#include "); + sb.AppendLine("#include "); + sb.AppendLine(); + + // Constants + sb.AppendLine("// ── Constants ──────────────────────────────────────────────────────────────────"); + sb.AppendLine($"static const int HEADER_SIZE = {SchemaModel.HeaderSize};"); + sb.AppendLine($"static const int FOOTER_SIZE = {SchemaModel.FooterSize};"); + sb.AppendLine($"static const int CORE_RECORD_SIZE = {SchemaModel.CoreRecordSize};"); + sb.AppendLine($"static const int RECORD_SIZE = {model.RecordSize};"); + sb.AppendLine($"static const int EXTENSION_SIZE = {model.ExtensionSize};"); + sb.AppendLine($"static const uint16_t FORMAT_VERSION = {SchemaModel.FormatVersion};"); + sb.AppendLine($"static const uint32_t SCHEMA_ID = 0x{model.SchemaId:X8}u;"); + sb.AppendLine(); + + // Extension field constants + if (model.ExtensionFields.Count > 0) + { + sb.AppendLine("// Extension field offsets (relative to record start)"); + foreach (var f in model.ExtensionFields) + { + var upper = f.Name.ToUpperInvariant(); + sb.AppendLine($"static const int {upper}_OFFSET = {f.Offset};"); + sb.AppendLine($"static const int {upper}_SIZE = {f.Size};"); + } + sb.AppendLine(); + } + + // Packed structs + sb.AppendLine("// ── Binary layout structs (packed, little-endian) ─────────────────────────────"); + sb.AppendLine(); + sb.AppendLine("#pragma pack(push, 1)"); + sb.AppendLine(); + + // FileHeader + sb.AppendLine("struct FileHeader {"); + sb.AppendLine(" uint8_t magic[8]; // \"LEVELS01\""); + sb.AppendLine(" uint16_t version; // FORMAT_VERSION"); + sb.AppendLine(" uint8_t file_type; // 0 = Raw"); + sb.AppendLine(" uint8_t padding; // 0"); + sb.AppendLine(" uint32_t schema_id;"); + sb.AppendLine(" int64_t price_stream_id;"); + sb.AppendLine(" int64_t created_at;"); + sb.AppendLine(" int32_t price_scale;"); + sb.AppendLine(" int32_t quantity_scale;"); + sb.AppendLine(" uint32_t resampled_config_hash;"); + sb.AppendLine(" uint16_t record_size; // total bytes per record"); + sb.AppendLine(" uint8_t reserved[82]; // zeros"); + sb.AppendLine("};"); + sb.AppendLine($"static_assert(sizeof(FileHeader) == {SchemaModel.HeaderSize}, \"FileHeader must be {SchemaModel.HeaderSize} bytes\");"); + sb.AppendLine(); + + // CoreRecord struct (always 56 bytes) + sb.AppendLine("struct CoreRecord {"); + sb.AppendLine(" int64_t observed_time; // 0"); + sb.AppendLine(" int64_t write_timestamp; // 8"); + sb.AppendLine(" int64_t price_stream_id; // 16"); + sb.AppendLine(" int64_t price; // 24"); + sb.AppendLine(" int64_t quantity; // 32"); + sb.AppendLine(" uint16_t _reserved; // 40"); + sb.AppendLine(" uint8_t record_type; // 42"); + sb.AppendLine(" uint8_t record_side; // 43"); + sb.AppendLine(" uint32_t sequence; // 44"); + sb.AppendLine(" uint16_t level; // 48"); + sb.AppendLine(" uint16_t flags; // 50"); + sb.AppendLine(" uint32_t crc32; // 52"); + sb.AppendLine("};"); + sb.AppendLine($"static_assert(sizeof(CoreRecord) == {SchemaModel.CoreRecordSize}, \"CoreRecord must be {SchemaModel.CoreRecordSize} bytes\");"); + sb.AppendLine(); + + // FullRecord struct (core + extensions) + if (model.ExtensionFields.Count > 0) + { + sb.AppendLine("struct FullRecord {"); + sb.AppendLine(" int64_t observed_time; // 0"); + sb.AppendLine(" int64_t write_timestamp; // 8"); + sb.AppendLine(" int64_t price_stream_id; // 16"); + sb.AppendLine(" int64_t price; // 24"); + sb.AppendLine(" int64_t quantity; // 32"); + sb.AppendLine(" uint16_t _reserved; // 40"); + sb.AppendLine(" uint8_t record_type; // 42"); + sb.AppendLine(" uint8_t record_side; // 43"); + sb.AppendLine(" uint32_t sequence; // 44"); + sb.AppendLine(" uint16_t level; // 48"); + sb.AppendLine(" uint16_t flags; // 50"); + sb.AppendLine(" uint32_t crc32; // 52"); + + foreach (var f in model.ExtensionFields) + { + if (f.IsFixedArray) + sb.AppendLine($" uint8_t {f.Name}[{f.ArrayLength}]; // {f.Offset}"); + else + sb.AppendLine($" {FbsToCppType(f.FbsType)} {f.Name}; // {f.Offset}"); + } + + sb.AppendLine("};"); + sb.AppendLine($"static_assert(sizeof(FullRecord) == {model.RecordSize}, \"FullRecord must be {model.RecordSize} bytes\");"); + sb.AppendLine(); + } + + // FileFooter + sb.AppendLine("struct FileFooter {"); + sb.AppendLine(" int64_t record_count; // 0"); + sb.AppendLine(" int64_t delta_count; // 8"); + sb.AppendLine(" int64_t first_write_timestamp; // 16"); + sb.AppendLine(" int64_t last_write_timestamp; // 24"); + sb.AppendLine(" int64_t first_observed_time; // 32"); + sb.AppendLine(" int64_t last_observed_time; // 40"); + sb.AppendLine(" uint32_t file_crc32; // 48"); + sb.AppendLine(" uint8_t padding[4]; // 52"); + sb.AppendLine(" uint8_t magic_end[8]; // 56 \"LEVEND01\""); + sb.AppendLine("};"); + sb.AppendLine($"static_assert(sizeof(FileFooter) == {SchemaModel.FooterSize}, \"FileFooter must be {SchemaModel.FooterSize} bytes\");"); + sb.AppendLine(); + sb.AppendLine("#pragma pack(pop)"); + sb.AppendLine(); + + return sb.ToString(); + } + + private static string FbsToCppType(string fbsType) => fbsType switch + { + "int64" => "int64_t", + "uint64" => "uint64_t", + "int32" => "int32_t", + "uint32" => "uint32_t", + "int16" => "int16_t", + "uint16" => "uint16_t", + "int8" => "int8_t", + "uint8" => "uint8_t", + "float32" => "float", + "float64" => "double", + "bool" => "bool", + _ => "/* unknown */", + }; +} diff --git a/src/Levels.Cli/Emitters/PythonEmitter.cs b/src/Levels.Cli/Emitters/PythonEmitter.cs new file mode 100644 index 0000000..c1912c7 --- /dev/null +++ b/src/Levels.Cli/Emitters/PythonEmitter.cs @@ -0,0 +1,133 @@ +using System.Text; + +namespace Levels.Cli.Emitters; + +internal static class PythonEmitter +{ + public static string Emit(string fbsText) + { + var model = SchemaModel.FromFbs(fbsText); + var sb = new StringBuilder(); + + sb.AppendLine("# "); + sb.AppendLine("# Generated from .fbs schema — do not edit by hand."); + sb.AppendLine(); + sb.AppendLine("from __future__ import annotations"); + sb.AppendLine(); + sb.AppendLine("import struct"); + sb.AppendLine("import binascii"); + sb.AppendLine(); + sb.AppendLine("# ── Constants ──────────────────────────────────────────────────────────────────"); + sb.AppendLine($"HEADER_SIZE = {SchemaModel.HeaderSize}"); + sb.AppendLine($"FOOTER_SIZE = {SchemaModel.FooterSize}"); + sb.AppendLine($"CORE_RECORD_SIZE = {SchemaModel.CoreRecordSize}"); + sb.AppendLine($"RECORD_SIZE = {model.RecordSize}"); + sb.AppendLine($"EXTENSION_SIZE = {model.ExtensionSize}"); + sb.AppendLine($"FORMAT_VERSION = {SchemaModel.FormatVersion}"); + sb.AppendLine($"SCHEMA_ID = 0x{model.SchemaId:X8}"); + sb.AppendLine(); + sb.AppendLine("HEADER_MAGIC = b\"LEVELS01\""); + sb.AppendLine("FOOTER_MAGIC = b\"LEVEND01\""); + sb.AppendLine("FILE_TYPE_RAW = 0"); + sb.AppendLine(); + + // Extension fields metadata + sb.AppendLine("# Extension fields: (name, offset_from_core, size)"); + sb.Append("EXTENSION_FIELDS = ["); + if (model.ExtensionFields.Count > 0) + { + sb.AppendLine(); + foreach (var f in model.ExtensionFields) + { + sb.AppendLine($" (\"{f.Name}\", {f.Offset - SchemaModel.CoreRecordSize}, {f.Size}),"); + } + } + sb.AppendLine("]"); + sb.AppendLine(); + + // Core struct format string + sb.AppendLine("# struct.pack format for the 56-byte core record (little-endian)"); + sb.AppendLine("# int64×5, uint16, uint8×2, uint32, uint16×2, uint32"); + sb.AppendLine("CORE_FMT = \" int:"); + sb.AppendLine(" \"\"\"Compute CRC-32 (IEEE) and return as unsigned 32-bit.\"\"\""); + sb.AppendLine(" return binascii.crc32(data) & 0xFFFFFFFF"); + sb.AppendLine(); + + // pack_header + sb.AppendLine(); + sb.AppendLine("def pack_header(price_stream_id: int, created_at: int,"); + sb.AppendLine(" price_scale: int, quantity_scale: int,"); + sb.AppendLine(" schema_id: int = SCHEMA_ID,"); + sb.AppendLine(" file_type: int = FILE_TYPE_RAW,"); + sb.AppendLine(" resampled_config_hash: int = 0) -> bytes:"); + sb.AppendLine(" buf = bytearray(HEADER_SIZE)"); + sb.AppendLine(" buf[0:8] = HEADER_MAGIC"); + sb.AppendLine(" struct.pack_into(\" bytes:"); + sb.AppendLine(" \"\"\"Pack a full record (core + extension) with computed CRC.\"\"\""); + sb.AppendLine(" core_data = struct.pack("); + sb.AppendLine(" CORE_FMT,"); + sb.AppendLine(" observed_time, write_timestamp, price_stream_id, price, quantity,"); + sb.AppendLine(" 0, # _reserved"); + sb.AppendLine(" record_type, record_side, sequence, level, flags,"); + sb.AppendLine(" 0, # CRC placeholder"); + sb.AppendLine(" )"); + sb.AppendLine(" ext = extension_bytes if extension_bytes is not None else b'\\x00' * EXTENSION_SIZE"); + sb.AppendLine(" if len(ext) != EXTENSION_SIZE:"); + sb.AppendLine(" raise ValueError(f\"Extension bytes must be {EXTENSION_SIZE} bytes, got {len(ext)}\")"); + sb.AppendLine(" # CRC covers bytes 0..51 + extension bytes (skipping CRC field at 52..55)"); + sb.AppendLine(" crc = compute_crc32(core_data[:52] + ext)"); + sb.AppendLine(" core_data = struct.pack("); + sb.AppendLine(" CORE_FMT,"); + sb.AppendLine(" observed_time, write_timestamp, price_stream_id, price, quantity,"); + sb.AppendLine(" 0, record_type, record_side, sequence, level, flags,"); + sb.AppendLine(" crc,"); + sb.AppendLine(" )"); + sb.AppendLine(" return core_data + ext"); + sb.AppendLine(); + + // pack_footer + sb.AppendLine(); + sb.AppendLine("def pack_footer(record_count: int, delta_count: int,"); + sb.AppendLine(" first_write_ts: int, last_write_ts: int,"); + sb.AppendLine(" first_observed: int, last_observed: int,"); + sb.AppendLine(" file_crc: int) -> bytes:"); + sb.AppendLine(" buf = bytearray(FOOTER_SIZE)"); + sb.AppendLine(" struct.pack_into(\""); + sb.AppendLine("// Generated from .fbs schema — do not edit by hand."); + sb.AppendLine(); + sb.AppendLine("import CRC32 from \"crc-32\";"); + sb.AppendLine(); + sb.AppendLine("// ── Constants ──────────────────────────────────────────────────────────────────"); + sb.AppendLine($"export const HEADER_SIZE = {SchemaModel.HeaderSize};"); + sb.AppendLine($"export const FOOTER_SIZE = {SchemaModel.FooterSize};"); + sb.AppendLine($"export const CORE_RECORD_SIZE = {SchemaModel.CoreRecordSize};"); + sb.AppendLine($"export const RECORD_SIZE = {model.RecordSize};"); + sb.AppendLine($"export const EXTENSION_SIZE = {model.ExtensionSize};"); + sb.AppendLine($"export const FORMAT_VERSION = {SchemaModel.FormatVersion};"); + sb.AppendLine($"export const SCHEMA_ID = 0x{model.SchemaId:X8};"); + sb.AppendLine(); + sb.AppendLine("export const HEADER_MAGIC = Buffer.from(\"LEVELS01\", \"ascii\");"); + sb.AppendLine("export const FOOTER_MAGIC = Buffer.from(\"LEVEND01\", \"ascii\");"); + sb.AppendLine("export const FILE_TYPE_RAW = 0;"); + sb.AppendLine(); + + // Extension fields metadata + sb.AppendLine("// Extension fields: [name, offsetFromCore, size]"); + sb.AppendLine("export const EXTENSION_FIELDS: [string, number, number][] = ["); + foreach (var f in model.ExtensionFields) + { + sb.AppendLine($" [\"{f.Name}\", {f.Offset - SchemaModel.CoreRecordSize}, {f.Size}],"); + } + sb.AppendLine("];"); + sb.AppendLine(); + + // RecordData interface + sb.AppendLine("export interface RecordData {"); + sb.AppendLine(" observedTime: bigint;"); + sb.AppendLine(" writeTimestamp: bigint;"); + sb.AppendLine(" priceStreamId: bigint;"); + sb.AppendLine(" price: bigint;"); + sb.AppendLine(" quantity: bigint;"); + sb.AppendLine(" recordType: number;"); + sb.AppendLine(" recordSide: number;"); + sb.AppendLine(" sequence: number;"); + sb.AppendLine(" level: number;"); + sb.AppendLine(" flags: number;"); + sb.AppendLine("}"); + sb.AppendLine(); + + // computeCrc32 + sb.AppendLine("export function computeCrc32(buf: Buffer): number {"); + sb.AppendLine(" return CRC32.buf(buf) >>> 0;"); + sb.AppendLine("}"); + sb.AppendLine(); + + // packHeader + sb.AppendLine("export function packHeader("); + sb.AppendLine(" priceStreamId: bigint,"); + sb.AppendLine(" createdAt: bigint,"); + sb.AppendLine(" priceScale: number,"); + sb.AppendLine(" quantityScale: number,"); + sb.AppendLine(" schemaId: number = SCHEMA_ID,"); + sb.AppendLine(" fileType: number = FILE_TYPE_RAW,"); + sb.AppendLine(" resampledConfigHash: number = 0,"); + sb.AppendLine("): Buffer {"); + sb.AppendLine(" const buf = Buffer.alloc(HEADER_SIZE);"); + sb.AppendLine(" HEADER_MAGIC.copy(buf, 0);"); + sb.AppendLine(" buf.writeUInt16LE(FORMAT_VERSION, 8);"); + sb.AppendLine(" buf[10] = fileType;"); + sb.AppendLine(" buf[11] = 0; // padding"); + sb.AppendLine(" buf.writeUInt32LE(schemaId, 12);"); + sb.AppendLine(" buf.writeBigInt64LE(priceStreamId, 16);"); + sb.AppendLine(" buf.writeBigInt64LE(createdAt, 24);"); + sb.AppendLine(" buf.writeInt32LE(priceScale, 32);"); + sb.AppendLine(" buf.writeInt32LE(quantityScale, 36);"); + sb.AppendLine(" buf.writeUInt32LE(resampledConfigHash, 40);"); + sb.AppendLine(" buf.writeUInt16LE(RECORD_SIZE, 44);"); + sb.AppendLine(" return buf;"); + sb.AppendLine("}"); + sb.AppendLine(); + + // packRecord + sb.AppendLine("export function packRecord(rec: RecordData, extensionBytes?: Buffer): Buffer {"); + sb.AppendLine(" const buf = Buffer.alloc(RECORD_SIZE);"); + sb.AppendLine(" buf.writeBigInt64LE(rec.observedTime, 0);"); + sb.AppendLine(" buf.writeBigInt64LE(rec.writeTimestamp, 8);"); + sb.AppendLine(" buf.writeBigInt64LE(rec.priceStreamId, 16);"); + sb.AppendLine(" buf.writeBigInt64LE(rec.price, 24);"); + sb.AppendLine(" buf.writeBigInt64LE(rec.quantity, 32);"); + sb.AppendLine(" buf.writeUInt16LE(0, 40); // _reserved"); + sb.AppendLine(" buf[42] = rec.recordType;"); + sb.AppendLine(" buf[43] = rec.recordSide;"); + sb.AppendLine(" buf.writeUInt32LE(rec.sequence, 44);"); + sb.AppendLine(" buf.writeUInt16LE(rec.level, 48);"); + sb.AppendLine(" buf.writeUInt16LE(rec.flags, 50);"); + sb.AppendLine(" // Extension bytes (already zero from Buffer.alloc)"); + sb.AppendLine(" if (extensionBytes) {"); + sb.AppendLine(" extensionBytes.copy(buf, CORE_RECORD_SIZE, 0, EXTENSION_SIZE);"); + sb.AppendLine(" }"); + sb.AppendLine(" // CRC covers bytes 0..51 + bytes 56..RECORD_SIZE"); + sb.AppendLine(" const crc = computeCrc32(Buffer.concat([buf.subarray(0, 52), buf.subarray(56)]));"); + sb.AppendLine(" buf.writeUInt32LE(crc, 52);"); + sb.AppendLine(" return buf;"); + sb.AppendLine("}"); + sb.AppendLine(); + + // packFooter + sb.AppendLine("export function packFooter("); + sb.AppendLine(" recordCount: bigint,"); + sb.AppendLine(" deltaCount: bigint,"); + sb.AppendLine(" firstWriteTs: bigint,"); + sb.AppendLine(" lastWriteTs: bigint,"); + sb.AppendLine(" firstObserved: bigint,"); + sb.AppendLine(" lastObserved: bigint,"); + sb.AppendLine(" fileCrc: number,"); + sb.AppendLine("): Buffer {"); + sb.AppendLine(" const buf = Buffer.alloc(FOOTER_SIZE);"); + sb.AppendLine(" buf.writeBigInt64LE(recordCount, 0);"); + sb.AppendLine(" buf.writeBigInt64LE(deltaCount, 8);"); + sb.AppendLine(" buf.writeBigInt64LE(firstWriteTs, 16);"); + sb.AppendLine(" buf.writeBigInt64LE(lastWriteTs, 24);"); + sb.AppendLine(" buf.writeBigInt64LE(firstObserved, 32);"); + sb.AppendLine(" buf.writeBigInt64LE(lastObserved, 40);"); + sb.AppendLine(" buf.writeUInt32LE(fileCrc, 48);"); + sb.AppendLine(" // 4 bytes padding at 52"); + sb.AppendLine(" FOOTER_MAGIC.copy(buf, 56);"); + sb.AppendLine(" return buf;"); + sb.AppendLine("}"); + sb.AppendLine(); + + return sb.ToString(); + } +} diff --git a/src/Levels.Cli/Levels.Cli.csproj b/src/Levels.Cli/Levels.Cli.csproj new file mode 100644 index 0000000..6dd9585 --- /dev/null +++ b/src/Levels.Cli/Levels.Cli.csproj @@ -0,0 +1,22 @@ + + + + Exe + net10.0 + enable + enable + preview + levels + true + levels + + + + + + + + + + + diff --git a/src/Levels.Cli/Program.cs b/src/Levels.Cli/Program.cs new file mode 100644 index 0000000..43587b1 --- /dev/null +++ b/src/Levels.Cli/Program.cs @@ -0,0 +1,44 @@ +using Levels.Cli.Commands; + +if (args.Length == 0) +{ + PrintUsage(); + return 1; +} + +var command = args[0]; +var commandArgs = args.AsSpan(1).ToArray(); + +return command switch +{ + "inspect" => await InspectCommand.RunAsync(commandArgs), + "compact" => await CompactCommand.RunAsync(commandArgs), + "codegen" => CodegenCommand.Run(commandArgs), + "resample" => await ResampleCommand.RunAsync(commandArgs), + "export" => await ExportCommand.RunAsync(commandArgs), + "--help" or "-h" => PrintUsage(), + _ => PrintUnknown(command), +}; + +static int PrintUsage() +{ + Console.WriteLine("levels — Levels CLI tool"); + Console.WriteLine(); + Console.WriteLine("Usage: levels [options]"); + Console.WriteLine(); + Console.WriteLine("Commands:"); + Console.WriteLine(" inspect Inspect an Levels data file (header, footer, records)"); + Console.WriteLine(" compact Run compaction on a data directory"); + Console.WriteLine(" codegen Generate typed accessors from a .fbs schema file"); + Console.WriteLine(" resample Run batch resampling on existing data"); + Console.WriteLine(" export Export data to CSV, Parquet, or Avro format"); + Console.WriteLine(); + return 0; +} + +static int PrintUnknown(string command) +{ + Console.Error.WriteLine($"Unknown command: {command}"); + Console.Error.WriteLine("Run 'levels --help' for usage."); + return 1; +} diff --git a/src/Levels.Cli/SchemaModel.cs b/src/Levels.Cli/SchemaModel.cs new file mode 100644 index 0000000..96f072c --- /dev/null +++ b/src/Levels.Cli/SchemaModel.cs @@ -0,0 +1,171 @@ +using System.Text.RegularExpressions; + +namespace Levels.Cli; + +internal record BinaryField( + string Name, + string FbsType, + string ClrType, + int Offset, + int Size, + bool IsFixedArray, + int ArrayLength); + +internal record SchemaModel( + string Namespace, + string StructName, + uint SchemaId, + int RecordSize, + IReadOnlyList CoreFields, + IReadOnlyList ExtensionFields, + IReadOnlyList AllFields) +{ + public const int CoreRecordSize = 56; + public const int HeaderSize = 128; + public const int FooterSize = 64; + public const int FormatVersion = 2; + + public int ExtensionSize => RecordSize - CoreRecordSize; + + private static readonly HashSet KnownCoreFields = new(StringComparer.Ordinal) + { + "exchange", "venue", "symbol", + "observed_time", "write_timestamp", "price_stream_id", + "price", "quantity", + "_reserved", "record_type", "record_side", + "sequence", "level", "flags", "crc32", + }; + + private static readonly List CoreBinaryFields = + [ + new("observed_time", "int64", "long", 0, 8, false, 0), + new("write_timestamp", "int64", "long", 8, 8, false, 0), + new("price_stream_id", "int64", "long", 16, 8, false, 0), + new("price", "int64", "long", 24, 8, false, 0), + new("quantity", "int64", "long", 32, 8, false, 0), + new("_reserved", "uint16", "ushort", 40, 2, false, 0), + new("record_type", "uint8", "byte", 42, 1, false, 0), + new("record_side", "uint8", "byte", 43, 1, false, 0), + new("sequence", "uint32", "uint", 44, 4, false, 0), + new("level", "uint16", "ushort", 48, 2, false, 0), + new("flags", "uint16", "ushort", 50, 2, false, 0), + new("crc32", "uint32", "uint", 52, 4, false, 0), + ]; + + private static readonly Dictionary TypeMap = new() + { + ["int64"] = ("long", 8), ["uint64"] = ("ulong", 8), + ["int32"] = ("int", 4), ["uint32"] = ("uint", 4), + ["int16"] = ("short", 2), ["uint16"] = ("ushort", 2), + ["int8"] = ("sbyte", 1), ["uint8"] = ("byte", 1), + ["float32"] = ("float", 4), ["float64"] = ("double", 8), + ["bool"] = ("bool", 1), + ["string"] = ("string", 0), + }; + + public static SchemaModel FromFbs(string fbsText) + { + var parsedFields = ParseFbs(fbsText); + uint schemaId = ComputeFnv1a(fbsText); + var ns = ParseNamespace(fbsText) ?? "Levels.Core.Generated"; + var structName = ParseStructName(fbsText) ?? "Record"; + + var extensionFields = new List(); + int extOffset = 0; + foreach (var f in parsedFields) + { + if (KnownCoreFields.Contains(f.Name)) + continue; + extensionFields.Add(new BinaryField(f.Name, f.FbsType, f.ClrType, + CoreRecordSize + extOffset, f.Size, f.IsFixedArray, f.ArrayLength)); + extOffset += f.Size; + } + + var allFields = new List(CoreBinaryFields); + allFields.AddRange(extensionFields); + + int recordSize = allFields.Count > 0 + ? allFields[^1].Offset + allFields[^1].Size + : CoreRecordSize; + + return new SchemaModel(ns, structName, schemaId, recordSize, + CoreBinaryFields, extensionFields, allFields); + } + + private static List ParseFbs(string fbsText) + { + var fields = new List(); + int offset = 0; + bool inStruct = false; + + foreach (var rawLine in fbsText.Split('\n')) + { + var line = rawLine.Trim(); + var commentIdx = line.IndexOf("//", StringComparison.Ordinal); + if (commentIdx >= 0) line = line[..commentIdx].Trim(); + if (line.Length == 0) continue; + + if (line.StartsWith("struct ") && line.EndsWith("{")) + { + inStruct = true; offset = 0; continue; + } + if (inStruct && line == "}") { inStruct = false; continue; } + if (!inStruct) continue; + + var fieldLine = line.TrimEnd(';'); + var colonIdx = fieldLine.IndexOf(':'); + if (colonIdx < 0) continue; + + var name = fieldLine[..colonIdx].Trim(); + var fbsType = fieldLine[(colonIdx + 1)..].Trim(); + + if (TypeMap.TryGetValue(fbsType, out var mapped)) + { + fields.Add(new BinaryField(name, fbsType, mapped.ClrType, offset, mapped.Size, false, 0)); + if (mapped.Size > 0) + offset += mapped.Size; + } + else + { + var arrayMatch = Regex.Match(fbsType, @"^byte\[(\d+)\]$"); + if (arrayMatch.Success) + { + int arrayLen = int.Parse(arrayMatch.Groups[1].Value); + fields.Add(new BinaryField(name, fbsType, "byte", offset, arrayLen, true, arrayLen)); + offset += arrayLen; + } + } + } + + return fields; + } + + private static string? ParseStructName(string fbsText) + { + foreach (var rawLine in fbsText.Split('\n')) + { + var line = rawLine.Trim(); + if (line.StartsWith("struct ") && line.EndsWith("{")) + return line["struct ".Length..^1].Trim(); + } + return null; + } + + private static string? ParseNamespace(string fbsText) + { + foreach (var rawLine in fbsText.Split('\n')) + { + var line = rawLine.Trim(); + if (line.StartsWith("namespace ") && line.EndsWith(";")) + return line["namespace ".Length..^1].Trim(); + } + return null; + } + + public static uint ComputeFnv1a(string text) + { + uint hash = 2166136261u; + foreach (char c in text) { hash ^= (byte)c; hash *= 16777619u; } + return hash; + } +} diff --git a/src/Levels.Client/Commands/BookCommand.cs b/src/Levels.Client/Commands/BookCommand.cs new file mode 100644 index 0000000..51d1d3f --- /dev/null +++ b/src/Levels.Client/Commands/BookCommand.cs @@ -0,0 +1,49 @@ +using Levels.Protocol; + +namespace Levels.Client.Commands; + +internal static class BookCommand +{ + public static async Task RunAsync(string[] args) + { + var venue = ClientHelpers.GetArg(args, "--venue"); + var symbol = ClientHelpers.GetArg(args, "--symbol"); + + if (venue is null || symbol is null) + { + Console.Error.WriteLine("Usage: levels-client book --server HOST:PORT --venue V --symbol S"); + return 1; + } + + var (client, reader, writer) = await ClientHelpers.ConnectQueryAsync(args); + using (client) + { + await writer.WriteGetBookAsync(venue, symbol, 0, 0, false); + await writer.FlushAsync(); + + var frame = await reader.ReadFrameAsync(); + if (frame is not { } f) + throw new ProtocolException("Server closed connection"); + + if (f.Type == MessageType.ErrorResponse) + { + Console.Error.WriteLine($"Error: {f.ReadErrorResponse()}"); + return 1; + } + + var response = f.ReadGetBookResponse(); + + Console.WriteLine($"Order book for {venue}/{symbol}:"); + Console.WriteLine(); + Console.WriteLine(" Asks:"); + foreach (var (price, qty) in response.Asks.Reverse()) + Console.WriteLine($" {price,15} | {qty,15}"); + Console.WriteLine(" --------"); + Console.WriteLine(" Bids:"); + foreach (var (price, qty) in response.Bids) + Console.WriteLine($" {price,15} | {qty,15}"); + } + + return 0; + } +} diff --git a/src/Levels.Client/Commands/BookL1Command.cs b/src/Levels.Client/Commands/BookL1Command.cs new file mode 100644 index 0000000..8217bad --- /dev/null +++ b/src/Levels.Client/Commands/BookL1Command.cs @@ -0,0 +1,50 @@ +using Levels.Protocol; + +namespace Levels.Client.Commands; + +internal static class BookL1Command +{ + public static async Task RunAsync(string[] args) + { + var venue = ClientHelpers.GetArg(args, "--venue"); + var symbol = ClientHelpers.GetArg(args, "--symbol"); + + if (venue is null || symbol is null) + { + Console.Error.WriteLine("Usage: levels-client book-l1 --server HOST:PORT --venue V --symbol S"); + return 1; + } + + var (client, reader, writer) = await ClientHelpers.ConnectQueryAsync(args); + using (client) + { + await writer.WriteGetBookL1Async(venue, symbol, 0, 0, false); + await writer.FlushAsync(); + + var frame = await reader.ReadFrameAsync(); + if (frame is not { } f) + throw new ProtocolException("Server closed connection"); + + if (f.Type == MessageType.ErrorResponse) + { + Console.Error.WriteLine($"Error: {f.ReadErrorResponse()}"); + return 1; + } + + var response = f.ReadGetBookL1Response(); + + Console.WriteLine($"L1 for {venue}/{symbol}:"); + if (response.BestBid is var (bp, bq)) + Console.WriteLine($" Best Bid: {bp} x {bq}"); + else + Console.WriteLine(" Best Bid: (none)"); + + if (response.BestAsk is var (ap, aq)) + Console.WriteLine($" Best Ask: {ap} x {aq}"); + else + Console.WriteLine(" Best Ask: (none)"); + } + + return 0; + } +} diff --git a/src/Levels.Client/Commands/ClientHelpers.cs b/src/Levels.Client/Commands/ClientHelpers.cs new file mode 100644 index 0000000..404ede9 --- /dev/null +++ b/src/Levels.Client/Commands/ClientHelpers.cs @@ -0,0 +1,69 @@ +using System.Net.Sockets; +using Levels.Protocol; + +namespace Levels.Client.Commands; + +internal static class ClientHelpers +{ + public static string GetServer(string[] args) => GetArg(args, "--server") + ?? Environment.GetEnvironmentVariable("LEVELS_SERVER") + ?? "localhost:5050"; + + public static string? GetArg(string[] args, string name) + { + for (int i = 0; i < args.Length - 1; i++) + { + if (args[i] == name) + return args[i + 1]; + } + return null; + } + + public static (string Host, int Port) ParseEndpoint(string endpoint) + { + var parts = endpoint.Split(':'); + return (parts[0], parts.Length > 1 ? int.Parse(parts[1]) : 5050); + } + + /// + /// Connects to the server and performs handshake. Returns (stream, reader, writer). + /// Caller owns the TcpClient and must dispose it. + /// + public static async Task<(TcpClient Client, FrameReader Reader, FrameWriter Writer)> ConnectAsync( + string[] args, uint schemaId = 0, int recordSize = 0, CancellationToken ct = default) + { + var (host, port) = ParseEndpoint(GetServer(args)); + var client = new TcpClient(); + await client.ConnectAsync(host, port, ct); + var stream = client.GetStream(); + var reader = new FrameReader(stream); + var writer = new FrameWriter(stream); + + // Handshake + await writer.WriteHandshakeAsync(schemaId, recordSize, ct); + await writer.FlushAsync(ct); + + var ackFrame = await reader.ReadFrameAsync(ct); + if (ackFrame is not { } af) + throw new ProtocolException("Server closed connection during handshake"); + + if (af.Type != MessageType.HandshakeAck) + throw new ProtocolException($"Expected HandshakeAck, got {af.Type}"); + + var ack = af.ReadHandshakeAck(); + if (ack.Status != HandshakeStatus.Ok) + throw new ProtocolException($"Handshake failed: {ack.Error}"); + + return (client, reader, writer); + } + + /// + /// Connects without schema validation (for query/admin commands). + /// Sends schema_id=0 and record_size=0. + /// + public static Task<(TcpClient Client, FrameReader Reader, FrameWriter Writer)> ConnectQueryAsync( + string[] args, CancellationToken ct = default) + { + return ConnectAsync(args, schemaId: 0, recordSize: 0, ct); + } +} diff --git a/src/Levels.Client/Commands/ExportCommand.cs b/src/Levels.Client/Commands/ExportCommand.cs new file mode 100644 index 0000000..ffcc84d --- /dev/null +++ b/src/Levels.Client/Commands/ExportCommand.cs @@ -0,0 +1,63 @@ +using Levels.Protocol; + +namespace Levels.Client.Commands; + +internal static class ExportCommand +{ + public static async Task RunAsync(string[] args) + { + var venue = ClientHelpers.GetArg(args, "--venue"); + var symbol = ClientHelpers.GetArg(args, "--symbol"); + var format = ClientHelpers.GetArg(args, "--format") ?? "csv"; + var output = ClientHelpers.GetArg(args, "--output"); + + if (venue is null || symbol is null) + { + Console.Error.WriteLine("Usage: levels-client export --server HOST:PORT --venue V --symbol S --format csv [--output file]"); + return 1; + } + + var (client, reader, writer) = await ClientHelpers.ConnectQueryAsync(args); + using (client) + { + await writer.WriteExportAsync(venue, symbol, format, 0, 0); + await writer.FlushAsync(); + + Stream outputStream = output is not null + ? File.Create(output) + : Console.OpenStandardOutput(); + + try + { + while (true) + { + var frame = await reader.ReadFrameAsync(); + if (frame is not { } f) + throw new ProtocolException("Server closed connection during export"); + + if (f.Type == MessageType.ErrorResponse) + { + Console.Error.WriteLine($"Error: {f.ReadErrorResponse()}"); + return 1; + } + + var chunk = f.ReadExportChunk(); + if (chunk.Data.Length == 0) + break; // End marker + + await outputStream.WriteAsync(chunk.Data); + } + } + finally + { + if (output is not null) + await outputStream.DisposeAsync(); + } + + if (output is not null) + Console.WriteLine($"Exported to {output}"); + } + + return 0; + } +} diff --git a/src/Levels.Client/Commands/HealthCommand.cs b/src/Levels.Client/Commands/HealthCommand.cs new file mode 100644 index 0000000..03af84e --- /dev/null +++ b/src/Levels.Client/Commands/HealthCommand.cs @@ -0,0 +1,32 @@ +using Levels.Protocol; + +namespace Levels.Client.Commands; + +internal static class HealthCommand +{ + public static async Task RunAsync(string[] args) + { + var (client, reader, writer) = await ClientHelpers.ConnectQueryAsync(args); + using (client) + { + await writer.WriteGetHealthAsync(); + await writer.FlushAsync(); + + var frame = await reader.ReadFrameAsync(); + if (frame is not { } f) + throw new ProtocolException("Server closed connection"); + + if (f.Type == MessageType.ErrorResponse) + { + Console.Error.WriteLine($"Error: {f.ReadErrorResponse()}"); + return 1; + } + + var response = f.ReadGetHealthResponse(); + Console.WriteLine($"Status: {response.Status}"); + Console.WriteLine($"Timestamp: {response.Timestamp}"); + } + + return 0; + } +} diff --git a/src/Levels.Client/Commands/SchemaCommand.cs b/src/Levels.Client/Commands/SchemaCommand.cs new file mode 100644 index 0000000..f1a2fe4 --- /dev/null +++ b/src/Levels.Client/Commands/SchemaCommand.cs @@ -0,0 +1,33 @@ +using Levels.Protocol; + +namespace Levels.Client.Commands; + +internal static class SchemaCommand +{ + public static async Task RunAsync(string[] args) + { + var (client, reader, writer) = await ClientHelpers.ConnectQueryAsync(args); + using (client) + { + await writer.WriteGetSchemaInfoAsync(); + await writer.FlushAsync(); + + var frame = await reader.ReadFrameAsync(); + if (frame is not { } f) + throw new ProtocolException("Server closed connection"); + + if (f.Type == MessageType.ErrorResponse) + { + Console.Error.WriteLine($"Error: {f.ReadErrorResponse()}"); + return 1; + } + + var response = f.ReadGetSchemaInfoResponse(); + Console.WriteLine($"Schema ID: 0x{response.SchemaId:X8}"); + Console.WriteLine($"Record Size: {response.RecordSize} bytes"); + Console.WriteLine($"Type Name: {response.TypeName}"); + } + + return 0; + } +} diff --git a/src/Levels.Client/Commands/StreamsCommand.cs b/src/Levels.Client/Commands/StreamsCommand.cs new file mode 100644 index 0000000..d242673 --- /dev/null +++ b/src/Levels.Client/Commands/StreamsCommand.cs @@ -0,0 +1,43 @@ +using Levels.Protocol; + +namespace Levels.Client.Commands; + +internal static class StreamsCommand +{ + public static async Task RunAsync(string[] args) + { + var (client, reader, writer) = await ClientHelpers.ConnectQueryAsync(args); + using (client) + { + await writer.WriteGetStreamsAsync(); + await writer.FlushAsync(); + + var frame = await reader.ReadFrameAsync(); + if (frame is not { } f) + throw new ProtocolException("Server closed connection"); + + if (f.Type == MessageType.ErrorResponse) + { + Console.Error.WriteLine($"Error: {f.ReadErrorResponse()}"); + return 1; + } + + var response = f.ReadGetStreamsResponse(); + + if (response.Streams.Count == 0) + { + Console.WriteLine("No streams found."); + return 0; + } + + Console.WriteLine($"{"Venue",-20} {"Symbol",-15}"); + Console.WriteLine(new string('-', 35)); + foreach (var (venue, symbol) in response.Streams) + { + Console.WriteLine($"{venue,-20} {symbol,-15}"); + } + } + + return 0; + } +} diff --git a/src/Levels.Client/Commands/WriteCommand.cs b/src/Levels.Client/Commands/WriteCommand.cs new file mode 100644 index 0000000..06da465 --- /dev/null +++ b/src/Levels.Client/Commands/WriteCommand.cs @@ -0,0 +1,75 @@ +using System.Buffers.Binary; +using Levels.Protocol; + +namespace Levels.Client.Commands; + +internal static class WriteCommand +{ + public static async Task RunAsync(string[] args) + { + var venue = ClientHelpers.GetArg(args, "--venue"); + var symbol = ClientHelpers.GetArg(args, "--symbol"); + var priceStr = ClientHelpers.GetArg(args, "--price"); + var qtyStr = ClientHelpers.GetArg(args, "--qty"); + var typeStr = ClientHelpers.GetArg(args, "--type") ?? "delta"; + var sideStr = ClientHelpers.GetArg(args, "--side") ?? "bid"; + var schemaIdStr = ClientHelpers.GetArg(args, "--schema-id") ?? "0"; + var recordSizeStr = ClientHelpers.GetArg(args, "--record-size") ?? "56"; + + if (venue is null || symbol is null || priceStr is null || qtyStr is null) + { + Console.Error.WriteLine("Usage: levels-client write --server HOST:PORT --schema-id ID --record-size SIZE --venue V --symbol S --price P --qty Q [--type delta|snap] [--side bid|ask]"); + return 1; + } + + var schemaId = uint.Parse(schemaIdStr, System.Globalization.NumberStyles.HexNumber); + var recordSize = int.Parse(recordSizeStr); + + var (client, reader, writer) = await ClientHelpers.ConnectAsync(args, schemaId, recordSize); + using (client) + { + // Build a minimal core record + var recordBytes = new byte[recordSize]; + var observedTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() * 1_000_000; + BinaryPrimitives.WriteInt64LittleEndian(recordBytes.AsSpan(0), observedTime); + // WriteTimestamp at offset 8 — leave 0, server/writer sets it + // PriceStreamId at offset 16 — leave 0, writer computes it + BinaryPrimitives.WriteInt64LittleEndian(recordBytes.AsSpan(24), long.Parse(priceStr)); + BinaryPrimitives.WriteInt64LittleEndian(recordBytes.AsSpan(32), long.Parse(qtyStr)); + recordBytes[42] = typeStr.ToLowerInvariant() switch + { + "snap" => 0, + "delta" => 1, + "tombstone" => 2, + _ => 1, + }; + recordBytes[43] = sideStr.ToLowerInvariant() switch + { + "bid" => 0, + "ask" => 1, + _ => 2, + }; + + await writer.WriteRecordAsync(venue, symbol, recordBytes, CancellationToken.None); + await writer.FlushAsync(); + + // Wait for ack + var frame = await reader.ReadFrameAsync(); + if (frame is not { } f) + throw new ProtocolException("Server closed connection"); + + if (f.Type == MessageType.WriteAck) + { + var ack = f.ReadWriteAck(); + Console.WriteLine($"OK: {ack.Accepted} accepted, {ack.Rejected} rejected"); + } + else if (f.Type == MessageType.ErrorResponse) + { + Console.Error.WriteLine($"Error: {f.ReadErrorResponse()}"); + return 1; + } + } + + return 0; + } +} diff --git a/src/Levels.Client/Commands/WriteStreamCommand.cs b/src/Levels.Client/Commands/WriteStreamCommand.cs new file mode 100644 index 0000000..df22343 --- /dev/null +++ b/src/Levels.Client/Commands/WriteStreamCommand.cs @@ -0,0 +1,104 @@ +using System.Buffers.Binary; +using System.Text.Json; +using Levels.Protocol; + +namespace Levels.Client.Commands; + +internal static class WriteStreamCommand +{ + public static async Task RunAsync(string[] args) + { + var filePath = ClientHelpers.GetArg(args, "--file"); + var schemaIdStr = ClientHelpers.GetArg(args, "--schema-id") ?? "0"; + var recordSizeStr = ClientHelpers.GetArg(args, "--record-size") ?? "56"; + + var schemaId = uint.Parse(schemaIdStr, System.Globalization.NumberStyles.HexNumber); + var recordSize = int.Parse(recordSizeStr); + + var (client, reader, writer) = await ClientHelpers.ConnectAsync(args, schemaId, recordSize); + using (client) + { + // Read acks on background task + var ackTask = Task.Run(async () => + { + while (true) + { + var frame = await reader.ReadFrameAsync(); + if (frame is not { } f) break; + if (f.Type == MessageType.WriteAck) + { + var ack = f.ReadWriteAck(); + Console.WriteLine($"Progress: {ack.Accepted} accepted, {ack.Rejected} rejected"); + } + } + }); + + var textReader = filePath is not null ? File.OpenText(filePath) : Console.In; + try + { + string? line; + while ((line = await textReader.ReadLineAsync()) is not null) + { + if (string.IsNullOrWhiteSpace(line)) continue; + + var record = JsonSerializer.Deserialize(line, JsonOptions); + if (record is null) continue; + + var recordBytes = new byte[recordSize]; + BinaryPrimitives.WriteInt64LittleEndian(recordBytes.AsSpan(0), record.ObservedTime); + BinaryPrimitives.WriteInt64LittleEndian(recordBytes.AsSpan(24), record.Price); + BinaryPrimitives.WriteInt64LittleEndian(recordBytes.AsSpan(32), record.Quantity); + recordBytes[42] = ParseType(record.Type); + recordBytes[43] = ParseSide(record.Side); + + await writer.WriteRecordAsync(record.Venue ?? "", record.Symbol ?? "", recordBytes); + } + } + finally + { + if (filePath is not null) + textReader.Dispose(); + } + + await writer.FlushAsync(); + + // Close our write side so server sends final ack + client.Client.Shutdown(System.Net.Sockets.SocketShutdown.Send); + await ackTask; + } + + Console.WriteLine("Stream complete."); + return 0; + } + + private static byte ParseType(string? type) => type?.ToLowerInvariant() switch + { + "snap" => 0, + "delta" => 1, + "tombstone" => 2, + _ => 1, + }; + + private static byte ParseSide(string? side) => side?.ToLowerInvariant() switch + { + "bid" => 0, + "ask" => 1, + _ => 2, + }; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true, + }; + + private sealed class JsonRecord + { + public string? Venue { get; set; } + public string? Symbol { get; set; } + public long ObservedTime { get; set; } + public long Price { get; set; } + public long Quantity { get; set; } + public string? Type { get; set; } + public string? Side { get; set; } + } +} diff --git a/src/Levels.Client/Levels.Client.csproj b/src/Levels.Client/Levels.Client.csproj new file mode 100644 index 0000000..4de76c5 --- /dev/null +++ b/src/Levels.Client/Levels.Client.csproj @@ -0,0 +1,17 @@ + + + + Exe + net10.0 + enable + enable + preview + levels-client + false + + + + + + + diff --git a/src/Levels.Client/Program.cs b/src/Levels.Client/Program.cs new file mode 100644 index 0000000..5fcc853 --- /dev/null +++ b/src/Levels.Client/Program.cs @@ -0,0 +1,48 @@ +using Levels.Client.Commands; + +if (args.Length == 0) +{ + PrintUsage(); + return 1; +} + +var command = args[0].ToLowerInvariant(); +var commandArgs = args[1..]; + +return await (command switch +{ + "write" => WriteCommand.RunAsync(commandArgs), + "write-stream" => WriteStreamCommand.RunAsync(commandArgs), + "streams" => StreamsCommand.RunAsync(commandArgs), + "book" => BookCommand.RunAsync(commandArgs), + "book-l1" => BookL1Command.RunAsync(commandArgs), + "export" => ExportCommand.RunAsync(commandArgs), + "health" => HealthCommand.RunAsync(commandArgs), + "schema" => SchemaCommand.RunAsync(commandArgs), + _ => Task.FromResult(UnknownCommand(command)), +}); + +static void PrintUsage() +{ + Console.WriteLine("Usage: levels-client [options]"); + Console.WriteLine(); + Console.WriteLine("Commands:"); + Console.WriteLine(" write Write a single record"); + Console.WriteLine(" write-stream Stream records from JSONL file or stdin"); + Console.WriteLine(" streams List available streams"); + Console.WriteLine(" book Get L2 order book snapshot"); + Console.WriteLine(" book-l1 Get L1 (best bid/ask)"); + Console.WriteLine(" export Export data to file"); + Console.WriteLine(" health Check server health"); + Console.WriteLine(" schema Get schema info"); + Console.WriteLine(); + Console.WriteLine("Options:"); + Console.WriteLine(" --server HOST:PORT Server address (default: LEVELS_SERVER env or localhost:5050)"); +} + +static int UnknownCommand(string command) +{ + Console.Error.WriteLine($"Unknown command: {command}"); + PrintUsage(); + return 1; +} diff --git a/src/Levels.Compaction/AggFileWriter.cs b/src/Levels.Compaction/AggFileWriter.cs new file mode 100644 index 0000000..4a80d2a --- /dev/null +++ b/src/Levels.Compaction/AggFileWriter.cs @@ -0,0 +1,67 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; + +namespace Levels.Compaction; + +public sealed class AggFileWriter +{ + public static async Task WriteAsync( + IEnumerable records, + string targetPath, + PriceStreamId priceStreamId, + int priceScale, + int quantityScale, + IReadOnlyList sourceRawFiles, + int recordSize = Constants.CoreRecordSize) + { + Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!); + + await using var fileStream = new FileStream( + targetPath, + FileMode.CreateNew, + FileAccess.Write, + FileShare.None, + bufferSize: 4096, + FileOptions.WriteThrough | FileOptions.Asynchronous); + + await using var writer = await BinaryRecordWriter.CreateAsync( + fileStream, + FileType.Agg, + priceStreamId, + priceScale, + quantityScale, + recordSize: recordSize); + + long syntheticSnapCount = 0; + + foreach (var record in records) + { + var orderIdBytes = record.OrderId; + await writer.WriteRecordAsync(record.Core, orderIdBytes); + + if (record.Core.Type == RecordType.Snap && (record.Core.Flags & Constants.SyntheticSnapFlag) != 0) + syntheticSnapCount++; + } + + await writer.SealAsync(); + + return new AggFileInfo( + targetPath, + priceStreamId, + writer.RecordCount, + writer.DeltaCount, + syntheticSnapCount, + writer.FirstObservedTime, + writer.LastObservedTime, + sourceRawFiles); + } + + public static string ComputeAggPath(string dataPath, string venue, long priceStreamId, long windowStartNanos) + { + var windowDate = DateTimeOffset.FromUnixTimeMilliseconds(windowStartNanos / 1_000_000); + var dir = Path.Combine(dataPath, venue, priceStreamId.ToString()); + var fileName = $"{windowDate.UtcDateTime:yyyyMMdd_HHmmss}.agg"; + return Path.Combine(dir, fileName); + } +} diff --git a/src/Levels.Compaction/ArchivalConfig.cs b/src/Levels.Compaction/ArchivalConfig.cs new file mode 100644 index 0000000..daa6afc --- /dev/null +++ b/src/Levels.Compaction/ArchivalConfig.cs @@ -0,0 +1,10 @@ +namespace Levels.Compaction; + +public sealed class ArchivalConfig +{ + public required string DataPath { get; init; } + public TimeSpan RawRetention { get; init; } = TimeSpan.FromDays(7); + public TimeSpan AggRetention { get; init; } = TimeSpan.FromDays(30); + public TimeSpan? PeriodRetention { get; init; } // null = never delete + public TimeSpan ScanInterval { get; init; } = TimeSpan.FromHours(1); +} diff --git a/src/Levels.Compaction/ArchivalHandler.cs b/src/Levels.Compaction/ArchivalHandler.cs new file mode 100644 index 0000000..9a2fc01 --- /dev/null +++ b/src/Levels.Compaction/ArchivalHandler.cs @@ -0,0 +1,206 @@ +using Microsoft.Extensions.Hosting; +using Levels.Core; +using Levels.Core.Diagnostics; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.DataFlow; +using Levels.Query; + +namespace Levels.Compaction; + +public sealed class ArchivalHandler : IDataFlowHandler, IHostedService, IAsyncDisposable +{ + private readonly ArchivalConfig _config; + private readonly FileIndex _fileIndex; + private Task? _backgroundTask; + private CancellationTokenSource? _cts; + + public ArchivalHandler(ArchivalConfig config, FileIndex fileIndex) + { + _config = config; + _fileIndex = fileIndex; + } + + public ValueTask OnRecordWritten(PriceStreamId stream, RawRecord record, CancellationToken ct) + => ValueTask.CompletedTask; + + public ValueTask OnFileSealed(SealedFileInfo file, CancellationToken ct) + => ValueTask.CompletedTask; + + public ValueTask OnAggCreated(AggFileInfo file, CancellationToken ct) + { + // Check if source RAW files exceed retention + var nowNanos = WriteTimestamp.Now(); + var retentionNanos = _config.RawRetention.Ticks * 100; + + foreach (var rawPath in file.SourceRawFiles) + { + try + { + using var fs = File.OpenRead(rawPath); + var reader = new BinaryRecordReader(fs); + if (reader.Footer is null) continue; + + if (nowNanos - reader.Footer.Value.LastObservedTime > retentionNanos) + { + SoftDelete(rawPath); + } + } + catch { } + } + + return ValueTask.CompletedTask; + } + + public Task StartAsync(CancellationToken cancellationToken) + { + _cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + _backgroundTask = RunScanLoopAsync(_cts.Token); + return Task.CompletedTask; + } + + private async Task RunScanLoopAsync(CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + try + { + await Task.Delay(_config.ScanInterval, ct); + ScanAndCleanup(); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) { break; } + catch { } + } + } + + private void ScanAndCleanup() + { + if (!Directory.Exists(_config.DataPath)) return; + + var nowNanos = WriteTimestamp.Now(); + var rawRetentionNanos = _config.RawRetention.Ticks * 100; + var aggRetentionNanos = _config.AggRetention.Ticks * 100; + + // Scan RAW files + foreach (var rawFile in Directory.GetFiles(_config.DataPath, "*.raw", SearchOption.AllDirectories)) + { + try + { + using var fs = File.OpenRead(rawFile); + if (fs.Length < Constants.HeaderSize + Constants.FooterSize) continue; + var reader = new BinaryRecordReader(fs); + if (reader.Footer is null) continue; + + if (nowNanos - reader.Footer.Value.LastObservedTime > rawRetentionNanos) + { + // Check that covering AGG exists + var header = reader.Header; + var streamId = new PriceStreamId(header.PriceStreamId); + if (_fileIndex.HasFile(streamId, FileType.Agg, + reader.Footer.Value.FirstObservedTime, reader.Footer.Value.LastObservedTime)) + { + fs.Dispose(); + SoftDelete(rawFile); + } + } + } + catch { } + } + + // Scan AGG files + if (_config.PeriodRetention is not null) // Only if period retention set, AGGs can be cleaned + { + foreach (var aggFile in Directory.GetFiles(_config.DataPath, "*.agg", SearchOption.AllDirectories)) + { + try + { + using var fs = File.OpenRead(aggFile); + if (fs.Length < Constants.HeaderSize + Constants.FooterSize) continue; + var reader = new BinaryRecordReader(fs); + if (reader.Footer is null) continue; + + if (nowNanos - reader.Footer.Value.LastObservedTime > aggRetentionNanos) + { + var header = reader.Header; + var streamId = new PriceStreamId(header.PriceStreamId); + if (_fileIndex.HasFile(streamId, FileType.Period, + reader.Footer.Value.FirstObservedTime, reader.Footer.Value.LastObservedTime)) + { + fs.Dispose(); + SoftDelete(aggFile); + } + } + } + catch { } + } + } + + // Clean up trash after 24h + CleanupTrash(); + } + + private void SoftDelete(string filePath) + { + var dir = Path.GetDirectoryName(filePath)!; + var trashDir = Path.Combine(dir, ".trash"); + Directory.CreateDirectory(trashDir); + + var trashPath = Path.Combine(trashDir, Path.GetFileName(filePath)); + try + { + File.Move(filePath, trashPath); + _fileIndex.Remove(filePath); + StlthLevelsMetrics.ArchivalFilesDeleted.Add(1); + } + catch { } + } + + private void CleanupTrash() + { + if (!Directory.Exists(_config.DataPath)) return; + + foreach (var trashDir in Directory.GetDirectories(_config.DataPath, ".trash", SearchOption.AllDirectories)) + { + foreach (var file in Directory.GetFiles(trashDir)) + { + try + { + var lastWrite = File.GetLastWriteTimeUtc(file); + if (DateTime.UtcNow - lastWrite > TimeSpan.FromHours(24)) + { + File.Delete(file); + } + } + catch { } + } + + // Remove empty trash directories + try + { + if (Directory.GetFiles(trashDir).Length == 0) + Directory.Delete(trashDir); + } + catch { } + } + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + if (_cts is not null) + { + await _cts.CancelAsync(); + if (_backgroundTask is not null) + { + try { await _backgroundTask; } + catch (OperationCanceledException) { } + } + _cts.Dispose(); + _cts = null; + } + } + + public async ValueTask DisposeAsync() + { + await StopAsync(CancellationToken.None); + } +} diff --git a/src/Levels.Compaction/CompactionConfig.cs b/src/Levels.Compaction/CompactionConfig.cs new file mode 100644 index 0000000..98896b7 --- /dev/null +++ b/src/Levels.Compaction/CompactionConfig.cs @@ -0,0 +1,11 @@ +namespace Levels.Compaction; + +public sealed class CompactionConfig +{ + public required string DataPath { get; init; } + public TimeSpan CompactionWindow { get; init; } = TimeSpan.FromHours(1); + public int SyntheticSnapIntervalDeltas { get; init; } = 1000; + public TimeSpan? SyntheticSnapIntervalTime { get; init; } + public TimeSpan RetentionWindow { get; init; } = TimeSpan.FromDays(7); + public TimeSpan WindowGracePeriod { get; init; } = TimeSpan.FromMinutes(5); +} diff --git a/src/Levels.Compaction/CompactionWindowKey.cs b/src/Levels.Compaction/CompactionWindowKey.cs new file mode 100644 index 0000000..c4f407f --- /dev/null +++ b/src/Levels.Compaction/CompactionWindowKey.cs @@ -0,0 +1,18 @@ +using Levels.Core; + +namespace Levels.Compaction; + +public readonly record struct CompactionWindowKey( + PriceStreamId StreamId, + string Venue, + long WindowStartNanos) +{ + public static long ComputeWindowStart(long observedTimeNanos, TimeSpan windowSpan) + { + var windowNanos = windowSpan.Ticks * 100; + if (windowNanos <= 0) + throw new ArgumentException("Window span must be positive.", nameof(windowSpan)); + + return observedTimeNanos / windowNanos * windowNanos; + } +} diff --git a/src/Levels.Compaction/EventSourcingCompaction.cs b/src/Levels.Compaction/EventSourcingCompaction.cs new file mode 100644 index 0000000..52d66ac --- /dev/null +++ b/src/Levels.Compaction/EventSourcingCompaction.cs @@ -0,0 +1,256 @@ +using System.Diagnostics; +using System.Threading.Channels; +using Microsoft.Extensions.Hosting; +using Levels.Core; +using Levels.Core.Diagnostics; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.DataFlow; + +namespace Levels.Compaction; + +public sealed class EventSourcingCompaction : IDataFlowHandler, IHostedService, IAsyncDisposable +{ + private readonly CompactionConfig _config; + private readonly DataFlowBus? _bus; + private readonly Dictionary> _windowFiles = new(); + private readonly Channel _workChannel; + private Task? _backgroundTask; + private CancellationTokenSource? _cts; + + public EventSourcingCompaction(CompactionConfig config, DataFlowBus? bus = null) + { + _config = config; + _bus = bus; + _workChannel = Channel.CreateUnbounded(); + } + + public ValueTask OnRecordWritten(PriceStreamId stream, RawRecord record, CancellationToken ct) + => ValueTask.CompletedTask; + + public ValueTask OnFileSealed(SealedFileInfo file, CancellationToken ct) + { + var windowStart = CompactionWindowKey.ComputeWindowStart( + file.FirstObservedTime, _config.CompactionWindow); + var windowEndNanos = windowStart + _config.CompactionWindow.Ticks * 100; + + // Extract venue from file path: {dataPath}/{venue}/{streamId}/file.raw + var venue = ExtractVenue(file.FilePath); + var key = new CompactionWindowKey(file.PriceStreamId, venue, windowStart); + + lock (_windowFiles) + { + if (!_windowFiles.TryGetValue(key, out var files)) + { + files = []; + _windowFiles[key] = files; + } + files.Add(file); + } + + // Check if window is closed (wall clock past window end + grace) + var nowNanos = WriteTimestamp.Now(); + if (nowNanos > windowEndNanos + _config.WindowGracePeriod.Ticks * 100) + { + _workChannel.Writer.TryWrite(key); + } + + return ValueTask.CompletedTask; + } + + public ValueTask OnAggCreated(AggFileInfo file, CancellationToken ct) + => ValueTask.CompletedTask; + + public Task StartAsync(CancellationToken cancellationToken) + { + // Startup recovery: clean up partial AGG files and enqueue missing windows + RecoverOnStartup(); + + _cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + _backgroundTask = Task.Run(() => ProcessWorkAsync(_cts.Token), _cts.Token); + return Task.CompletedTask; + } + + private void RecoverOnStartup() + { + if (!Directory.Exists(_config.DataPath)) + return; + + // Delete partial AGG files (no valid footer) + foreach (var aggFile in Directory.GetFiles(_config.DataPath, "*.agg", SearchOption.AllDirectories)) + { + try + { + using var fs = File.OpenRead(aggFile); + if (fs.Length < Constants.HeaderSize + Constants.FooterSize) + { + fs.Dispose(); + File.Delete(aggFile); + continue; + } + + var reader = new BinaryRecordReader(fs); + if (reader.Footer is null) + { + fs.Dispose(); + File.Delete(aggFile); + } + } + catch + { + try { File.Delete(aggFile); } catch { } + } + } + + // Find windows with sealed RAW files but no AGG file + foreach (var venueDir in Directory.GetDirectories(_config.DataPath)) + { + var venue = Path.GetFileName(venueDir); + foreach (var streamDir in Directory.GetDirectories(venueDir)) + { + var streamIdStr = Path.GetFileName(streamDir); + if (!long.TryParse(streamIdStr, out var streamIdValue)) + continue; + + var priceStreamId = new PriceStreamId(streamIdValue); + var rawFiles = RawFileLocator.FindSealedFiles(_config.DataPath, venue, streamIdValue); + var aggFiles = Directory.GetFiles(streamDir, "*.agg") + .Where(f => IsSealedAgg(f)) + .ToHashSet(); + + foreach (var rawFile in rawFiles) + { + using var fs = File.OpenRead(rawFile); + var reader = new BinaryRecordReader(fs); + if (reader.Footer is null) continue; + + var windowStart = CompactionWindowKey.ComputeWindowStart( + reader.Footer.Value.FirstObservedTime, _config.CompactionWindow); + var aggPath = AggFileWriter.ComputeAggPath( + _config.DataPath, venue, streamIdValue, windowStart); + + if (!aggFiles.Contains(aggPath)) + { + var key = new CompactionWindowKey(priceStreamId, venue, windowStart); + _workChannel.Writer.TryWrite(key); + } + } + } + } + } + + private static bool IsSealedAgg(string path) + { + try + { + using var fs = File.OpenRead(path); + if (fs.Length < Constants.HeaderSize + Constants.FooterSize) + return false; + var reader = new BinaryRecordReader(fs); + return reader.Footer is not null; + } + catch + { + return false; + } + } + + private async Task ProcessWorkAsync(CancellationToken ct) + { + var processed = new HashSet(); + + await foreach (var key in _workChannel.Reader.ReadAllAsync(ct)) + { + if (!processed.Add(key)) + continue; // Already processed this window + + try + { + await CompactWindowAsync(key, ct); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + break; + } + catch + { + // Log error in production; for now skip failed windows + } + } + } + + private async Task CompactWindowAsync(CompactionWindowKey key, CancellationToken ct) + { + using var activity = StlthLevelsMetrics.ActivitySource.StartActivity("levels.compaction"); + activity?.SetTag("stream_id", key.StreamId.Value); + activity?.SetTag("venue", key.Venue); + var start = Stopwatch.GetTimestamp(); + + var rawFiles = RawFileLocator.FindSealedFilesInWindow( + _config.DataPath, key.Venue, key.StreamId.Value, + key.WindowStartNanos, _config.CompactionWindow); + + if (rawFiles.Count == 0) + return; + + var aggPath = AggFileWriter.ComputeAggPath( + _config.DataPath, key.Venue, key.StreamId.Value, key.WindowStartNanos); + + // Skip if AGG already exists and is sealed + if (File.Exists(aggPath) && IsSealedAgg(aggPath)) + return; + + // Read header from first RAW file to get scales + int priceScale = 0, quantityScale = 0; + using (var fs = File.OpenRead(rawFiles[0])) + { + var reader = new BinaryRecordReader(fs); + priceScale = reader.Header.PriceScale; + quantityScale = reader.Header.QuantityScale; + } + + var engine = new OrderbookReplayEngine(_config); + var replayRecords = engine.Replay(rawFiles); + + var aggInfo = await AggFileWriter.WriteAsync( + replayRecords, + aggPath, + key.StreamId, + priceScale, + quantityScale, + rawFiles); + + StlthLevelsMetrics.CompactionDuration.Record(Stopwatch.GetElapsedTime(start).TotalMilliseconds); + StlthLevelsMetrics.CompactionCompleted.Add(1); + + if (_bus is { } bus) + await bus.PublishAggCreatedAsync(aggInfo); + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + _workChannel.Writer.TryComplete(); + + if (_backgroundTask is not null) + { + try { await _backgroundTask; } + catch (OperationCanceledException) { } + } + + _cts?.Dispose(); + _cts = null; + } + + public async ValueTask DisposeAsync() + { + await StopAsync(CancellationToken.None); + } + + private string ExtractVenue(string filePath) + { + // Path format: {dataPath}/{venue}/{streamId}/file.raw + var relativePath = Path.GetRelativePath(_config.DataPath, filePath); + var parts = relativePath.Split(Path.DirectorySeparatorChar); + return parts.Length >= 3 ? parts[0] : "unknown"; + } +} diff --git a/src/Levels.Compaction/Levels.Compaction.csproj b/src/Levels.Compaction/Levels.Compaction.csproj new file mode 100644 index 0000000..5859284 --- /dev/null +++ b/src/Levels.Compaction/Levels.Compaction.csproj @@ -0,0 +1,24 @@ + + + + net10.0 + enable + enable + preview + + + + + + + + + + + + + + + + + diff --git a/src/Levels.Compaction/Log.cs b/src/Levels.Compaction/Log.cs new file mode 100644 index 0000000..2bd17b8 --- /dev/null +++ b/src/Levels.Compaction/Log.cs @@ -0,0 +1,15 @@ +using Microsoft.Extensions.Logging; + +namespace Levels.Compaction; + +internal static partial class Log +{ + [LoggerMessage(Level = LogLevel.Information, Message = "Compaction started: Venue={Venue}, StreamId={StreamId}, WindowStart={WindowStart}")] + public static partial void CompactionStarted(ILogger logger, string venue, string streamId, string windowStart); + + [LoggerMessage(Level = LogLevel.Information, Message = "Compaction completed: Venue={Venue}, StreamId={StreamId}, RecordCount={RecordCount}")] + public static partial void CompactionCompleted(ILogger logger, string venue, string streamId, long recordCount); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Compaction recovery: FilePath={FilePath}")] + public static partial void CompactionRecovery(ILogger logger, string filePath); +} diff --git a/src/Levels.Compaction/OrderbookReplayEngine.cs b/src/Levels.Compaction/OrderbookReplayEngine.cs new file mode 100644 index 0000000..62d90e2 --- /dev/null +++ b/src/Levels.Compaction/OrderbookReplayEngine.cs @@ -0,0 +1,143 @@ +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Core.Orderbook; + +namespace Levels.Compaction; + +public sealed class OrderbookReplayEngine +{ + private readonly CompactionConfig _config; + + public OrderbookReplayEngine(CompactionConfig config) + { + _config = config; + } + + public IEnumerable Replay( + IReadOnlyList sortedRawFiles) + { + var bids = new OrderbookSide(); + var asks = new OrderbookSide(); + var deltaSinceSnap = 0; + var lastSnapTimeNanos = 0L; + var isFirstFile = true; + + foreach (var filePath in sortedRawFiles) + { + using var fs = File.OpenRead(filePath); + var reader = new BinaryRecordReader(fs); + var isLeadingSnap = true; + + foreach (var record in reader.ReadRecords(validateCrc: true)) + { + var core = record.Core; + + if (core.Type == RecordType.Snap) + { + // Leading SNAPs reset orderbook state at file boundaries + if (isLeadingSnap && !isFirstFile) + { + var side = core.Side == RecordSide.Bid ? bids : asks; + if (isLeadingSnap) + { + } + } + + var obSide = core.Side == RecordSide.Bid ? bids : asks; + obSide.Apply(core.Price, core.Quantity); + + yield return record; + } + else + { + isLeadingSnap = false; + + if (core.Type == RecordType.Delta) + { + var obSide = core.Side == RecordSide.Bid ? bids : asks; + obSide.Apply(core.Price, core.Quantity); + deltaSinceSnap++; + + if (lastSnapTimeNanos == 0) + lastSnapTimeNanos = core.ObservedTime; + } + + yield return record; + + // Check if synthetic SNAP should be emitted + if (ShouldEmitSyntheticSnap(deltaSinceSnap, core.ObservedTime, lastSnapTimeNanos)) + { + foreach (var snap in EmitSyntheticSnaps(bids, asks, core.ObservedTime, core.PriceStreamId)) + yield return snap; + + deltaSinceSnap = 0; + lastSnapTimeNanos = core.ObservedTime; + } + } + } + + // After first file's leading SNAPs, subsequent files reset state + if (isFirstFile) + { + isFirstFile = false; + if (lastSnapTimeNanos == 0) + lastSnapTimeNanos = bids.Levels.Any() || asks.Levels.Any() + ? 0 // will be set on first delta + : 0; + } + } + } + + private bool ShouldEmitSyntheticSnap(int deltaSinceSnap, long currentTimeNanos, long lastSnapTimeNanos) + { + if (deltaSinceSnap >= _config.SyntheticSnapIntervalDeltas) + return true; + + if (_config.SyntheticSnapIntervalTime is { } interval && lastSnapTimeNanos > 0) + { + var intervalNanos = interval.Ticks * 100; + if (currentTimeNanos - lastSnapTimeNanos >= intervalNanos) + return true; + } + + return false; + } + + private static IEnumerable EmitSyntheticSnaps( + OrderbookSide bids, OrderbookSide asks, long observedTime, long priceStreamId) + { + uint seq = 0; + + foreach (var level in bids.Levels) + { + var core = new CoreRecordLayout + { + ObservedTime = observedTime, + PriceStreamId = priceStreamId, + Price = level.Key, + Quantity = level.Value, + Type = RecordType.Snap, + Side = RecordSide.Bid, + Flags = Constants.SyntheticSnapFlag, + Sequence = seq++, + }; + yield return new RawRecord { Core = core }; + } + + foreach (var level in asks.Levels) + { + var core = new CoreRecordLayout + { + ObservedTime = observedTime, + PriceStreamId = priceStreamId, + Price = level.Key, + Quantity = level.Value, + Type = RecordType.Snap, + Side = RecordSide.Ask, + Flags = Constants.SyntheticSnapFlag, + Sequence = seq++, + }; + yield return new RawRecord { Core = core }; + } + } +} diff --git a/src/Levels.Compaction/RawFileLocator.cs b/src/Levels.Compaction/RawFileLocator.cs new file mode 100644 index 0000000..d8f3994 --- /dev/null +++ b/src/Levels.Compaction/RawFileLocator.cs @@ -0,0 +1,67 @@ +using Levels.Core.Format; +using Levels.Core.IO; + +namespace Levels.Compaction; + +public static class RawFileLocator +{ + public static List FindSealedFiles(string dataPath, string venue, long priceStreamId) + { + var dir = Path.Combine(dataPath, venue, priceStreamId.ToString()); + if (!Directory.Exists(dir)) + return []; + + var result = new List(); + foreach (var file in Directory.GetFiles(dir, "*.raw").OrderBy(f => f)) + { + if (IsSealedFile(file)) + result.Add(file); + } + + return result; + } + + public static List FindSealedFilesInWindow( + string dataPath, string venue, long priceStreamId, + long windowStartNanos, TimeSpan windowSpan) + { + var windowEndNanos = windowStartNanos + windowSpan.Ticks * 100; + var allSealed = FindSealedFiles(dataPath, venue, priceStreamId); + var result = new List(); + + foreach (var file in allSealed) + { + using var fs = File.OpenRead(file); + var reader = new BinaryRecordReader(fs); + + if (reader.Footer is null) + continue; + + var firstObs = reader.Footer.Value.FirstObservedTime; + var lastObs = reader.Footer.Value.LastObservedTime; + + // File overlaps window if its range intersects [windowStart, windowEnd) + if (firstObs < windowEndNanos && lastObs >= windowStartNanos) + result.Add(file); + } + + return result; + } + + private static bool IsSealedFile(string filePath) + { + try + { + using var fs = File.OpenRead(filePath); + if (fs.Length < Constants.HeaderSize + Constants.FooterSize) + return false; + + var reader = new BinaryRecordReader(fs); + return reader.Footer is not null; + } + catch + { + return false; + } + } +} diff --git a/src/Levels.Core/AggFileInfo.cs b/src/Levels.Core/AggFileInfo.cs new file mode 100644 index 0000000..ba6e322 --- /dev/null +++ b/src/Levels.Core/AggFileInfo.cs @@ -0,0 +1,11 @@ +namespace Levels.Core; + +public readonly record struct AggFileInfo( + string FilePath, + PriceStreamId PriceStreamId, + long RecordCount, + long DeltaCount, + long SyntheticSnapCount, + long FirstObservedTime, + long LastObservedTime, + IReadOnlyList SourceRawFiles); diff --git a/src/Levels.Core/Diagnostics/StlthLevelsMetrics.cs b/src/Levels.Core/Diagnostics/StlthLevelsMetrics.cs new file mode 100644 index 0000000..5e08e71 --- /dev/null +++ b/src/Levels.Core/Diagnostics/StlthLevelsMetrics.cs @@ -0,0 +1,42 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; + +namespace Levels.Core.Diagnostics; + +public static class StlthLevelsMetrics +{ + private static readonly Meter Meter = new("Levels", "1.0.0"); + public static readonly ActivitySource ActivitySource = new("Levels", "1.0.0"); + + // Writer metrics + public static readonly Counter RecordsWritten = Meter.CreateCounter("pricestorage.records.written"); + public static readonly Histogram RecordWriteLatency = Meter.CreateHistogram("pricestorage.record.write_latency", "ms"); + public static readonly Counter FilesSealed = Meter.CreateCounter("pricestorage.files.sealed"); + + // Sink metrics + public static readonly UpDownCounter ActiveStreams = Meter.CreateUpDownCounter("pricestorage.streams.active"); + public static readonly Counter CircuitBreakerTrips = Meter.CreateCounter("pricestorage.circuit_breaker.trips"); + + // Compaction metrics + public static readonly Counter CompactionCompleted = Meter.CreateCounter("pricestorage.compaction.completed"); + public static readonly Histogram CompactionDuration = Meter.CreateHistogram("pricestorage.compaction.duration", "ms"); + + // Health check & promotion metrics + public static readonly Counter HealthCheckPassed = Meter.CreateCounter("pricestorage.healthcheck.passed"); + public static readonly Counter HealthCheckFailed = Meter.CreateCounter("pricestorage.healthcheck.failed"); + public static readonly Counter PromotionsCompleted = Meter.CreateCounter("pricestorage.promotions.completed"); + + // Query metrics + public static readonly Histogram QueryLatency = Meter.CreateHistogram("pricestorage.query.latency", "ms"); + public static readonly Counter QueriesExecuted = Meter.CreateCounter("pricestorage.queries.executed"); + + // DataFlow metrics + public static readonly Counter DataFlowBackpressure = Meter.CreateCounter("pricestorage.dataflow.backpressure"); + + // TCP server metrics + public static readonly UpDownCounter ActiveConnections = Meter.CreateUpDownCounter("pricestorage.connections.active"); + public static readonly Counter TcpMessages = Meter.CreateCounter("pricestorage.tcp.messages"); + + // Archival metrics + public static readonly Counter ArchivalFilesDeleted = Meter.CreateCounter("pricestorage.archival.files_deleted"); +} diff --git a/src/Levels.Core/Format/Constants.cs b/src/Levels.Core/Format/Constants.cs new file mode 100644 index 0000000..fcff1f6 --- /dev/null +++ b/src/Levels.Core/Format/Constants.cs @@ -0,0 +1,30 @@ +namespace Levels.Core.Format; + +/// +/// Levels binary format constants. +/// The canonical byte order is little-endian. All MemoryMarshal operations +/// assume LE layout. A runtime check in the static constructor ensures this. +/// +public static class Constants +{ + static Constants() + { + if (!BitConverter.IsLittleEndian) + throw new PlatformNotSupportedException( + "Levels binary format requires a little-endian platform."); + } + + public static ReadOnlySpan HeaderMagic => "LEVELS01"u8; + public static ReadOnlySpan FooterMagicEnd => "LEVEND01"u8; + + public static readonly ulong HeaderMagicUInt64 = BitConverter.ToUInt64(HeaderMagic); + public static readonly ulong FooterMagicUInt64 = BitConverter.ToUInt64(FooterMagicEnd); + + public const ushort FormatVersion = 2; + public const int HeaderSize = 128; + public const int FooterSize = 64; + public const int CoreRecordSize = 56; + + public const ushort SyntheticSnapFlag = 0x0001; + public const ushort IsOwnerFlag = 0x0002; +} diff --git a/src/Levels.Core/Format/CoreRecordLayout.cs b/src/Levels.Core/Format/CoreRecordLayout.cs new file mode 100644 index 0000000..de88092 --- /dev/null +++ b/src/Levels.Core/Format/CoreRecordLayout.cs @@ -0,0 +1,43 @@ +using System.Runtime.InteropServices; + +namespace Levels.Core.Format; + +[StructLayout(LayoutKind.Explicit, Size = Constants.CoreRecordSize)] +public struct CoreRecordLayout +{ + [FieldOffset(0)] + public long ObservedTime; + + [FieldOffset(8)] + public long WriteTimestamp; + + [FieldOffset(16)] + public long PriceStreamId; + + [FieldOffset(24)] + public long Price; + + [FieldOffset(32)] + public long Quantity; + + [FieldOffset(40)] + public ushort Reserved; + + [FieldOffset(42)] + public RecordType Type; + + [FieldOffset(43)] + public RecordSide Side; + + [FieldOffset(44)] + public uint Sequence; + + [FieldOffset(48)] + public ushort Level; + + [FieldOffset(50)] + public ushort Flags; + + [FieldOffset(52)] + public uint Crc32; +} diff --git a/src/Levels.Core/Format/Crc32.cs b/src/Levels.Core/Format/Crc32.cs new file mode 100644 index 0000000..1c68d42 --- /dev/null +++ b/src/Levels.Core/Format/Crc32.cs @@ -0,0 +1,22 @@ +using System.IO.Hashing; + +namespace Levels.Core.Format; + +public static class Crc32Util +{ + public static uint Compute(ReadOnlySpan data) + { + return Crc32.HashToUInt32(data); + } + + public static uint ComputeRecord(ReadOnlySpan coreWithoutCrc, ReadOnlySpan ext) + { + var crc = new Crc32(); + crc.Append(coreWithoutCrc); + if (ext.Length > 0) + crc.Append(ext); + return crc.GetCurrentHashAsUInt32(); + } + + public static Crc32 CreateIncremental() => new(); +} diff --git a/src/Levels.Core/Format/FileFooter.cs b/src/Levels.Core/Format/FileFooter.cs new file mode 100644 index 0000000..f7bc029 --- /dev/null +++ b/src/Levels.Core/Format/FileFooter.cs @@ -0,0 +1,44 @@ +using System.Runtime.InteropServices; + +namespace Levels.Core.Format; + +[StructLayout(LayoutKind.Explicit, Size = Constants.FooterSize)] +public struct FileFooter +{ + [FieldOffset(0)] + public long RecordCount; + + [FieldOffset(8)] + public long DeltaCount; + + [FieldOffset(16)] + public long FirstWriteTimestamp; + + [FieldOffset(24)] + public long LastWriteTimestamp; + + [FieldOffset(32)] + public long FirstObservedTime; + + [FieldOffset(40)] + public long LastObservedTime; + + [FieldOffset(48)] + public uint FileCrc32; + + [FieldOffset(52)] + public uint FooterCrc32; + + [FieldOffset(56)] + public ulong MagicEnd; + + public static void WriteTo(Span destination, in FileFooter footer) + { + MemoryMarshal.Write(destination, in footer); + } + + public static FileFooter ReadFrom(ReadOnlySpan source) + { + return MemoryMarshal.Read(source); + } +} diff --git a/src/Levels.Core/Format/FileHeader.cs b/src/Levels.Core/Format/FileHeader.cs new file mode 100644 index 0000000..23a14e3 --- /dev/null +++ b/src/Levels.Core/Format/FileHeader.cs @@ -0,0 +1,57 @@ +using System.Runtime.InteropServices; + +namespace Levels.Core.Format; + +[StructLayout(LayoutKind.Explicit, Size = Constants.HeaderSize)] +public struct FileHeader +{ + [FieldOffset(0)] + public ulong Magic; + + [FieldOffset(8)] + public ushort Version; + + [FieldOffset(10)] + public FileType FileType; + + // 1 byte padding at offset 11 + + [FieldOffset(12)] + public uint SchemaId; + + [FieldOffset(16)] + public long PriceStreamId; + + [FieldOffset(24)] + public long CreatedAt; + + [FieldOffset(32)] + public int PriceScale; + + [FieldOffset(36)] + public int QuantityScale; + + /// + /// XxHash32 of the resampled config. Only meaningful when FileType == Resampled; zero otherwise. + /// + [FieldOffset(40)] + public uint ResampledConfigHash; + + /// + /// Total size in bytes of one record (core + schema-defined fields like OrderId). + /// + [FieldOffset(44)] + public ushort RecordSize; + + // Bytes 46..127 are reserved (82 bytes) + + public static void WriteTo(Span destination, in FileHeader header) + { + MemoryMarshal.Write(destination, in header); + } + + public static FileHeader ReadFrom(ReadOnlySpan source) + { + return MemoryMarshal.Read(source); + } +} diff --git a/src/Levels.Core/Format/FileType.cs b/src/Levels.Core/Format/FileType.cs new file mode 100644 index 0000000..52c580b --- /dev/null +++ b/src/Levels.Core/Format/FileType.cs @@ -0,0 +1,9 @@ +namespace Levels.Core.Format; + +public enum FileType : byte +{ + Raw = 0, + Agg = 1, + Period = 2, + Resampled = 3, +} diff --git a/src/Levels.Core/Format/RawRecord.cs b/src/Levels.Core/Format/RawRecord.cs new file mode 100644 index 0000000..394b4cc --- /dev/null +++ b/src/Levels.Core/Format/RawRecord.cs @@ -0,0 +1,15 @@ +namespace Levels.Core.Format; + +public readonly struct RawRecord +{ + public CoreRecordLayout Core { get; init; } + public ReadOnlyMemory RecordBytes { get; init; } + + /// + /// Returns the OrderId portion of the record (bytes beyond the 56-byte core). + /// + public ReadOnlyMemory OrderId => + RecordBytes.Length > Constants.CoreRecordSize + ? RecordBytes[Constants.CoreRecordSize..] + : default; +} diff --git a/src/Levels.Core/Format/RecordSide.cs b/src/Levels.Core/Format/RecordSide.cs new file mode 100644 index 0000000..7e6e7fa --- /dev/null +++ b/src/Levels.Core/Format/RecordSide.cs @@ -0,0 +1,8 @@ +namespace Levels.Core.Format; + +public enum RecordSide : byte +{ + Bid = 0, + Ask = 1, + Unknown = 2, +} diff --git a/src/Levels.Core/Format/RecordType.cs b/src/Levels.Core/Format/RecordType.cs new file mode 100644 index 0000000..8bd293a --- /dev/null +++ b/src/Levels.Core/Format/RecordType.cs @@ -0,0 +1,8 @@ +namespace Levels.Core.Format; + +public enum RecordType : byte +{ + Snap = 0, + Delta = 1, + Tombstone = 2, +} diff --git a/src/Levels.Core/Format/SchemaId.cs b/src/Levels.Core/Format/SchemaId.cs new file mode 100644 index 0000000..d5b7ea4 --- /dev/null +++ b/src/Levels.Core/Format/SchemaId.cs @@ -0,0 +1,18 @@ +namespace Levels.Core.Format; + +public static class SchemaId +{ + private const uint FnvOffsetBasis = 2166136261; + private const uint FnvPrime = 16777619; + + public static uint Compute(ReadOnlySpan data) + { + uint hash = FnvOffsetBasis; + for (int i = 0; i < data.Length; i++) + { + hash ^= data[i]; + hash *= FnvPrime; + } + return hash; + } +} diff --git a/src/Levels.Core/IDataSink.cs b/src/Levels.Core/IDataSink.cs new file mode 100644 index 0000000..263435f --- /dev/null +++ b/src/Levels.Core/IDataSink.cs @@ -0,0 +1,6 @@ +namespace Levels.Core; + +public interface IDataSink +{ + ValueTask WriteAsync(T record, CancellationToken ct = default); +} diff --git a/src/Levels.Core/IO/BinaryRecordReader.cs b/src/Levels.Core/IO/BinaryRecordReader.cs new file mode 100644 index 0000000..13bb206 --- /dev/null +++ b/src/Levels.Core/IO/BinaryRecordReader.cs @@ -0,0 +1,209 @@ +using System.Runtime.InteropServices; +using Levels.Core.Format; +using System.IO.Hashing; + +namespace Levels.Core.IO; + +public sealed class BinaryRecordReader : IDisposable +{ + private readonly Stream _stream; + private readonly FileHeader _header; + private readonly FileFooter? _footer; + private readonly int _recordSize; + private bool _disposed; + + public FileHeader Header => _header; + public FileFooter? Footer => _footer; + public bool IsPartial => _footer is null; + + public BinaryRecordReader(Stream stream) + { + _stream = stream; + + // Read and validate header + Span headerBytes = stackalloc byte[Constants.HeaderSize]; + ReadExact(stream, headerBytes); + _header = FileHeader.ReadFrom(headerBytes); + + if (_header.Magic != Constants.HeaderMagicUInt64) + throw new InvalidDataException("Invalid file header magic bytes."); + + if (_header.Version != Constants.FormatVersion) + throw new InvalidDataException($"Unsupported format version: {_header.Version}."); + + // Determine record size from header + _recordSize = _header.RecordSize > 0 ? _header.RecordSize : Constants.CoreRecordSize; + + if (_recordSize < Constants.CoreRecordSize) + throw new InvalidDataException($"Record size {_recordSize} is smaller than core record size {Constants.CoreRecordSize}."); + + // Try to read footer + _footer = TryReadFooter(stream); + + // Structural size validation for complete files + if (_footer is not null && stream.CanSeek) + { + long dataSize = stream.Length - Constants.HeaderSize - Constants.FooterSize; + if (dataSize < 0) + throw new InvalidDataException("File too small to contain header and footer."); + + if (dataSize % _recordSize != 0) + throw new InvalidDataException( + $"File data region ({dataSize} bytes) is not evenly divisible by record size ({_recordSize} bytes)."); + + long expectedRecords = dataSize / _recordSize; + if (expectedRecords < _footer.Value.RecordCount) + throw new InvalidDataException( + $"File data region ({dataSize} bytes) is too small for {_footer.Value.RecordCount} records."); + } + } + + private static FileFooter? TryReadFooter(Stream stream) + { + if (!stream.CanSeek) + return null; + + var length = stream.Length; + if (length < Constants.HeaderSize + Constants.FooterSize) + return null; + + var savedPosition = stream.Position; + try + { + stream.Seek(-Constants.FooterSize, SeekOrigin.End); + Span footerBytes = stackalloc byte[Constants.FooterSize]; + ReadExact(stream, footerBytes); + var footer = FileFooter.ReadFrom(footerBytes); + + if (footer.MagicEnd != Constants.FooterMagicUInt64) + return null; + + // Validate footer CRC (backward compat: old files have 0 at offset 52) + if (footer.FooterCrc32 != 0) + { + uint computed = Crc32Util.Compute(footerBytes[..52]); + if (computed != footer.FooterCrc32) + return null; + } + + return footer; + } + finally + { + stream.Position = savedPosition; + } + } + + public IEnumerable ReadRecords(bool validateCrc = true) + { + _stream.Position = Constants.HeaderSize; + + long endPosition = IsPartial + ? _stream.Length + : _stream.Length - Constants.FooterSize; + + byte[] recordBuffer = new byte[_recordSize]; + long count = 0; + + while (_stream.Position + _recordSize <= endPosition) + { + int bytesRead = _stream.Read(recordBuffer, 0, _recordSize); + if (bytesRead < _recordSize) + break; // truncated record, discard + + var core = MemoryMarshal.Read(recordBuffer); + + // Validate CRC: record[0..52] + record[56.._recordSize] + if (validateCrc) + { + uint storedCrc = core.Crc32; + var crcData2 = _recordSize > Constants.CoreRecordSize + ? recordBuffer.AsSpan(Constants.CoreRecordSize) + : ReadOnlySpan.Empty; + uint computedCrc = Crc32Util.ComputeRecord( + recordBuffer.AsSpan(0, 52), + crcData2); + + if (storedCrc != computedCrc) + throw new InvalidDataException( + $"CRC mismatch at record {count}: stored=0x{storedCrc:X8}, computed=0x{computedCrc:X8}."); + } + + count++; + yield return new RawRecord + { + Core = core, + RecordBytes = recordBuffer.ToArray(), + }; + } + + // Validate record count for complete files + if (!IsPartial && _footer!.Value.RecordCount != count) + throw new InvalidDataException( + $"Record count mismatch: footer says {_footer.Value.RecordCount}, but read {count}."); + } + + /// + /// Reads CRC-valid records from a partial (unsealed) file, stopping at the first invalid record. + /// Returns the count of valid records and the byte position after the last valid record. + /// + public (List Records, long ValidDataEnd) ReadValidRecordsFromPartial() + { + if (!IsPartial) + return (ReadRecords(validateCrc: true).ToList(), _stream.Length - Constants.FooterSize); + + _stream.Position = Constants.HeaderSize; + var records = new List(); + byte[] recordBuffer = new byte[_recordSize]; + + while (_stream.Position + _recordSize <= _stream.Length) + { + long recordStart = _stream.Position; + int bytesRead = _stream.Read(recordBuffer, 0, _recordSize); + if (bytesRead < _recordSize) + break; + + var core = MemoryMarshal.Read(recordBuffer); + uint storedCrc = core.Crc32; + var crcData2 = _recordSize > Constants.CoreRecordSize + ? recordBuffer.AsSpan(Constants.CoreRecordSize) + : ReadOnlySpan.Empty; + uint computedCrc = Crc32Util.ComputeRecord( + recordBuffer.AsSpan(0, 52), + crcData2); + + if (storedCrc != computedCrc) + break; // Stop at first invalid record + + records.Add(new RawRecord + { + Core = core, + RecordBytes = recordBuffer.ToArray(), + }); + } + + long validEnd = Constants.HeaderSize + (long)records.Count * _recordSize; + return (records, validEnd); + } + + private static void ReadExact(Stream stream, Span buffer) + { + int totalRead = 0; + while (totalRead < buffer.Length) + { + int read = stream.Read(buffer[totalRead..]); + if (read == 0) + throw new EndOfStreamException(); + totalRead += read; + } + } + + public void Dispose() + { + if (!_disposed) + { + _disposed = true; + // Don't dispose the stream — caller owns it. + } + } +} diff --git a/src/Levels.Core/IO/BinaryRecordWriter.cs b/src/Levels.Core/IO/BinaryRecordWriter.cs new file mode 100644 index 0000000..631be18 --- /dev/null +++ b/src/Levels.Core/IO/BinaryRecordWriter.cs @@ -0,0 +1,212 @@ +using System.Buffers; +using System.IO.Hashing; +using System.IO.Pipelines; +using System.Runtime.InteropServices; +using Levels.Core.Format; + +namespace Levels.Core.IO; + +public sealed class BinaryRecordWriter : IAsyncDisposable +{ + private readonly PipeWriter _pipeWriter; + private readonly Stream _stream; + private readonly Crc32 _fileCrc = Crc32Util.CreateIncremental(); + private readonly int _recordSize; + private readonly int _flushThresholdMs; + private readonly int _flushBufferSize; + private int _unflushedCount; + private long _lastFlushTicks; + + private long _recordCount; + private long _deltaCount; + private long _firstWriteTimestamp; + private long _lastWriteTimestamp; + private long _firstObservedTime; + private long _lastObservedTime; + private bool _sealed; + + public long RecordCount => _recordCount; + public long DeltaCount => _deltaCount; + public long FirstObservedTime => _firstObservedTime; + public long LastObservedTime => _lastObservedTime; + + private BinaryRecordWriter(PipeWriter pipeWriter, Stream stream, int recordSize, int flushThresholdMs, int flushBufferSize) + { + _pipeWriter = pipeWriter; + _stream = stream; + _recordSize = recordSize; + _flushThresholdMs = flushThresholdMs; + _flushBufferSize = flushBufferSize; + _lastFlushTicks = Environment.TickCount64; + } + + public static async Task CreateAsync( + Stream stream, + FileType fileType, + PriceStreamId priceStreamId, + int priceScale = 0, + int quantityScale = 0, + uint resampledConfigHash = 0, + uint schemaId = 0, + int recordSize = Constants.CoreRecordSize, + int flushThresholdMs = 0, + int flushBufferSize = 0) + { + var pipeWriter = PipeWriter.Create(stream, new StreamPipeWriterOptions(leaveOpen: true)); + var writer = new BinaryRecordWriter(pipeWriter, stream, recordSize, flushThresholdMs, flushBufferSize); + + var header = new FileHeader + { + Magic = Constants.HeaderMagicUInt64, + Version = Constants.FormatVersion, + FileType = fileType, + SchemaId = schemaId, + PriceStreamId = priceStreamId.Value, + CreatedAt = WriteTimestamp.Now(), + PriceScale = priceScale, + QuantityScale = quantityScale, + ResampledConfigHash = resampledConfigHash, + RecordSize = (ushort)recordSize, + }; + + var span = pipeWriter.GetSpan(Constants.HeaderSize); + span[..Constants.HeaderSize].Clear(); + FileHeader.WriteTo(span, in header); + pipeWriter.Advance(Constants.HeaderSize); + await pipeWriter.FlushAsync(); + + return writer; + } + + public ValueTask WriteRecordAsync(CoreRecordLayout core, ReadOnlyMemory orderIdBytes = default) + { + if (_sealed) + throw new InvalidOperationException("Writer has been sealed."); + + return WriteRecordCoreAsync(core, orderIdBytes); + } + + private async ValueTask WriteRecordCoreAsync(CoreRecordLayout core, ReadOnlyMemory orderIdBytes) + { + core.WriteTimestamp = WriteTimestamp.Now(); + core.Reserved = 0; + + // Build full record buffer + byte[]? rented = null; + Span recordBuf = _recordSize <= 256 + ? stackalloc byte[_recordSize] + : (rented = ArrayPool.Shared.Rent(_recordSize)).AsSpan(0, _recordSize); + + recordBuf.Clear(); + MemoryMarshal.Write(recordBuf, in core); + + // Copy extension bytes into bytes [56.._recordSize] + if (orderIdBytes.Length > 0 && _recordSize > Constants.CoreRecordSize) + { + var orderIdSpan = orderIdBytes.Span; + var destSlice = recordBuf[Constants.CoreRecordSize..]; + if (orderIdSpan.Length > destSlice.Length) + throw new ArgumentException( + $"Extension bytes ({orderIdSpan.Length}) exceed record extension space ({destSlice.Length})."); + orderIdSpan.CopyTo(destSlice); + } + + // Compute CRC over record[0..52] + record[56.._recordSize] + var crcData1 = recordBuf[..52]; + var crcData2 = _recordSize > Constants.CoreRecordSize + ? recordBuf[Constants.CoreRecordSize..] + : ReadOnlySpan.Empty; + core.Crc32 = Crc32Util.ComputeRecord(crcData1, crcData2); + + // Re-write core with CRC set + MemoryMarshal.Write(recordBuf, in core); + + // Write full record to pipe + var destination = _pipeWriter.GetSpan(_recordSize); + recordBuf.CopyTo(destination); + _pipeWriter.Advance(_recordSize); + + // Update file-level CRC + _fileCrc.Append(recordBuf); + + if (rented is not null) + ArrayPool.Shared.Return(rented); + + // Update stats + _recordCount++; + if (core.Type == RecordType.Delta) + _deltaCount++; + + if (_firstWriteTimestamp == 0) + _firstWriteTimestamp = core.WriteTimestamp; + _lastWriteTimestamp = core.WriteTimestamp; + + if (_firstObservedTime == 0 || core.ObservedTime < _firstObservedTime) + _firstObservedTime = core.ObservedTime; + if (core.ObservedTime > _lastObservedTime) + _lastObservedTime = core.ObservedTime; + + _unflushedCount++; + bool batchingEnabled = _flushThresholdMs > 0 || _flushBufferSize > 0; + bool shouldFlush = !batchingEnabled + || (_flushBufferSize > 0 && _unflushedCount >= _flushBufferSize) + || (_flushThresholdMs > 0 && (Environment.TickCount64 - _lastFlushTicks) >= _flushThresholdMs); + + if (shouldFlush) + { + await _pipeWriter.FlushAsync(); + _unflushedCount = 0; + _lastFlushTicks = Environment.TickCount64; + } + + return core; + } + + public async ValueTask SealAsync() + { + if (_sealed) + throw new InvalidOperationException("Writer has already been sealed."); + _sealed = true; + + var footer = new FileFooter + { + RecordCount = _recordCount, + DeltaCount = _deltaCount, + FirstWriteTimestamp = _firstWriteTimestamp, + LastWriteTimestamp = _lastWriteTimestamp, + FirstObservedTime = _firstObservedTime, + LastObservedTime = _lastObservedTime, + FileCrc32 = _fileCrc.GetCurrentHashAsUInt32(), + MagicEnd = Constants.FooterMagicUInt64, + }; + + // Compute footer CRC over bytes [0..52) (all fields before FooterCrc32) + Span tmp = stackalloc byte[Constants.FooterSize]; + tmp.Clear(); + FileFooter.WriteTo(tmp, in footer); + footer.FooterCrc32 = Crc32Util.Compute(tmp[..52]); + + var span = _pipeWriter.GetSpan(Constants.FooterSize); + span[..Constants.FooterSize].Clear(); + FileFooter.WriteTo(span, in footer); + _pipeWriter.Advance(Constants.FooterSize); + + await _pipeWriter.FlushAsync(); + await _pipeWriter.CompleteAsync(); + + // Ensure all data is durable on disk (WriteThrough alone is insufficient on macOS/some Linux) + if (_stream is FileStream fs) + fs.Flush(flushToDisk: true); + else + _stream.Flush(); + } + + public async ValueTask DisposeAsync() + { + if (!_sealed) + { + try { await _pipeWriter.CompleteAsync(); } + catch { /* best effort */ } + } + } +} diff --git a/src/Levels.Core/IO/WriteAheadLog.cs b/src/Levels.Core/IO/WriteAheadLog.cs new file mode 100644 index 0000000..94bff15 --- /dev/null +++ b/src/Levels.Core/IO/WriteAheadLog.cs @@ -0,0 +1,174 @@ +using System.Buffers.Binary; +using System.IO.Hashing; +using System.Text; +using Levels.Core.Format; + +namespace Levels.Core.IO; + +/// +/// Lightweight write-ahead log for buffered/batched writes. +/// Entry format: [PriceStreamId:8][VenueLen:2][Venue:N][SymbolLen:2][Symbol:M][RecordSize:4][RecordBytes:R][CRC32:4] +/// The WAL is append-only and truncated after data files confirm durability. +/// +public sealed class WriteAheadLog : IAsyncDisposable +{ + private const int EntryCrcSize = 4; + + private readonly FileStream _stream; + private readonly string _filePath; + private bool _disposed; + + private WriteAheadLog(FileStream stream, string filePath) + { + _stream = stream; + _filePath = filePath; + } + + public string FilePath => _filePath; + public long Length => _stream.Length; + + public static WriteAheadLog Open(string directory, int partitionIndex) + { + Directory.CreateDirectory(directory); + var filePath = Path.Combine(directory, $"wal_{partitionIndex:D4}.log"); + var stream = new FileStream( + filePath, + FileMode.OpenOrCreate, + FileAccess.ReadWrite, + FileShare.Read, + bufferSize: 4096, + FileOptions.WriteThrough | FileOptions.Asynchronous); + return new WriteAheadLog(stream, filePath); + } + + /// + /// Appends a record to the WAL with venue and symbol metadata for crash recovery. + /// + public async ValueTask AppendAsync(long priceStreamId, string venue, string symbol, ReadOnlyMemory recordBytes) + { + var venueBytes = Encoding.UTF8.GetBytes(venue); + var symbolBytes = Encoding.UTF8.GetBytes(symbol); + + // [PriceStreamId:8][VenueLen:2][Venue:N][SymbolLen:2][Symbol:M][RecordSize:4][RecordBytes:R][CRC32:4] + var entrySize = 8 + 2 + venueBytes.Length + 2 + symbolBytes.Length + 4 + recordBytes.Length + EntryCrcSize; + var buffer = new byte[entrySize]; + var offset = 0; + + BinaryPrimitives.WriteInt64LittleEndian(buffer.AsSpan(offset), priceStreamId); offset += 8; + BinaryPrimitives.WriteUInt16LittleEndian(buffer.AsSpan(offset), (ushort)venueBytes.Length); offset += 2; + venueBytes.CopyTo(buffer.AsSpan(offset)); offset += venueBytes.Length; + BinaryPrimitives.WriteUInt16LittleEndian(buffer.AsSpan(offset), (ushort)symbolBytes.Length); offset += 2; + symbolBytes.CopyTo(buffer.AsSpan(offset)); offset += symbolBytes.Length; + BinaryPrimitives.WriteInt32LittleEndian(buffer.AsSpan(offset), recordBytes.Length); offset += 4; + recordBytes.Span.CopyTo(buffer.AsSpan(offset)); offset += recordBytes.Length; + + var crc = Crc32.HashToUInt32(buffer.AsSpan(0, offset)); + BinaryPrimitives.WriteUInt32LittleEndian(buffer.AsSpan(offset), crc); + + _stream.Seek(0, SeekOrigin.End); + await _stream.WriteAsync(buffer); + _stream.Flush(flushToDisk: true); + } + + /// + /// Truncates the WAL after data files have confirmed durability. + /// + public void Truncate() + { + _stream.SetLength(0); + _stream.Flush(flushToDisk: true); + } + + /// + /// Reads all valid WAL entries for replay on recovery. + /// + public static List Replay(string directory, int partitionIndex) + { + var filePath = Path.Combine(directory, $"wal_{partitionIndex:D4}.log"); + if (!File.Exists(filePath)) + return []; + + var entries = new List(); + using var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + + while (fs.Position < fs.Length) + { + var entry = TryReadEntry(fs); + if (entry is null) break; + entries.Add(entry.Value); + } + + return entries; + } + + private static WalEntry? TryReadEntry(FileStream fs) + { + // PriceStreamId(8) + var buf8 = new byte[8]; + if (fs.Read(buf8) < 8) return null; + var priceStreamId = BinaryPrimitives.ReadInt64LittleEndian(buf8); + + var lenBuf = new byte[2]; + + // VenueLen(2) + Venue(N) + if (fs.Read(lenBuf) < 2) return null; + var venueLen = BinaryPrimitives.ReadUInt16LittleEndian(lenBuf); + if (venueLen > 1024) return null; + + var venueBytes = new byte[venueLen]; + if (venueLen > 0 && fs.Read(venueBytes) < venueLen) return null; + + // SymbolLen(2) + Symbol(M) + if (fs.Read(lenBuf) < 2) return null; + var symbolLen = BinaryPrimitives.ReadUInt16LittleEndian(lenBuf); + if (symbolLen > 1024) return null; + + var symbolBytes = new byte[symbolLen]; + if (symbolLen > 0 && fs.Read(symbolBytes) < symbolLen) return null; + + // RecordSize(4) + RecordBytes(R) + var sizeBuf = new byte[4]; + if (fs.Read(sizeBuf) < 4) return null; + var recordSize = BinaryPrimitives.ReadInt32LittleEndian(sizeBuf); + if (recordSize <= 0 || recordSize > 64 * 1024) return null; + + var recordBytes = new byte[recordSize]; + if (fs.Read(recordBytes) < recordSize) return null; + + // CRC32(4) + var crcBuf = new byte[EntryCrcSize]; + if (fs.Read(crcBuf) < EntryCrcSize) return null; + var storedCrc = BinaryPrimitives.ReadUInt32LittleEndian(crcBuf); + + // Rebuild the entry buffer for CRC validation + var entryLen = 8 + 2 + venueLen + 2 + symbolLen + 4 + recordSize; + var crcInput = new byte[entryLen]; + var offset = 0; + BinaryPrimitives.WriteInt64LittleEndian(crcInput.AsSpan(offset), priceStreamId); offset += 8; + BinaryPrimitives.WriteUInt16LittleEndian(crcInput.AsSpan(offset), venueLen); offset += 2; + venueBytes.CopyTo(crcInput.AsSpan(offset)); offset += venueLen; + BinaryPrimitives.WriteUInt16LittleEndian(crcInput.AsSpan(offset), symbolLen); offset += 2; + symbolBytes.CopyTo(crcInput.AsSpan(offset)); offset += symbolLen; + BinaryPrimitives.WriteInt32LittleEndian(crcInput.AsSpan(offset), recordSize); offset += 4; + recordBytes.CopyTo(crcInput.AsSpan(offset)); + + var computedCrc = Crc32.HashToUInt32(crcInput); + if (storedCrc != computedCrc) + return null; + + var venue = Encoding.UTF8.GetString(venueBytes); + var symbol = Encoding.UTF8.GetString(symbolBytes); + return new WalEntry(priceStreamId, venue, symbol, recordBytes); + } + + public async ValueTask DisposeAsync() + { + if (!_disposed) + { + _disposed = true; + await _stream.DisposeAsync(); + } + } +} + +public readonly record struct WalEntry(long PriceStreamId, string Venue, string Symbol, byte[] RecordBytes); diff --git a/src/Levels.Core/IO/WriteTimestamp.cs b/src/Levels.Core/IO/WriteTimestamp.cs new file mode 100644 index 0000000..6541474 --- /dev/null +++ b/src/Levels.Core/IO/WriteTimestamp.cs @@ -0,0 +1,6 @@ +namespace Levels.Core.IO; + +public static class WriteTimestamp +{ + public static long Now() => DateTime.UtcNow.Ticks * 100; +} diff --git a/src/Levels.Core/ISchemaDescriptor.cs b/src/Levels.Core/ISchemaDescriptor.cs new file mode 100644 index 0000000..db9a0e9 --- /dev/null +++ b/src/Levels.Core/ISchemaDescriptor.cs @@ -0,0 +1,15 @@ +namespace Levels.Core; + +/// +/// Implemented by codegen-generated schema types. +/// Carries schema metadata (ID, record stride) as static abstract members, +/// allowing generic registration like AddLevels<TSchema>(). +/// +public interface ISchemaDescriptor +{ + /// FNV-1a hash of the .fbs schema file. + static abstract uint SchemaId { get; } + + /// Size in bytes of a single core record (the struct from the .fbs). + static abstract int RecordSize { get; } +} diff --git a/src/Levels.Core/ISchemaEvent.cs b/src/Levels.Core/ISchemaEvent.cs new file mode 100644 index 0000000..338290f --- /dev/null +++ b/src/Levels.Core/ISchemaEvent.cs @@ -0,0 +1,26 @@ +using Levels.Core.Format; + +namespace Levels.Core; + +/// +/// Implemented by codegen-generated schema types to provide both schema metadata +/// and instance data for the write pipeline. The 7 properties match the core +/// user-fillable fields. serializes any extension +/// fields (bytes 56+) using generated code. +/// +public interface ISchemaEvent +{ + string Venue { get; } + string Symbol { get; } + long ObservedTime { get; } + long Price { get; } + long Quantity { get; } + RecordType RecordType { get; } + RecordSide RecordSide { get; } + + /// + /// Writes schema extension fields (bytes 56+) to the destination span. + /// Called by the sink to serialize schema-specific fields. + /// + void WriteExtension(Span destination); +} diff --git a/src/Levels.Core/Levels.Core.csproj b/src/Levels.Core/Levels.Core.csproj new file mode 100644 index 0000000..d5c7db1 --- /dev/null +++ b/src/Levels.Core/Levels.Core.csproj @@ -0,0 +1,25 @@ + + + + net10.0 + enable + enable + true + preview + + + + + + + + + + + + + + + + + diff --git a/src/Levels.Core/LevelsOptions.cs b/src/Levels.Core/LevelsOptions.cs new file mode 100644 index 0000000..933d574 --- /dev/null +++ b/src/Levels.Core/LevelsOptions.cs @@ -0,0 +1,42 @@ +namespace Levels.Core; + +public sealed class LevelsOptions +{ + public required string DataPath { get; set; } + public int BackpressureLimit { get; set; } = 8192; + public long RolloverSize { get; set; } = 256 * 1024 * 1024; + public TimeSpan RolloverInterval { get; set; } = TimeSpan.FromHours(1); + public int PriceScale { get; set; } + public int QuantityScale { get; set; } + + public TimeSpan CompactionWindow { get; set; } = TimeSpan.FromHours(1); + public int SyntheticSnapIntervalDeltas { get; set; } = 1000; + public TimeSpan? SyntheticSnapIntervalTime { get; set; } + public TimeSpan RetentionWindow { get; set; } = TimeSpan.FromDays(7); + public TimeSpan WindowGracePeriod { get; set; } = TimeSpan.FromMinutes(5); + + public string ConfigVersion { get; set; } = "v1"; + public bool EnableCompaction { get; set; } = true; + public bool EnablePeriodPromotion { get; set; } = true; + public int FlushThresholdMs { get; set; } = 0; + public int FlushBufferSize { get; set; } = 0; + + /// + /// Schema ID written into file headers. + /// Set automatically by AddLevels<TSchema>() from ISchemaDescriptor.SchemaId, + /// or set manually to the generated SchemaInfo.GeneratedSchemaId from your codegen output. + /// Defaults to 0 (no schema). + /// + public uint SchemaId { get; set; } + + /// + /// Total size in bytes of one record (core + schema-defined fields). + /// Set automatically by AddLevels<TSchema>() from ISchemaDescriptor.RecordSize. + /// + public int RecordSize { get; set; } = Format.Constants.CoreRecordSize; + + // Telemetry options + public bool EnableMetrics { get; set; } + public string? OtlpEndpoint { get; set; } + public bool EnableTracing { get; set; } +} diff --git a/src/Levels.Core/Orderbook/L3OrderbookSide.cs b/src/Levels.Core/Orderbook/L3OrderbookSide.cs new file mode 100644 index 0000000..324a037 --- /dev/null +++ b/src/Levels.Core/Orderbook/L3OrderbookSide.cs @@ -0,0 +1,79 @@ +namespace Levels.Core.Orderbook; + +public sealed class L3OrderbookSide +{ + private readonly Dictionary _orderIndex = new(); + private readonly SortedDictionary> _priceLevels = new(); + + public IReadOnlyDictionary Orders => _orderIndex; + public SortedDictionary> PriceLevels => _priceLevels; + + public void Apply(string orderId, long price, long quantity) + { + // Remove old entry for this order if it exists + if (_orderIndex.TryGetValue(orderId, out var old)) + { + RemoveFromPriceLevels(old.Price, orderId); + _orderIndex.Remove(orderId); + } + + if (quantity == 0) + return; + + _orderIndex[orderId] = (price, quantity); + + if (!_priceLevels.TryGetValue(price, out var level)) + { + level = new SortedDictionary(StringComparer.Ordinal); + _priceLevels[price] = level; + } + + level[orderId] = quantity; + } + + public void Remove(string orderId) + { + if (_orderIndex.TryGetValue(orderId, out var old)) + { + RemoveFromPriceLevels(old.Price, orderId); + _orderIndex.Remove(orderId); + } + } + + public void Clear() + { + _orderIndex.Clear(); + _priceLevels.Clear(); + } + + public long AggregatedQuantityAt(long price) + { + if (!_priceLevels.TryGetValue(price, out var level)) + return 0; + long total = 0; + foreach (var qty in level.Values) + total += qty; + return total; + } + + public IEnumerable> ToL2Levels() + { + foreach (var (price, orders) in _priceLevels) + { + long totalQty = 0; + foreach (var qty in orders.Values) + totalQty += qty; + yield return new KeyValuePair(price, totalQty); + } + } + + private void RemoveFromPriceLevels(long price, string orderId) + { + if (_priceLevels.TryGetValue(price, out var level)) + { + level.Remove(orderId); + if (level.Count == 0) + _priceLevels.Remove(price); + } + } +} diff --git a/src/Levels.Core/Orderbook/OrderbookSide.cs b/src/Levels.Core/Orderbook/OrderbookSide.cs new file mode 100644 index 0000000..a19f4e3 --- /dev/null +++ b/src/Levels.Core/Orderbook/OrderbookSide.cs @@ -0,0 +1,33 @@ +using Levels.Core.Format; + +namespace Levels.Core.Orderbook; + +public sealed class OrderbookSide +{ + private readonly SortedDictionary _levels = new(); + + public IEnumerable> Levels => _levels; + + public OrderbookSide() : this(RecordSide.Unknown) { } + + public OrderbookSide(RecordSide side) + { + } + + public void Apply(long price, long quantity) + { + if (quantity == 0) + { + _levels.Remove(price); + } + else + { + _levels[price] = quantity; + } + } + + public void Clear() + { + _levels.Clear(); + } +} diff --git a/src/Levels.Core/PriceStreamId.cs b/src/Levels.Core/PriceStreamId.cs new file mode 100644 index 0000000..fffa0d6 --- /dev/null +++ b/src/Levels.Core/PriceStreamId.cs @@ -0,0 +1,26 @@ +using System.IO.Hashing; +using System.Text; + +namespace Levels.Core; + +public readonly record struct PriceStreamId(long Value) +{ + [Obsolete("Use FromVenueSymbol for correct venue+symbol hashing")] + public static PriceStreamId FromSymbol(string symbol) + { + var hash = (long)XxHash64.HashToUInt64(Encoding.UTF8.GetBytes(symbol)); + return new PriceStreamId(hash); + } + + public static PriceStreamId FromVenueSymbol(string venue, string symbol) + { + var venueByteCount = Encoding.UTF8.GetByteCount(venue); + var symbolByteCount = Encoding.UTF8.GetByteCount(symbol); + Span buf = stackalloc byte[venueByteCount + 1 + symbolByteCount]; + var written = Encoding.UTF8.GetBytes(venue, buf); + buf[written] = 0; // null separator prevents "ab"+"cd" == "a"+"bcd" + Encoding.UTF8.GetBytes(symbol, buf[(written + 1)..]); + var hash = (long)XxHash64.HashToUInt64(buf); + return new PriceStreamId(hash); + } +} diff --git a/src/Levels.Core/ResampledFileInfo.cs b/src/Levels.Core/ResampledFileInfo.cs new file mode 100644 index 0000000..e5d5f24 --- /dev/null +++ b/src/Levels.Core/ResampledFileInfo.cs @@ -0,0 +1,9 @@ +namespace Levels.Core; + +public readonly record struct ResampledFileInfo( + string FilePath, + string ConfigVersion, + long RecordCount, + long FirstObservedTime, + long LastObservedTime, + IReadOnlyList SourceStreams); diff --git a/src/Levels.Core/SealedFileInfo.cs b/src/Levels.Core/SealedFileInfo.cs new file mode 100644 index 0000000..7df789a --- /dev/null +++ b/src/Levels.Core/SealedFileInfo.cs @@ -0,0 +1,9 @@ +namespace Levels.Core; + +public readonly record struct SealedFileInfo( + string FilePath, + PriceStreamId PriceStreamId, + long RecordCount, + long DeltaCount, + long FirstObservedTime, + long LastObservedTime); diff --git a/src/Levels.DataFlow/DataFlowBackpressureException.cs b/src/Levels.DataFlow/DataFlowBackpressureException.cs new file mode 100644 index 0000000..5811db2 --- /dev/null +++ b/src/Levels.DataFlow/DataFlowBackpressureException.cs @@ -0,0 +1,10 @@ +namespace Levels.DataFlow; + +public sealed class DataFlowBackpressureException : Exception +{ + public DataFlowBackpressureException() + : base("DataFlow handler channel is full. Backpressure applied.") { } + + public DataFlowBackpressureException(string handlerName) + : base($"DataFlow handler '{handlerName}' channel is full. Backpressure applied.") { } +} diff --git a/src/Levels.DataFlow/DataFlowBus.cs b/src/Levels.DataFlow/DataFlowBus.cs new file mode 100644 index 0000000..cb06d9f --- /dev/null +++ b/src/Levels.DataFlow/DataFlowBus.cs @@ -0,0 +1,162 @@ +using System.Diagnostics; +using System.Threading.Channels; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Levels.Core; +using Levels.Core.Diagnostics; +using Levels.Core.Format; + +namespace Levels.DataFlow; + +public sealed class DataFlowBus : IHostedService, IAsyncDisposable +{ + private readonly IDataFlowHandler[] _handlers; + private readonly Channel[] _channels; + private readonly int _channelCapacity; + private readonly ILogger _logger; + private Task[]? _consumerTasks; + private CancellationTokenSource? _cts; + private long _droppedRecordCount; + + public long DroppedRecordCount => Interlocked.Read(ref _droppedRecordCount); + + public DataFlowBus(IEnumerable handlers, int channelCapacity = 4096, ILogger? logger = null) + { + _handlers = handlers.ToArray(); + _channelCapacity = channelCapacity; + _logger = logger ?? (ILogger)NullLogger.Instance; + _channels = new Channel[_handlers.Length]; + + for (var i = 0; i < _handlers.Length; i++) + { + _channels[i] = Channel.CreateBounded( + new BoundedChannelOptions(_channelCapacity) + { + SingleReader = true, + SingleWriter = false, + }); + } + } + + public void PublishRecordWritten(PriceStreamId stream, RawRecord record) + { + var message = new RecordWrittenMessage(stream, record); + for (var i = 0; i < _channels.Length; i++) + { + if (!_channels[i].Writer.TryWrite(message)) + { + var count = Interlocked.Increment(ref _droppedRecordCount); + Log.RecordWrittenDropped(_logger, _handlers[i].GetType().Name, count); + var tags = new TagList { { "handler", _handlers[i].GetType().Name } }; + StlthLevelsMetrics.DataFlowBackpressure.Add(1, tags); + } + } + } + + public async ValueTask PublishFileSealedAsync(SealedFileInfo file) + { + var message = new FileSealedMessage(file); + for (var i = 0; i < _channels.Length; i++) + { + while (!_channels[i].Writer.TryWrite(message)) + { + await _channels[i].Writer.WaitToWriteAsync(); + } + } + } + + public async ValueTask PublishAggCreatedAsync(AggFileInfo file) + { + var message = new AggCreatedMessage(file); + for (var i = 0; i < _channels.Length; i++) + { + while (!_channels[i].Writer.TryWrite(message)) + { + await _channels[i].Writer.WaitToWriteAsync(); + } + } + } + + // Keep sync versions for backward compatibility (delegate to async) + public void PublishFileSealed(SealedFileInfo file) + { + var message = new FileSealedMessage(file); + for (var i = 0; i < _channels.Length; i++) + { + if (!_channels[i].Writer.TryWrite(message)) + throw new DataFlowBackpressureException(_handlers[i].GetType().Name); + } + } + + public void PublishAggCreated(AggFileInfo file) + { + var message = new AggCreatedMessage(file); + for (var i = 0; i < _channels.Length; i++) + { + if (!_channels[i].Writer.TryWrite(message)) + throw new DataFlowBackpressureException(_handlers[i].GetType().Name); + } + } + + public Task StartAsync(CancellationToken cancellationToken) + { + _cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + _consumerTasks = new Task[_handlers.Length]; + + for (var i = 0; i < _handlers.Length; i++) + { + var index = i; + _consumerTasks[i] = Task.Run(() => ConsumeAsync(index, _cts.Token), _cts.Token); + } + + return Task.CompletedTask; + } + + private async Task ConsumeAsync(int index, CancellationToken ct) + { + var handler = _handlers[index]; + var reader = _channels[index].Reader; + + await foreach (var message in reader.ReadAllAsync(ct)) + { + switch (message) + { + case RecordWrittenMessage rw: + await handler.OnRecordWritten(rw.Stream, rw.Record, ct); + break; + case FileSealedMessage fs: + await handler.OnFileSealed(fs.File, ct); + break; + case AggCreatedMessage ac: + await handler.OnAggCreated(ac.File, ct); + break; + } + } + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + // Complete all channels + for (var i = 0; i < _channels.Length; i++) + _channels[i].Writer.TryComplete(); + + // Wait for consumers to drain + if (_consumerTasks is not null) + { + try + { + await Task.WhenAll(_consumerTasks); + } + catch (OperationCanceledException) { } + } + + _cts?.Dispose(); + _cts = null; + } + + public async ValueTask DisposeAsync() + { + await StopAsync(CancellationToken.None); + } +} diff --git a/src/Levels.DataFlow/DataFlowMessage.cs b/src/Levels.DataFlow/DataFlowMessage.cs new file mode 100644 index 0000000..a2345ca --- /dev/null +++ b/src/Levels.DataFlow/DataFlowMessage.cs @@ -0,0 +1,9 @@ +using Levels.Core; +using Levels.Core.Format; + +namespace Levels.DataFlow; + +public abstract record DataFlowMessage; +public sealed record RecordWrittenMessage(PriceStreamId Stream, RawRecord Record) : DataFlowMessage; +public sealed record FileSealedMessage(SealedFileInfo File) : DataFlowMessage; +public sealed record AggCreatedMessage(AggFileInfo File) : DataFlowMessage; diff --git a/src/Levels.DataFlow/IDataFlowHandler.cs b/src/Levels.DataFlow/IDataFlowHandler.cs new file mode 100644 index 0000000..c424393 --- /dev/null +++ b/src/Levels.DataFlow/IDataFlowHandler.cs @@ -0,0 +1,11 @@ +using Levels.Core; +using Levels.Core.Format; + +namespace Levels.DataFlow; + +public interface IDataFlowHandler +{ + ValueTask OnRecordWritten(PriceStreamId stream, RawRecord record, CancellationToken ct); + ValueTask OnFileSealed(SealedFileInfo file, CancellationToken ct); + ValueTask OnAggCreated(AggFileInfo file, CancellationToken ct); +} diff --git a/src/Levels.DataFlow/Levels.DataFlow.csproj b/src/Levels.DataFlow/Levels.DataFlow.csproj new file mode 100644 index 0000000..2b0ba98 --- /dev/null +++ b/src/Levels.DataFlow/Levels.DataFlow.csproj @@ -0,0 +1,23 @@ + + + + net10.0 + enable + enable + preview + + + + + + + + + + + + + + + + diff --git a/src/Levels.DataFlow/Log.cs b/src/Levels.DataFlow/Log.cs new file mode 100644 index 0000000..026e88d --- /dev/null +++ b/src/Levels.DataFlow/Log.cs @@ -0,0 +1,15 @@ +using Microsoft.Extensions.Logging; + +namespace Levels.DataFlow; + +internal static partial class Log +{ + [LoggerMessage(Level = LogLevel.Information, Message = "Handler registered: HandlerName={HandlerName}")] + public static partial void HandlerRegistered(ILogger logger, string handlerName); + + [LoggerMessage(Level = LogLevel.Error, Message = "Backpressure detected: HandlerName={HandlerName}")] + public static partial void BackpressureDetected(ILogger logger, string handlerName); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Dropped RecordWrittenMessage for handler {HandlerName} (total dropped: {DroppedCount})")] + public static partial void RecordWrittenDropped(ILogger logger, string handlerName, long droppedCount); +} diff --git a/src/Levels.Export/AvroExportAdapter.cs b/src/Levels.Export/AvroExportAdapter.cs new file mode 100644 index 0000000..9242400 --- /dev/null +++ b/src/Levels.Export/AvroExportAdapter.cs @@ -0,0 +1,57 @@ +using Avro; +using Avro.File; +using Avro.Generic; +using Avro.IO; +using AvroSchema = Avro.Schema; + +namespace Levels.Export; + +public sealed class AvroExportAdapter : IExportAdapter +{ + public string Format => "avro"; + + private static readonly RecordSchema AvroRecordSchema = (RecordSchema)AvroSchema.Parse(@"{ + ""type"": ""record"", + ""name"": ""ExportRecord"", + ""namespace"": ""Levels.Export"", + ""fields"": [ + {""name"": ""ObservedTime"", ""type"": ""long""}, + {""name"": ""WriteTimestamp"", ""type"": ""long""}, + {""name"": ""PriceStreamId"", ""type"": ""long""}, + {""name"": ""Price"", ""type"": ""long""}, + {""name"": ""Quantity"", ""type"": ""long""}, + {""name"": ""Type"", ""type"": ""int""}, + {""name"": ""Side"", ""type"": ""int""}, + {""name"": ""Sequence"", ""type"": ""long""}, + {""name"": ""Level"", ""type"": ""int""}, + {""name"": ""Flags"", ""type"": ""int""}, + {""name"": ""OrderId"", ""type"": [""null"", ""string""], ""default"": null} + ] + }"); + + public async Task ExportAsync(IAsyncEnumerable records, Stream output, CancellationToken ct = default) + { + var datumWriter = new GenericDatumWriter(AvroRecordSchema); + using var fileWriter = DataFileWriter.OpenWriter(datumWriter, output, Codec.CreateCodec(Codec.Type.Null)); + + await foreach (var record in records.WithCancellation(ct)) + { + var avroRecord = new GenericRecord(AvroRecordSchema); + avroRecord.Add("ObservedTime", record.ObservedTime); + avroRecord.Add("WriteTimestamp", record.WriteTimestamp); + avroRecord.Add("PriceStreamId", record.PriceStreamId); + avroRecord.Add("Price", record.Price); + avroRecord.Add("Quantity", record.Quantity); + avroRecord.Add("Type", (int)record.Type); + avroRecord.Add("Side", (int)record.Side); + avroRecord.Add("Sequence", (long)record.Sequence); + avroRecord.Add("Level", (int)record.Level); + avroRecord.Add("Flags", (int)record.Flags); + avroRecord.Add("OrderId", record.OrderId); + + fileWriter.Append(avroRecord); + } + + fileWriter.Flush(); + } +} diff --git a/src/Levels.Export/CsvExportAdapter.cs b/src/Levels.Export/CsvExportAdapter.cs new file mode 100644 index 0000000..65386f5 --- /dev/null +++ b/src/Levels.Export/CsvExportAdapter.cs @@ -0,0 +1,21 @@ +using System.Globalization; +using System.Text; + +namespace Levels.Export; + +public sealed class CsvExportAdapter : IExportAdapter +{ + public string Format => "csv"; + + public async Task ExportAsync(IAsyncEnumerable records, Stream output, CancellationToken ct = default) + { + await using var writer = new StreamWriter(output, Encoding.UTF8, leaveOpen: true); + await writer.WriteLineAsync("ObservedTime,WriteTimestamp,PriceStreamId,Price,Quantity,Type,Side,Sequence,Level,Flags,OrderId"); + + await foreach (var record in records.WithCancellation(ct)) + { + await writer.WriteLineAsync(string.Create(CultureInfo.InvariantCulture, + $"{record.ObservedTime},{record.WriteTimestamp},{record.PriceStreamId},{record.Price},{record.Quantity},{record.Type},{record.Side},{record.Sequence},{record.Level},{record.Flags},{record.OrderId ?? ""}")); + } + } +} diff --git a/src/Levels.Export/ExportPipeline.cs b/src/Levels.Export/ExportPipeline.cs new file mode 100644 index 0000000..f8252ed --- /dev/null +++ b/src/Levels.Export/ExportPipeline.cs @@ -0,0 +1,88 @@ +using System.Runtime.CompilerServices; +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Query; + +namespace Levels.Export; + +public sealed class ExportPipeline +{ + private readonly QueryLayer _queryLayer; + private readonly Dictionary _adapters = new(StringComparer.OrdinalIgnoreCase); + + public ExportPipeline(QueryLayer queryLayer, IEnumerable adapters) + { + _queryLayer = queryLayer; + foreach (var adapter in adapters) + _adapters[adapter.Format] = adapter; + } + + public IReadOnlyCollection SupportedFormats => _adapters.Keys; + + public async Task ExportAsync( + PriceStreamId streamId, + long fromNanos, + long toNanos, + string format, + Stream output, + CancellationToken ct = default) + { + if (!_adapters.TryGetValue(format, out var adapter)) + throw new ArgumentException($"Unsupported export format: {format}. Supported: {string.Join(", ", _adapters.Keys)}"); + + var records = StreamRecords(streamId, fromNanos, toNanos, ct); + await adapter.ExportAsync(records, output, ct); + } + + private async IAsyncEnumerable StreamRecords( + PriceStreamId streamId, + long fromNanos, + long toNanos, + [EnumeratorCancellation] CancellationToken ct) + { + var entries = _queryLayer.Resolve(streamId, fromNanos, toNanos); + + foreach (var entry in entries) + { + ct.ThrowIfCancellationRequested(); + + using var fs = File.OpenRead(entry.FilePath); + var reader = new BinaryRecordReader(fs); + + foreach (var record in reader.ReadRecords()) + { + ct.ThrowIfCancellationRequested(); + + var core = record.Core; + if (core.ObservedTime > toNanos) yield break; + if (core.ObservedTime < fromNanos && core.Type != RecordType.Snap) continue; + + string? orderId = null; + var orderIdMem = record.OrderId; + if (orderIdMem.Length > 0) + { + var span = orderIdMem.Span; + // Trim trailing null bytes for string OrderIds + int len = span.Length; + while (len > 0 && span[len - 1] == 0) len--; + if (len > 0) + orderId = System.Text.Encoding.UTF8.GetString(span[..len]); + } + + yield return new ExportRecord( + core.ObservedTime, + core.WriteTimestamp, + core.PriceStreamId, + core.Price, + core.Quantity, + core.Type, + core.Side, + core.Sequence, + core.Level, + core.Flags, + orderId); + } + } + } +} diff --git a/src/Levels.Export/ExportRecord.cs b/src/Levels.Export/ExportRecord.cs new file mode 100644 index 0000000..2c223a2 --- /dev/null +++ b/src/Levels.Export/ExportRecord.cs @@ -0,0 +1,16 @@ +using Levels.Core.Format; + +namespace Levels.Export; + +public readonly record struct ExportRecord( + long ObservedTime, + long WriteTimestamp, + long PriceStreamId, + long Price, + long Quantity, + RecordType Type, + RecordSide Side, + uint Sequence, + ushort Level, + ushort Flags, + string? OrderId); diff --git a/src/Levels.Export/IExportAdapter.cs b/src/Levels.Export/IExportAdapter.cs new file mode 100644 index 0000000..28d60ce --- /dev/null +++ b/src/Levels.Export/IExportAdapter.cs @@ -0,0 +1,7 @@ +namespace Levels.Export; + +public interface IExportAdapter +{ + string Format { get; } + Task ExportAsync(IAsyncEnumerable records, Stream output, CancellationToken ct = default); +} diff --git a/src/Levels.Export/Levels.Export.csproj b/src/Levels.Export/Levels.Export.csproj new file mode 100644 index 0000000..f7fc80f --- /dev/null +++ b/src/Levels.Export/Levels.Export.csproj @@ -0,0 +1,24 @@ + + + + net10.0 + enable + enable + preview + + + + + + + + + + + + + + + + + diff --git a/src/Levels.Export/ParquetExportAdapter.cs b/src/Levels.Export/ParquetExportAdapter.cs new file mode 100644 index 0000000..69ef7f9 --- /dev/null +++ b/src/Levels.Export/ParquetExportAdapter.cs @@ -0,0 +1,94 @@ +using Parquet; +using Parquet.Data; +using Parquet.Schema; + +namespace Levels.Export; + +public sealed class ParquetExportAdapter : IExportAdapter +{ + private const int BatchSize = 10_000; + + public string Format => "parquet"; + + public async Task ExportAsync(IAsyncEnumerable records, Stream output, CancellationToken ct = default) + { + var schema = new ParquetSchema( + new DataField("ObservedTime"), + new DataField("WriteTimestamp"), + new DataField("PriceStreamId"), + new DataField("Price"), + new DataField("Quantity"), + new DataField("Type"), + new DataField("Side"), + new DataField("Sequence"), + new DataField("Level"), + new DataField("Flags"), + new DataField("OrderId")); + + using var parquetWriter = await ParquetWriter.CreateAsync(schema, output); + + var batch = new List(BatchSize); + + await foreach (var record in records.WithCancellation(ct)) + { + batch.Add(record); + + if (batch.Count >= BatchSize) + { + await WriteRowGroupAsync(parquetWriter, schema, batch); + batch.Clear(); + } + } + + if (batch.Count > 0) + { + await WriteRowGroupAsync(parquetWriter, schema, batch); + } + } + + private static async Task WriteRowGroupAsync(ParquetWriter writer, ParquetSchema schema, List batch) + { + var count = batch.Count; + + var observedTimes = new long[count]; + var writeTimestamps = new long[count]; + var priceStreamIds = new long[count]; + var prices = new long[count]; + var quantities = new long[count]; + var types = new byte[count]; + var sides = new byte[count]; + var sequences = new int[count]; + var levels = new short[count]; + var flags = new short[count]; + var orderIds = new string[count]; + + for (int i = 0; i < count; i++) + { + var r = batch[i]; + observedTimes[i] = r.ObservedTime; + writeTimestamps[i] = r.WriteTimestamp; + priceStreamIds[i] = r.PriceStreamId; + prices[i] = r.Price; + quantities[i] = r.Quantity; + types[i] = (byte)r.Type; + sides[i] = (byte)r.Side; + sequences[i] = (int)r.Sequence; + levels[i] = (short)r.Level; + flags[i] = (short)r.Flags; + orderIds[i] = r.OrderId ?? ""; + } + + using var rowGroup = writer.CreateRowGroup(); + await rowGroup.WriteColumnAsync(new DataColumn(schema.DataFields[0], observedTimes)); + await rowGroup.WriteColumnAsync(new DataColumn(schema.DataFields[1], writeTimestamps)); + await rowGroup.WriteColumnAsync(new DataColumn(schema.DataFields[2], priceStreamIds)); + await rowGroup.WriteColumnAsync(new DataColumn(schema.DataFields[3], prices)); + await rowGroup.WriteColumnAsync(new DataColumn(schema.DataFields[4], quantities)); + await rowGroup.WriteColumnAsync(new DataColumn(schema.DataFields[5], types)); + await rowGroup.WriteColumnAsync(new DataColumn(schema.DataFields[6], sides)); + await rowGroup.WriteColumnAsync(new DataColumn(schema.DataFields[7], sequences)); + await rowGroup.WriteColumnAsync(new DataColumn(schema.DataFields[8], levels)); + await rowGroup.WriteColumnAsync(new DataColumn(schema.DataFields[9], flags)); + await rowGroup.WriteColumnAsync(new DataColumn(schema.DataFields[10], orderIds)); + } +} diff --git a/src/Levels.Hints/HintEntry.cs b/src/Levels.Hints/HintEntry.cs new file mode 100644 index 0000000..02d9e1c --- /dev/null +++ b/src/Levels.Hints/HintEntry.cs @@ -0,0 +1,13 @@ +namespace Levels.Hints; + +internal sealed class HintEntry +{ + public string Id { get; set; } = ""; + public string FilePath { get; set; } = ""; + public long PriceStreamId { get; set; } + public string Venue { get; set; } = ""; + public int FileType { get; set; } + public long FirstObservedTime { get; set; } + public long LastObservedTime { get; set; } + public long RecordCount { get; set; } +} diff --git a/src/Levels.Hints/HintsDb.cs b/src/Levels.Hints/HintsDb.cs new file mode 100644 index 0000000..0f64d23 --- /dev/null +++ b/src/Levels.Hints/HintsDb.cs @@ -0,0 +1,110 @@ +using LiteDB; +using Levels.Core; +using Levels.Core.Format; +using Levels.Query; + +namespace Levels.Hints; + +public sealed class HintsDb : IDisposable +{ + private readonly LiteDatabase _db; + private readonly ILiteCollection _entries; + + public HintsDb(string dbPath) + { + _db = new LiteDatabase(dbPath); + _entries = _db.GetCollection("hints"); + _entries.EnsureIndex(x => x.PriceStreamId); + _entries.EnsureIndex(x => x.FilePath, unique: true); + } + + public void Upsert(FileIndexEntry entry) + { + var hint = new HintEntry + { + Id = entry.FilePath, + FilePath = entry.FilePath, + PriceStreamId = entry.PriceStreamId.Value, + Venue = entry.Venue, + FileType = (int)entry.FileType, + FirstObservedTime = entry.FirstObservedTime, + LastObservedTime = entry.LastObservedTime, + RecordCount = entry.RecordCount, + }; + + _entries.Upsert(hint); + } + + public void Remove(string filePath) + { + _entries.DeleteMany(x => x.FilePath == filePath); + } + + public IReadOnlyList GetAll() + { + return _entries.FindAll() + .Select(h => new FileIndexEntry( + h.FilePath, + new PriceStreamId(h.PriceStreamId), + h.Venue, + (FileType)h.FileType, + h.FirstObservedTime, + h.LastObservedTime, + h.RecordCount)) + .ToList(); + } + + public void LoadInto(FileIndex fileIndex) + { + foreach (var entry in GetAll()) + { + if (File.Exists(entry.FilePath)) + fileIndex.Register(entry); + } + } + + public int ConsistencyCheck(string dataPath, FileIndex fileIndex) + { + var staleCount = 0; + var allHints = GetAll(); + + foreach (var entry in allHints) + { + if (!File.Exists(entry.FilePath)) + { + Remove(entry.FilePath); + staleCount++; + } + } + + // Fallback: scan disk for files not in hints + if (Directory.Exists(dataPath)) + { + var extensions = new[] { "*.raw", "*.agg", "*.period", "*.resampled" }; + var knownFiles = new HashSet(allHints.Select(e => e.FilePath)); + + foreach (var ext in extensions) + { + foreach (var file in Directory.GetFiles(dataPath, ext, SearchOption.AllDirectories)) + { + if (!knownFiles.Contains(file)) + { + var diskEntry = FileIndex.TryReadEntryStatic(dataPath, file); + if (diskEntry is not null) + { + Upsert(diskEntry.Value); + fileIndex.Register(diskEntry.Value); + } + } + } + } + } + + return staleCount; + } + + public void Dispose() + { + _db.Dispose(); + } +} diff --git a/src/Levels.Hints/HintsDbSyncHandler.cs b/src/Levels.Hints/HintsDbSyncHandler.cs new file mode 100644 index 0000000..56dee43 --- /dev/null +++ b/src/Levels.Hints/HintsDbSyncHandler.cs @@ -0,0 +1,58 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.DataFlow; +using Levels.Query; + +namespace Levels.Hints; + +public sealed class HintsDbSyncHandler : IDataFlowHandler +{ + private readonly HintsDb _hintsDb; + private readonly string _dataPath; + + public HintsDbSyncHandler(HintsDb hintsDb, string dataPath) + { + _hintsDb = hintsDb; + _dataPath = dataPath; + } + + public ValueTask OnRecordWritten(PriceStreamId stream, RawRecord record, CancellationToken ct) + => ValueTask.CompletedTask; + + public ValueTask OnFileSealed(SealedFileInfo file, CancellationToken ct) + { + var entry = new FileIndexEntry( + file.FilePath, + file.PriceStreamId, + ExtractVenue(file.FilePath), + FileType.Raw, + file.FirstObservedTime, + file.LastObservedTime, + file.RecordCount); + + _hintsDb.Upsert(entry); + return ValueTask.CompletedTask; + } + + public ValueTask OnAggCreated(AggFileInfo file, CancellationToken ct) + { + var entry = new FileIndexEntry( + file.FilePath, + file.PriceStreamId, + ExtractVenue(file.FilePath), + FileType.Agg, + file.FirstObservedTime, + file.LastObservedTime, + file.RecordCount); + + _hintsDb.Upsert(entry); + return ValueTask.CompletedTask; + } + + private string ExtractVenue(string filePath) + { + var relativePath = Path.GetRelativePath(_dataPath, filePath); + var parts = relativePath.Split(Path.DirectorySeparatorChar); + return parts.Length >= 3 ? parts[0] : "unknown"; + } +} diff --git a/src/Levels.Hints/Levels.Hints.csproj b/src/Levels.Hints/Levels.Hints.csproj new file mode 100644 index 0000000..e120d44 --- /dev/null +++ b/src/Levels.Hints/Levels.Hints.csproj @@ -0,0 +1,24 @@ + + + + net10.0 + enable + enable + preview + + + + + + + + + + + + + + + + + diff --git a/src/Levels.Hosting/DataSinkAdapter.cs b/src/Levels.Hosting/DataSinkAdapter.cs new file mode 100644 index 0000000..624cd55 --- /dev/null +++ b/src/Levels.Hosting/DataSinkAdapter.cs @@ -0,0 +1,64 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Sinks; + +namespace Levels.Hosting; + +public sealed class DataSinkAdapter : IDataSink +{ + private readonly PriceStreamSink _sink; + private readonly Func _map; + + public DataSinkAdapter(PriceStreamSink sink) + { + _sink = sink; + + if (typeof(ISchemaEvent).IsAssignableFrom(typeof(T))) + { + int extSize = GetRecordSize() - Constants.CoreRecordSize; + + _map = record => + { + var evt = (ISchemaEvent)(object)record!; + ReadOnlyMemory extensionBytes = default; + if (extSize > 0) + { + var buffer = new byte[extSize]; + evt.WriteExtension(buffer); + extensionBytes = buffer; + } + + return new RawMarketEvent( + evt.Venue, + evt.Symbol, + evt.ObservedTime, + evt.Price, + evt.Quantity, + evt.RecordType, + evt.RecordSide, + ExtensionBytes: extensionBytes); + }; + } + else + { + var mapper = new FieldMapper(); + _map = mapper.Map; + } + } + + public async ValueTask WriteAsync(T record, CancellationToken ct = default) + { + var evt = _map(record); + await _sink.WriteAsync(evt, ct); + } + + private static int GetRecordSize() + { + var prop = typeof(T).GetProperty("RecordSize", + System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.FlattenHierarchy); + if (prop is not null) + return (int)prop.GetValue(null)!; + + return Constants.CoreRecordSize; + } +} diff --git a/src/Levels.Hosting/FieldMapper.cs b/src/Levels.Hosting/FieldMapper.cs new file mode 100644 index 0000000..42a3ad0 --- /dev/null +++ b/src/Levels.Hosting/FieldMapper.cs @@ -0,0 +1,90 @@ +using System.Linq.Expressions; +using System.Reflection; +using Levels.Core.Format; +using Levels.Sinks; + +namespace Levels.Hosting; + +internal sealed class FieldMapper +{ + private readonly Func _getVenue; + private readonly Func _getSymbol; + private readonly Func _getObservedTime; + private readonly Func _getPrice; + private readonly Func _getQuantity; + private readonly Func _getType; + private readonly Func _getSide; + private readonly Func? _getOrderId; + private readonly Func? _getIsOwner; + private readonly Func>? _getExtensionBytes; + + public FieldMapper() + { + var type = typeof(T); + + _getVenue = BuildGetter(type, "Venue"); + _getSymbol = BuildGetter(type, "Symbol"); + _getObservedTime = BuildGetter(type, "ObservedTime"); + _getPrice = BuildGetter(type, "Price"); + _getQuantity = BuildGetter(type, "Quantity"); + _getType = BuildGetter(type, "Type"); + _getSide = BuildGetter(type, "Side"); + + _getOrderId = TryBuildGetter(type, "OrderId"); + _getIsOwner = TryBuildGetter(type, "IsOwner"); + _getExtensionBytes = TryBuildGetter>(type, "ExtensionBytes"); + } + + public RawMarketEvent Map(T record) + { + ReadOnlyMemory extensionBytes = default; + + // If the type has ExtensionBytes, use it directly + if (_getExtensionBytes is not null) + { + extensionBytes = _getExtensionBytes(record); + } + // Backward compat: if the type has OrderId, UTF-8 encode it as extension bytes + else if (_getOrderId is not null) + { + var orderId = _getOrderId(record); + if (orderId is not null) + extensionBytes = System.Text.Encoding.UTF8.GetBytes(orderId); + } + + return new RawMarketEvent( + _getVenue(record), + _getSymbol(record), + _getObservedTime(record), + _getPrice(record), + _getQuantity(record), + _getType(record), + _getSide(record), + ExtensionBytes: extensionBytes, + IsOwner: _getIsOwner?.Invoke(record) ?? false); + } + + private static Func BuildGetter(Type type, string propertyName) + { + var prop = type.GetProperty(propertyName, BindingFlags.Public | BindingFlags.Instance) + ?? throw new InvalidOperationException( + $"Type {type.Name} must have a public '{propertyName}' property for convention-based mapping."); + + var param = Expression.Parameter(typeof(T), "x"); + var access = Expression.Property(param, prop); + var convert = Expression.Convert(access, typeof(TField)); + return Expression.Lambda>(convert, param).Compile(); + } + + private static Func? TryBuildGetter(Type type, string propertyName) + { + var prop = type.GetProperty(propertyName, BindingFlags.Public | BindingFlags.Instance); + if (prop is null) + return null; + + var param = Expression.Parameter(typeof(T), "x"); + var access = Expression.Property(param, prop); + var convert = Expression.Convert(access, typeof(TField)); + return Expression.Lambda>(convert, param).Compile(); + } +} diff --git a/src/Levels.Hosting/Levels.Hosting.csproj b/src/Levels.Hosting/Levels.Hosting.csproj new file mode 100644 index 0000000..f1fea39 --- /dev/null +++ b/src/Levels.Hosting/Levels.Hosting.csproj @@ -0,0 +1,30 @@ + + + + net10.0 + enable + enable + preview + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Levels.Hosting/OpenTelemetryExtensions.cs b/src/Levels.Hosting/OpenTelemetryExtensions.cs new file mode 100644 index 0000000..4c9901d --- /dev/null +++ b/src/Levels.Hosting/OpenTelemetryExtensions.cs @@ -0,0 +1,47 @@ +using Microsoft.Extensions.DependencyInjection; +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; +using Levels.Core; + +namespace Levels.Hosting; + +public static class OpenTelemetryExtensions +{ + public static IServiceCollection AddLevelsOpenTelemetry( + this IServiceCollection services, + LevelsOptions options) + { + if (!options.EnableMetrics && !options.EnableTracing) + return services; + + var otel = services.AddOpenTelemetry(); + + if (options.EnableMetrics) + { + otel.WithMetrics(metrics => + { + metrics.AddMeter("Levels"); + + if (!string.IsNullOrEmpty(options.OtlpEndpoint)) + { + metrics.AddOtlpExporter(o => o.Endpoint = new Uri(options.OtlpEndpoint)); + } + }); + } + + if (options.EnableTracing) + { + otel.WithTracing(traces => + { + traces.AddSource("Levels"); + + if (!string.IsNullOrEmpty(options.OtlpEndpoint)) + { + traces.AddOtlpExporter(o => o.Endpoint = new Uri(options.OtlpEndpoint)); + } + }); + } + + return services; + } +} diff --git a/src/Levels.Hosting/ServiceCollectionExtensions.cs b/src/Levels.Hosting/ServiceCollectionExtensions.cs new file mode 100644 index 0000000..436ff6a --- /dev/null +++ b/src/Levels.Hosting/ServiceCollectionExtensions.cs @@ -0,0 +1,145 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Levels.Compaction; +using Levels.Core; +using Levels.DataFlow; +using Levels.Period; +using Levels.Query; +using Levels.Sinks; + +namespace Levels.Hosting; + +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddLevels( + this IServiceCollection services, + Action configure) + { + var options = new LevelsOptions { DataPath = "" }; + configure(options); + + if (string.IsNullOrEmpty(options.DataPath)) + throw new ArgumentException("DataPath must be set in LevelsOptions."); + + services.AddSingleton(options); + + // FileIndex + QueryLayer + var fileIndex = new FileIndex(); + fileIndex.LoadFromDisk(options.DataPath); + services.AddSingleton(fileIndex); + + var queryConfig = new QueryConfig { DataPath = options.DataPath, CompactionWindow = options.CompactionWindow }; + services.AddSingleton(queryConfig); + services.AddSingleton(sp => new QueryLayer(sp.GetRequiredService(), sp.GetRequiredService())); + + // DataFlow handlers + var handlers = new List(); + + // FileIndex sync + var syncHandler = new FileIndexSyncHandler(fileIndex, options.DataPath); + handlers.Add(syncHandler); + + // Compaction + if (options.EnableCompaction) + { + var compactionConfig = new CompactionConfig + { + DataPath = options.DataPath, + CompactionWindow = options.CompactionWindow, + SyntheticSnapIntervalDeltas = options.SyntheticSnapIntervalDeltas, + SyntheticSnapIntervalTime = options.SyntheticSnapIntervalTime, + RetentionWindow = options.RetentionWindow, + WindowGracePeriod = options.WindowGracePeriod, + }; + services.AddSingleton(compactionConfig); + + // DataFlowBus needs to be created after all handlers are registered + // Store compaction config for later + } + + // Period promotion + if (options.EnablePeriodPromotion) + { + var periodConfig = new PeriodConfig { DataPath = options.DataPath, ConfigVersion = options.ConfigVersion }; + services.AddSingleton(periodConfig); + } + + // Register DataFlowBus + services.AddSingleton(sp => + { + var allHandlers = new List { syncHandler }; + + if (options.EnableCompaction) + { + var bus = new DataFlowBus([]); // temporary, will be set properly + var compactionConfig = sp.GetRequiredService(); + var compaction = new EventSourcingCompaction(compactionConfig, bus); + allHandlers.Add(compaction); + } + + return new DataFlowBus(allHandlers); + }); + + // SinkConfig + PriceStreamSink + services.AddSingleton(sp => + { + var bus = sp.GetRequiredService(); + return new SinkConfig + { + OutputPath = options.DataPath, + BackpressureLimit = options.BackpressureLimit, + RolloverSize = options.RolloverSize, + RolloverInterval = options.RolloverInterval, + PriceScale = options.PriceScale, + QuantityScale = options.QuantityScale, + SchemaId = options.SchemaId, + RecordSize = options.RecordSize, + FlushThresholdMs = options.FlushThresholdMs, + FlushBufferSize = options.FlushBufferSize, + DataFlowBus = bus, + }; + }); + + services.AddSingleton(sp => new PriceStreamSink( + sp.GetRequiredService(), + sp.GetService>())); + + // Register as hosted services + services.AddHostedService(sp => sp.GetRequiredService()); + services.AddHostedService(sp => sp.GetRequiredService()); + + // OpenTelemetry (opt-in via EnableMetrics/EnableTracing) + services.AddLevelsOpenTelemetry(options); + + return services; + } + + /// + /// Registers Levels services with schema metadata inferred from . + /// The schema ID is set automatically from the generated descriptor type. + /// + public static IServiceCollection AddLevels( + this IServiceCollection services, + Action configure) + where TSchema : ISchemaDescriptor + { + return services.AddLevels(opts => + { + configure(opts); + opts.SchemaId = TSchema.SchemaId; + opts.RecordSize = TSchema.RecordSize; + }); + } + + public static IServiceCollection AddLevelsDataSink(this IServiceCollection services) + { + services.TryAddSingleton>(sp => + { + var sink = sp.GetRequiredService(); + return new DataSinkAdapter(sink); + }); + + return services; + } +} diff --git a/src/Levels.Period/CrossedBookDetector.cs b/src/Levels.Period/CrossedBookDetector.cs new file mode 100644 index 0000000..e08b730 --- /dev/null +++ b/src/Levels.Period/CrossedBookDetector.cs @@ -0,0 +1,53 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.Orderbook; + +namespace Levels.Period; + +public sealed class CrossedBookDetector : IHealthCheck +{ + public string Name => "CrossedBook"; + + public HealthCheckResult Check(IReadOnlyList records, PriceStreamId streamId) + { + var bids = new OrderbookSide(); + var asks = new OrderbookSide(); + + for (var i = 0; i < records.Count; i++) + { + var core = records[i].Core; + + if (core.Type is RecordType.Snap or RecordType.Delta) + { + var side = core.Side == RecordSide.Bid ? bids : asks; + side.Apply(core.Price, core.Quantity); + } + + // Check for crossed book after each record + var bidLevels = bids.Levels.GetEnumerator(); + var askLevels = asks.Levels.GetEnumerator(); + + if (!bidLevels.MoveNext() || !askLevels.MoveNext()) + continue; // One or both sides empty — no cross possible + + // SortedDictionary is ascending: last bid = highest, first ask = lowest + // We need to iterate to find the highest bid + long bestBid = 0; + foreach (var kvp in bids.Levels) + bestBid = kvp.Key; // Will end up with the last (highest) key + + long bestAsk = long.MaxValue; + foreach (var kvp in asks.Levels) + { + bestAsk = kvp.Key; // First key is lowest + break; + } + + if (bestBid >= bestAsk) + return new HealthCheckResult(false, + $"Crossed book at record {i}: bestBid={bestBid} >= bestAsk={bestAsk}."); + } + + return new HealthCheckResult(true); + } +} diff --git a/src/Levels.Period/DemotionService.cs b/src/Levels.Period/DemotionService.cs new file mode 100644 index 0000000..ef8e1e9 --- /dev/null +++ b/src/Levels.Period/DemotionService.cs @@ -0,0 +1,21 @@ +using Levels.Query; + +namespace Levels.Period; + +public static class DemotionService +{ + public static void Demote(string periodFilePath, FileIndex fileIndex) + { + // Delete PERIOD file + if (File.Exists(periodFilePath)) + File.Delete(periodFilePath); + + // Delete manifest sidecar + var manifestPath = periodFilePath + ".manifest.json"; + if (File.Exists(manifestPath)) + File.Delete(manifestPath); + + // Remove from index + fileIndex.Remove(periodFilePath); + } +} diff --git a/src/Levels.Period/IHealthCheck.cs b/src/Levels.Period/IHealthCheck.cs new file mode 100644 index 0000000..428efd2 --- /dev/null +++ b/src/Levels.Period/IHealthCheck.cs @@ -0,0 +1,12 @@ +using Levels.Core; +using Levels.Core.Format; + +namespace Levels.Period; + +public interface IHealthCheck +{ + string Name { get; } + HealthCheckResult Check(IReadOnlyList records, PriceStreamId streamId); +} + +public readonly record struct HealthCheckResult(bool Passed, string? FailureReason = null); diff --git a/src/Levels.Period/Levels.Period.csproj b/src/Levels.Period/Levels.Period.csproj new file mode 100644 index 0000000..8443d7f --- /dev/null +++ b/src/Levels.Period/Levels.Period.csproj @@ -0,0 +1,24 @@ + + + + net10.0 + enable + enable + preview + + + + + + + + + + + + + + + + + diff --git a/src/Levels.Period/Log.cs b/src/Levels.Period/Log.cs new file mode 100644 index 0000000..db010f8 --- /dev/null +++ b/src/Levels.Period/Log.cs @@ -0,0 +1,18 @@ +using Microsoft.Extensions.Logging; + +namespace Levels.Period; + +internal static partial class Log +{ + [LoggerMessage(Level = LogLevel.Information, Message = "Health check passed: CheckName={CheckName}, StreamId={StreamId}")] + public static partial void HealthCheckPassed(ILogger logger, string checkName, string streamId); + + [LoggerMessage(Level = LogLevel.Information, Message = "Health check failed: CheckName={CheckName}, StreamId={StreamId}, Reason={Reason}")] + public static partial void HealthCheckFailed(ILogger logger, string checkName, string streamId, string reason); + + [LoggerMessage(Level = LogLevel.Information, Message = "Promotion completed: FilePath={FilePath}")] + public static partial void PromotionCompleted(ILogger logger, string filePath); + + [LoggerMessage(Level = LogLevel.Information, Message = "Demotion completed: FilePath={FilePath}")] + public static partial void DemotionCompleted(ILogger logger, string filePath); +} diff --git a/src/Levels.Period/MissingValueConfig.cs b/src/Levels.Period/MissingValueConfig.cs new file mode 100644 index 0000000..0955544 --- /dev/null +++ b/src/Levels.Period/MissingValueConfig.cs @@ -0,0 +1,13 @@ +using Levels.Core; + +namespace Levels.Period; + +public sealed class MissingValueConfig +{ + public long DefaultGapThresholdNanos { get; init; } = 5L * 60 * 1_000_000_000; // 5 minutes + + public Dictionary StreamOverrides { get; init; } = new(); + + public long GetThreshold(PriceStreamId streamId) + => StreamOverrides.TryGetValue(streamId, out var threshold) ? threshold : DefaultGapThresholdNanos; +} diff --git a/src/Levels.Period/MissingValueDetector.cs b/src/Levels.Period/MissingValueDetector.cs new file mode 100644 index 0000000..04db0ed --- /dev/null +++ b/src/Levels.Period/MissingValueDetector.cs @@ -0,0 +1,39 @@ +using Levels.Core; +using Levels.Core.Format; + +namespace Levels.Period; + +public sealed class MissingValueDetector : IHealthCheck +{ + private readonly MissingValueConfig _config; + + public MissingValueDetector(MissingValueConfig config) + { + _config = config; + } + + public string Name => "MissingValue"; + + public HealthCheckResult Check(IReadOnlyList records, PriceStreamId streamId) + { + if (records.Count <= 1) + return new HealthCheckResult(true); + + var threshold = _config.GetThreshold(streamId); + var lastObservedTime = records[0].Core.ObservedTime; + + for (var i = 1; i < records.Count; i++) + { + var current = records[i].Core.ObservedTime; + var gap = current - lastObservedTime; + + if (gap > threshold) + return new HealthCheckResult(false, + $"Gap of {gap} nanos between records {i - 1} and {i} exceeds threshold {threshold}."); + + lastObservedTime = current; + } + + return new HealthCheckResult(true); + } +} diff --git a/src/Levels.Period/PeriodConfig.cs b/src/Levels.Period/PeriodConfig.cs new file mode 100644 index 0000000..b3461ef --- /dev/null +++ b/src/Levels.Period/PeriodConfig.cs @@ -0,0 +1,9 @@ +namespace Levels.Period; + +public sealed class PeriodConfig +{ + public required string DataPath { get; init; } + public TimeSpan CompactionWindow { get; init; } = TimeSpan.FromHours(1); + public required string ConfigVersion { get; init; } + public MissingValueConfig MissingValueConfig { get; init; } = new(); +} diff --git a/src/Levels.Period/PeriodPromotion.cs b/src/Levels.Period/PeriodPromotion.cs new file mode 100644 index 0000000..5d2242d --- /dev/null +++ b/src/Levels.Period/PeriodPromotion.cs @@ -0,0 +1,120 @@ +using Levels.Core; +using Levels.Core.Diagnostics; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.DataFlow; +using Levels.Query; + +namespace Levels.Period; + +public sealed class PeriodPromotion : IDataFlowHandler +{ + private readonly PeriodConfig _config; + private readonly FileIndex _fileIndex; + private readonly IReadOnlyList _checks; + + public PeriodPromotion(PeriodConfig config, FileIndex fileIndex, IReadOnlyList checks) + { + _config = config; + _fileIndex = fileIndex; + _checks = checks; + } + + public ValueTask OnRecordWritten(PriceStreamId stream, RawRecord record, CancellationToken ct) + => ValueTask.CompletedTask; + + public ValueTask OnFileSealed(SealedFileInfo file, CancellationToken ct) + => ValueTask.CompletedTask; + + public ValueTask OnAggCreated(AggFileInfo file, CancellationToken ct) + { + Promote(file); + return ValueTask.CompletedTask; + } + + internal void Promote(AggFileInfo file) + { + var aggPath = file.FilePath; + var periodPath = Path.ChangeExtension(aggPath, ".period"); + var manifestPath = periodPath + ".manifest.json"; + + // Re-promotion check + if (_fileIndex.HasFile(file.PriceStreamId, FileType.Period, file.FirstObservedTime, file.LastObservedTime)) + return; + + // Read all records from AGG + using var fs = File.OpenRead(aggPath); + var reader = new BinaryRecordReader(fs); + var records = reader.ReadRecords(validateCrc: true).ToList(); + + // Run all health checks + var checkResults = new List(); + var allPassed = true; + + foreach (var check in _checks) + { + var result = check.Check(records, file.PriceStreamId); + checkResults.Add(new CheckResultEntry + { + CheckName = check.Name, + Passed = result.Passed, + FailureReason = result.FailureReason, + }); + if (!result.Passed) + allPassed = false; + } + + var manifest = new PromotionManifest + { + AggFilePath = aggPath, + PeriodFilePath = periodPath, + PriceStreamId = file.PriceStreamId.Value.ToString(), + PromotedAt = DateTimeOffset.UtcNow, + ConfigVersion = _config.ConfigVersion, + CheckResults = checkResults, + }; + + if (!allPassed) + { + StlthLevelsMetrics.HealthCheckFailed.Add(1); + // Write manifest recording failures, but do NOT create PERIOD file + manifest.WriteTo(manifestPath); + return; + } + + StlthLevelsMetrics.HealthCheckPassed.Add(1); + + // Copy AGG → PERIOD, then patch FileType byte + File.Copy(aggPath, periodPath); + + // Patch FileType byte at offset 10 from Agg (0x01) to Period (0x02) + using (var patchStream = new FileStream(periodPath, FileMode.Open, FileAccess.Write, FileShare.None)) + { + patchStream.Seek(10, SeekOrigin.Begin); + patchStream.WriteByte((byte)FileType.Period); + } + + // Write manifest + manifest.WriteTo(manifestPath); + + // Register in index + var entry = new FileIndexEntry( + periodPath, + file.PriceStreamId, + ExtractVenue(periodPath), + FileType.Period, + file.FirstObservedTime, + file.LastObservedTime, + file.RecordCount); + + _fileIndex.Register(entry); + StlthLevelsMetrics.PromotionsCompleted.Add(1); + } + + private string ExtractVenue(string filePath) + { + var relativePath = Path.GetRelativePath(_config.DataPath, filePath); + var parts = relativePath.Split(Path.DirectorySeparatorChar); + return parts.Length >= 3 ? parts[0] : "unknown"; + } +} diff --git a/src/Levels.Period/PromotionManifest.cs b/src/Levels.Period/PromotionManifest.cs new file mode 100644 index 0000000..5b6458d --- /dev/null +++ b/src/Levels.Period/PromotionManifest.cs @@ -0,0 +1,40 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Levels.Period; + +public sealed class PromotionManifest +{ + public required string AggFilePath { get; init; } + public required string PeriodFilePath { get; init; } + public required string PriceStreamId { get; init; } + public required DateTimeOffset PromotedAt { get; init; } + public required string ConfigVersion { get; init; } + public required List CheckResults { get; init; } + + private static readonly JsonSerializerOptions JsonOptions = new() + { + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + public void WriteTo(string path) + { + var json = JsonSerializer.Serialize(this, JsonOptions); + File.WriteAllText(path, json); + } + + public static PromotionManifest ReadFrom(string path) + { + var json = File.ReadAllText(path); + return JsonSerializer.Deserialize(json, JsonOptions) + ?? throw new InvalidDataException("Failed to deserialize promotion manifest."); + } +} + +public sealed class CheckResultEntry +{ + public required string CheckName { get; init; } + public required bool Passed { get; init; } + public string? FailureReason { get; init; } +} diff --git a/src/Levels.Period/SequenceIntegrityChecker.cs b/src/Levels.Period/SequenceIntegrityChecker.cs new file mode 100644 index 0000000..843cef5 --- /dev/null +++ b/src/Levels.Period/SequenceIntegrityChecker.cs @@ -0,0 +1,29 @@ +using Levels.Core; +using Levels.Core.Format; + +namespace Levels.Period; + +public sealed class SequenceIntegrityChecker : IHealthCheck +{ + public string Name => "SequenceIntegrity"; + + public HealthCheckResult Check(IReadOnlyList records, PriceStreamId streamId) + { + if (records.Count <= 1) + return new HealthCheckResult(true); + + var lastWriteTimestamp = records[0].Core.WriteTimestamp; + + for (var i = 1; i < records.Count; i++) + { + var current = records[i].Core.WriteTimestamp; + if (current < lastWriteTimestamp) + return new HealthCheckResult(false, + $"Backwards WriteTimestamp jump at record {i}: {current} < {lastWriteTimestamp}."); + + lastWriteTimestamp = current; + } + + return new HealthCheckResult(true); + } +} diff --git a/src/Levels.Period/SnapCoverageChecker.cs b/src/Levels.Period/SnapCoverageChecker.cs new file mode 100644 index 0000000..27eff4a --- /dev/null +++ b/src/Levels.Period/SnapCoverageChecker.cs @@ -0,0 +1,20 @@ +using Levels.Core; +using Levels.Core.Format; + +namespace Levels.Period; + +public sealed class SnapCoverageChecker : IHealthCheck +{ + public string Name => "SnapCoverage"; + + public HealthCheckResult Check(IReadOnlyList records, PriceStreamId streamId) + { + if (records.Count == 0) + return new HealthCheckResult(false, "No records in file."); + + if (records[0].Core.Type != RecordType.Snap) + return new HealthCheckResult(false, "First record is not a SNAP."); + + return new HealthCheckResult(true); + } +} diff --git a/src/Levels.Protocol/FrameReader.cs b/src/Levels.Protocol/FrameReader.cs new file mode 100644 index 0000000..7061fca --- /dev/null +++ b/src/Levels.Protocol/FrameReader.cs @@ -0,0 +1,255 @@ +using System.Buffers.Binary; +using System.Text; + +namespace Levels.Protocol; + +/// +/// Reads length-prefixed binary protocol frames from a stream. +/// +public sealed class FrameReader +{ + private readonly Stream _stream; + + public FrameReader(Stream stream) + { + _stream = stream; + } + + /// + /// Reads the next frame. Returns null on end of stream. + /// + public async ValueTask ReadFrameAsync(CancellationToken ct = default) + { + var header = new byte[5]; + var bytesRead = await ReadExactAsync(header, ct); + if (bytesRead == 0) + return null; + if (bytesRead < 5) + throw new ProtocolException("Incomplete frame header"); + + var frameLength = BinaryPrimitives.ReadInt32LittleEndian(header); + var messageType = (MessageType)header[4]; + var payloadLength = frameLength - 1; + + if (payloadLength < 0 || payloadLength > 16 * 1024 * 1024) + throw new ProtocolException($"Invalid frame length: {frameLength}"); + + var payload = new byte[payloadLength]; + if (payloadLength > 0) + { + var read = await ReadExactAsync(payload, ct); + if (read < payloadLength) + throw new ProtocolException("Incomplete frame payload"); + } + + return new Frame(messageType, payload); + } + + private async ValueTask ReadExactAsync(Memory buffer, CancellationToken ct) + { + var totalRead = 0; + while (totalRead < buffer.Length) + { + var read = await _stream.ReadAsync(buffer[totalRead..], ct); + if (read == 0) + return totalRead; + totalRead += read; + } + return totalRead; + } +} + +public readonly record struct Frame(MessageType Type, ReadOnlyMemory Payload) +{ + public HandshakeMessage ReadHandshake() + { + var span = Payload.Span; + var schemaId = BinaryPrimitives.ReadUInt32LittleEndian(span); + var recordSize = BinaryPrimitives.ReadInt32LittleEndian(span[4..]); + return new HandshakeMessage(schemaId, recordSize); + } + + public HandshakeAckMessage ReadHandshakeAck() + { + var span = Payload.Span; + var status = (HandshakeStatus)span[0]; + string? error = null; + if (status != HandshakeStatus.Ok && span.Length > 1) + { + var errorLen = BinaryPrimitives.ReadUInt16LittleEndian(span[1..]); + error = Encoding.UTF8.GetString(span.Slice(3, errorLen)); + } + return new HandshakeAckMessage(status, error); + } + + public WriteRecordMessage ReadWriteRecord() + { + var span = Payload.Span; + var offset = 0; + + var venueLen = BinaryPrimitives.ReadUInt16LittleEndian(span[offset..]); + offset += 2; + var venue = Encoding.UTF8.GetString(span.Slice(offset, venueLen)); + offset += venueLen; + + var symbolLen = BinaryPrimitives.ReadUInt16LittleEndian(span[offset..]); + offset += 2; + var symbol = Encoding.UTF8.GetString(span.Slice(offset, symbolLen)); + offset += symbolLen; + + var recordBytes = Payload[offset..]; + return new WriteRecordMessage(venue, symbol, recordBytes); + } + + public WriteAckMessage ReadWriteAck() + { + var span = Payload.Span; + var accepted = BinaryPrimitives.ReadInt64LittleEndian(span); + var rejected = BinaryPrimitives.ReadInt64LittleEndian(span[8..]); + return new WriteAckMessage(accepted, rejected); + } + + public GetBookRequestMessage ReadGetBookRequest() + { + var span = Payload.Span; + var offset = 0; + var (venue, symbol, newOffset) = ReadVenueSymbol(span, offset); + offset = newOffset; + var fromNanos = BinaryPrimitives.ReadInt64LittleEndian(span[offset..]); + offset += 8; + var toNanos = BinaryPrimitives.ReadInt64LittleEndian(span[offset..]); + offset += 8; + var excludeOwner = span[offset] != 0; + return new GetBookRequestMessage(venue, symbol, fromNanos, toNanos, excludeOwner); + } + + public GetBookResponseMessage ReadGetBookResponse() + { + var span = Payload.Span; + var offset = 0; + + var bidCount = BinaryPrimitives.ReadInt32LittleEndian(span[offset..]); + offset += 4; + var bids = new (long Price, long Qty)[bidCount]; + for (int i = 0; i < bidCount; i++) + { + bids[i] = (BinaryPrimitives.ReadInt64LittleEndian(span[offset..]), + BinaryPrimitives.ReadInt64LittleEndian(span[(offset + 8)..])); + offset += 16; + } + + var askCount = BinaryPrimitives.ReadInt32LittleEndian(span[offset..]); + offset += 4; + var asks = new (long Price, long Qty)[askCount]; + for (int i = 0; i < askCount; i++) + { + asks[i] = (BinaryPrimitives.ReadInt64LittleEndian(span[offset..]), + BinaryPrimitives.ReadInt64LittleEndian(span[(offset + 8)..])); + offset += 16; + } + + return new GetBookResponseMessage(bids, asks); + } + + public GetBookL1ResponseMessage ReadGetBookL1Response() + { + var span = Payload.Span; + (long, long)? bestBid = null; + (long, long)? bestAsk = null; + + if (span[0] != 0) + bestBid = (BinaryPrimitives.ReadInt64LittleEndian(span[1..]), BinaryPrimitives.ReadInt64LittleEndian(span[9..])); + if (span[17] != 0) + bestAsk = (BinaryPrimitives.ReadInt64LittleEndian(span[18..]), BinaryPrimitives.ReadInt64LittleEndian(span[26..])); + + return new GetBookL1ResponseMessage(bestBid, bestAsk); + } + + public GetStreamsResponseMessage ReadGetStreamsResponse() + { + var span = Payload.Span; + var offset = 0; + var count = BinaryPrimitives.ReadInt32LittleEndian(span); + offset += 4; + var streams = new (string Venue, string Symbol)[count]; + for (int i = 0; i < count; i++) + { + var (venue, symbol, newOffset) = ReadVenueSymbol(span, offset); + offset = newOffset; + streams[i] = (venue, symbol); + } + return new GetStreamsResponseMessage(streams); + } + + public ExportRequestMessage ReadExportRequest() + { + var span = Payload.Span; + var offset = 0; + var (venue, symbol, o1) = ReadVenueSymbol(span, offset); + offset = o1; + var (format, o2) = ReadString(span, offset); + offset = o2; + var fromNanos = BinaryPrimitives.ReadInt64LittleEndian(span[offset..]); + offset += 8; + var toNanos = BinaryPrimitives.ReadInt64LittleEndian(span[offset..]); + return new ExportRequestMessage(venue, symbol, format, fromNanos, toNanos); + } + + public ExportChunkMessage ReadExportChunk() + { + var span = Payload.Span; + var len = BinaryPrimitives.ReadInt32LittleEndian(span); + return new ExportChunkMessage(Payload.Slice(4, len)); + } + + public GetHealthResponseMessage ReadGetHealthResponse() + { + var span = Payload.Span; + var (status, o1) = ReadString(span, 0); + var (timestamp, _) = ReadString(span, o1); + return new GetHealthResponseMessage(status, timestamp); + } + + public GetSchemaInfoResponseMessage ReadGetSchemaInfoResponse() + { + var span = Payload.Span; + var schemaId = BinaryPrimitives.ReadUInt32LittleEndian(span); + var recordSize = BinaryPrimitives.ReadInt32LittleEndian(span[4..]); + var (typeName, _) = ReadString(span, 8); + return new GetSchemaInfoResponseMessage(schemaId, recordSize, typeName); + } + + public string ReadErrorResponse() + { + var (msg, _) = ReadString(Payload.Span, 0); + return msg; + } + + private static (string Venue, string Symbol, int NewOffset) ReadVenueSymbol(ReadOnlySpan span, int offset) + { + var (venue, o1) = ReadString(span, offset); + var (symbol, o2) = ReadString(span, o1); + return (venue, symbol, o2); + } + + private static (string Value, int NewOffset) ReadString(ReadOnlySpan span, int offset) + { + var len = BinaryPrimitives.ReadUInt16LittleEndian(span[offset..]); + offset += 2; + var value = Encoding.UTF8.GetString(span.Slice(offset, len)); + return (value, offset + len); + } +} + +public readonly record struct HandshakeMessage(uint SchemaId, int RecordSize); +public readonly record struct HandshakeAckMessage(HandshakeStatus Status, string? Error); +public readonly record struct WriteRecordMessage(string Venue, string Symbol, ReadOnlyMemory RecordBytes); +public readonly record struct WriteAckMessage(long Accepted, long Rejected); +public readonly record struct GetBookRequestMessage(string Venue, string Symbol, long FromNanos, long ToNanos, bool ExcludeOwner); +public readonly record struct GetBookResponseMessage(IReadOnlyList<(long Price, long Qty)> Bids, IReadOnlyList<(long Price, long Qty)> Asks); +public readonly record struct GetBookL1ResponseMessage((long Price, long Qty)? BestBid, (long Price, long Qty)? BestAsk); +public readonly record struct GetStreamsResponseMessage(IReadOnlyList<(string Venue, string Symbol)> Streams); +public readonly record struct ExportRequestMessage(string Venue, string Symbol, string Format, long FromNanos, long ToNanos); +public readonly record struct ExportChunkMessage(ReadOnlyMemory Data); +public readonly record struct GetHealthResponseMessage(string Status, string Timestamp); +public readonly record struct GetSchemaInfoResponseMessage(uint SchemaId, int RecordSize, string TypeName); diff --git a/src/Levels.Protocol/FrameWriter.cs b/src/Levels.Protocol/FrameWriter.cs new file mode 100644 index 0000000..fb320d0 --- /dev/null +++ b/src/Levels.Protocol/FrameWriter.cs @@ -0,0 +1,251 @@ +using System.Buffers.Binary; +using System.Text; + +namespace Levels.Protocol; + +/// +/// Writes length-prefixed binary protocol frames to a stream. +/// Frame: [4 bytes payload length, little-endian][1 byte message type][payload] +/// +public sealed class FrameWriter +{ + private readonly Stream _stream; + + public FrameWriter(Stream stream) + { + _stream = stream; + } + + public async ValueTask WriteHandshakeAsync(uint schemaId, int recordSize, CancellationToken ct = default) + { + var payload = new byte[8]; + BinaryPrimitives.WriteUInt32LittleEndian(payload, schemaId); + BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan(4), recordSize); + await WriteFrameAsync(MessageType.Handshake, payload, ct); + } + + public async ValueTask WriteHandshakeAckAsync(HandshakeStatus status, string? error = null, CancellationToken ct = default) + { + if (status == HandshakeStatus.Ok) + { + await WriteFrameAsync(MessageType.HandshakeAck, new byte[] { (byte)status }, ct); + } + else + { + var errorBytes = Encoding.UTF8.GetBytes(error ?? ""); + var payload = new byte[1 + 2 + errorBytes.Length]; + payload[0] = (byte)status; + BinaryPrimitives.WriteUInt16LittleEndian(payload.AsSpan(1), (ushort)errorBytes.Length); + errorBytes.CopyTo(payload.AsSpan(3)); + await WriteFrameAsync(MessageType.HandshakeAck, payload, ct); + } + } + + public async ValueTask WriteRecordAsync(string venue, string symbol, ReadOnlyMemory recordBytes, CancellationToken ct = default) + { + var venueBytes = Encoding.UTF8.GetBytes(venue); + var symbolBytes = Encoding.UTF8.GetBytes(symbol); + var payloadSize = 2 + venueBytes.Length + 2 + symbolBytes.Length + recordBytes.Length; + var payload = new byte[payloadSize]; + var offset = 0; + + BinaryPrimitives.WriteUInt16LittleEndian(payload.AsSpan(offset), (ushort)venueBytes.Length); + offset += 2; + venueBytes.CopyTo(payload.AsSpan(offset)); + offset += venueBytes.Length; + + BinaryPrimitives.WriteUInt16LittleEndian(payload.AsSpan(offset), (ushort)symbolBytes.Length); + offset += 2; + symbolBytes.CopyTo(payload.AsSpan(offset)); + offset += symbolBytes.Length; + + recordBytes.Span.CopyTo(payload.AsSpan(offset)); + + await WriteFrameAsync(MessageType.WriteRecord, payload, ct); + } + + public async ValueTask WriteAckAsync(long accepted, long rejected, CancellationToken ct = default) + { + var payload = new byte[16]; + BinaryPrimitives.WriteInt64LittleEndian(payload, accepted); + BinaryPrimitives.WriteInt64LittleEndian(payload.AsSpan(8), rejected); + await WriteFrameAsync(MessageType.WriteAck, payload, ct); + } + + // ---- Query messages ---- + + public async ValueTask WriteGetStreamsAsync(CancellationToken ct = default) + { + await WriteFrameAsync(MessageType.GetStreams, ReadOnlyMemory.Empty, ct); + } + + public async ValueTask WriteGetStreamsResponseAsync(IReadOnlyList<(string Venue, string Symbol)> streams, CancellationToken ct = default) + { + using var ms = new MemoryStream(); + WriteInt32(ms, streams.Count); + foreach (var (venue, symbol) in streams) + { + WriteString(ms, venue); + WriteString(ms, symbol); + } + await WriteFrameAsync(MessageType.GetStreamsResponse, ms.ToArray(), ct); + } + + public async ValueTask WriteGetBookAsync(string venue, string symbol, long fromNanos, long toNanos, bool excludeOwner, CancellationToken ct = default) + { + using var ms = new MemoryStream(); + WriteString(ms, venue); + WriteString(ms, symbol); + WriteInt64(ms, fromNanos); + WriteInt64(ms, toNanos); + ms.WriteByte(excludeOwner ? (byte)1 : (byte)0); + await WriteFrameAsync(MessageType.GetBook, ms.ToArray(), ct); + } + + public async ValueTask WriteGetBookResponseAsync(IReadOnlyList<(long Price, long Qty)> bids, IReadOnlyList<(long Price, long Qty)> asks, CancellationToken ct = default) + { + using var ms = new MemoryStream(); + WriteInt32(ms, bids.Count); + foreach (var (price, qty) in bids) + { + WriteInt64(ms, price); + WriteInt64(ms, qty); + } + WriteInt32(ms, asks.Count); + foreach (var (price, qty) in asks) + { + WriteInt64(ms, price); + WriteInt64(ms, qty); + } + await WriteFrameAsync(MessageType.GetBookResponse, ms.ToArray(), ct); + } + + public async ValueTask WriteGetBookL1Async(string venue, string symbol, long fromNanos, long toNanos, bool excludeOwner, CancellationToken ct = default) + { + using var ms = new MemoryStream(); + WriteString(ms, venue); + WriteString(ms, symbol); + WriteInt64(ms, fromNanos); + WriteInt64(ms, toNanos); + ms.WriteByte(excludeOwner ? (byte)1 : (byte)0); + await WriteFrameAsync(MessageType.GetBookL1, ms.ToArray(), ct); + } + + public async ValueTask WriteGetBookL1ResponseAsync((long Price, long Qty)? bestBid, (long Price, long Qty)? bestAsk, CancellationToken ct = default) + { + var payload = new byte[34]; // 1+8+8 + 1+8+8 + var offset = 0; + if (bestBid is var (bp, bq)) + { + payload[offset] = 1; + BinaryPrimitives.WriteInt64LittleEndian(payload.AsSpan(offset + 1), bp); + BinaryPrimitives.WriteInt64LittleEndian(payload.AsSpan(offset + 9), bq); + } + offset = 17; + if (bestAsk is var (ap, aq)) + { + payload[offset] = 1; + BinaryPrimitives.WriteInt64LittleEndian(payload.AsSpan(offset + 1), ap); + BinaryPrimitives.WriteInt64LittleEndian(payload.AsSpan(offset + 9), aq); + } + await WriteFrameAsync(MessageType.GetBookL1Response, payload, ct); + } + + public async ValueTask WriteExportAsync(string venue, string symbol, string format, long fromNanos, long toNanos, CancellationToken ct = default) + { + using var ms = new MemoryStream(); + WriteString(ms, venue); + WriteString(ms, symbol); + WriteString(ms, format); + WriteInt64(ms, fromNanos); + WriteInt64(ms, toNanos); + await WriteFrameAsync(MessageType.Export, ms.ToArray(), ct); + } + + public async ValueTask WriteExportChunkAsync(ReadOnlyMemory data, CancellationToken ct = default) + { + var payload = new byte[4 + data.Length]; + BinaryPrimitives.WriteInt32LittleEndian(payload, data.Length); + data.Span.CopyTo(payload.AsSpan(4)); + await WriteFrameAsync(MessageType.ExportChunk, payload, ct); + } + + // ---- Admin messages ---- + + public async ValueTask WriteGetHealthAsync(CancellationToken ct = default) + { + await WriteFrameAsync(MessageType.GetHealth, ReadOnlyMemory.Empty, ct); + } + + public async ValueTask WriteGetHealthResponseAsync(string status, string timestamp, CancellationToken ct = default) + { + using var ms = new MemoryStream(); + WriteString(ms, status); + WriteString(ms, timestamp); + await WriteFrameAsync(MessageType.GetHealthResponse, ms.ToArray(), ct); + } + + public async ValueTask WriteGetSchemaInfoAsync(CancellationToken ct = default) + { + await WriteFrameAsync(MessageType.GetSchemaInfo, ReadOnlyMemory.Empty, ct); + } + + public async ValueTask WriteGetSchemaInfoResponseAsync(uint schemaId, int recordSize, string typeName, CancellationToken ct = default) + { + using var ms = new MemoryStream(); + var buf = new byte[8]; + BinaryPrimitives.WriteUInt32LittleEndian(buf, schemaId); + ms.Write(buf, 0, 4); + BinaryPrimitives.WriteInt32LittleEndian(buf, recordSize); + ms.Write(buf, 0, 4); + WriteString(ms, typeName); + await WriteFrameAsync(MessageType.GetSchemaInfoResponse, ms.ToArray(), ct); + } + + public async ValueTask WriteErrorResponseAsync(string error, CancellationToken ct = default) + { + using var ms = new MemoryStream(); + WriteString(ms, error); + await WriteFrameAsync(MessageType.ErrorResponse, ms.ToArray(), ct); + } + + public async ValueTask FlushAsync(CancellationToken ct = default) + { + await _stream.FlushAsync(ct); + } + + // ---- Helpers ---- + + private static void WriteString(MemoryStream ms, string value) + { + var bytes = Encoding.UTF8.GetBytes(value); + var lenBuf = new byte[2]; + BinaryPrimitives.WriteUInt16LittleEndian(lenBuf, (ushort)bytes.Length); + ms.Write(lenBuf, 0, 2); + ms.Write(bytes, 0, bytes.Length); + } + + private static void WriteInt32(MemoryStream ms, int value) + { + var buf = new byte[4]; + BinaryPrimitives.WriteInt32LittleEndian(buf, value); + ms.Write(buf, 0, 4); + } + + private static void WriteInt64(MemoryStream ms, long value) + { + var buf = new byte[8]; + BinaryPrimitives.WriteInt64LittleEndian(buf, value); + ms.Write(buf, 0, 8); + } + + private async ValueTask WriteFrameAsync(MessageType type, ReadOnlyMemory payload, CancellationToken ct) + { + var header = new byte[5]; + BinaryPrimitives.WriteInt32LittleEndian(header, payload.Length + 1); // +1 for message type byte + header[4] = (byte)type; + + await _stream.WriteAsync(header, ct); + await _stream.WriteAsync(payload, ct); + } +} diff --git a/src/Levels.Protocol/Levels.Protocol.csproj b/src/Levels.Protocol/Levels.Protocol.csproj new file mode 100644 index 0000000..08fea1b --- /dev/null +++ b/src/Levels.Protocol/Levels.Protocol.csproj @@ -0,0 +1,14 @@ + + + + net10.0 + enable + enable + preview + + + + + + + diff --git a/src/Levels.Protocol/MessageType.cs b/src/Levels.Protocol/MessageType.cs new file mode 100644 index 0000000..ca26347 --- /dev/null +++ b/src/Levels.Protocol/MessageType.cs @@ -0,0 +1,110 @@ +namespace Levels.Protocol; + +/// +/// Wire protocol message types. +/// Frame format: [4 bytes length][1 byte type][payload] +/// All values are little-endian. +/// +public enum MessageType : byte +{ + /// + /// Client → Server. Payload: [4 bytes schema_id][4 bytes record_size] + /// + Handshake = 0x01, + + /// + /// Server → Client. Payload: [1 byte status][optional: 2 bytes error_len + error UTF-8] + /// + HandshakeAck = 0x02, + + /// + /// Client → Server. Payload: [2 bytes venue_len][venue UTF-8][2 bytes symbol_len][symbol UTF-8][record_size bytes record] + /// + WriteRecord = 0x10, + + /// + /// Client → Server. Payload: [4 bytes count][count × WriteRecord payloads] + /// + WriteBatch = 0x12, + + /// + /// Server → Client. Payload: [8 bytes accepted][8 bytes rejected] + /// + WriteAck = 0x11, + + // ---- Query messages ---- + + /// + /// Client → Server. Payload: empty + /// + GetStreams = 0x20, + + /// + /// Server → Client. Payload: [4 bytes count][count × (2 bytes venue_len + venue + 2 bytes symbol_len + symbol)] + /// + GetStreamsResponse = 0x21, + + /// + /// Client → Server. Payload: [2 bytes venue_len][venue][2 bytes symbol_len][symbol][8 bytes from_nanos][8 bytes to_nanos][1 byte exclude_owner] + /// + GetBook = 0x22, + + /// + /// Server → Client. Payload: [4 bytes bid_count][bid_count × (8 bytes price + 8 bytes qty)][4 bytes ask_count][ask_count × (8 bytes price + 8 bytes qty)] + /// + GetBookResponse = 0x23, + + /// + /// Client → Server. Same payload as GetBook. + /// + GetBookL1 = 0x24, + + /// + /// Server → Client. Payload: [1 byte has_bid][8 bytes bid_price][8 bytes bid_qty][1 byte has_ask][8 bytes ask_price][8 bytes ask_qty] + /// + GetBookL1Response = 0x25, + + /// + /// Client → Server. Payload: [2 bytes venue_len][venue][2 bytes symbol_len][symbol][2 bytes format_len][format][8 bytes from_nanos][8 bytes to_nanos] + /// + Export = 0x26, + + /// + /// Server → Client. Payload: [4 bytes chunk_len][chunk_bytes]. Sent repeatedly, final chunk has length 0. + /// + ExportChunk = 0x27, + + // ---- Admin messages ---- + + /// + /// Client → Server. Payload: empty + /// + GetHealth = 0x30, + + /// + /// Server → Client. Payload: [1 byte status_len][status UTF-8][2 bytes timestamp_len][timestamp UTF-8] + /// + GetHealthResponse = 0x31, + + /// + /// Client → Server. Payload: empty + /// + GetSchemaInfo = 0x32, + + /// + /// Server → Client. Payload: [4 bytes schema_id][4 bytes record_size][2 bytes type_name_len][type_name UTF-8] + /// + GetSchemaInfoResponse = 0x33, + + /// + /// Server → Client. Sent when a request fails. Payload: [2 bytes error_len][error UTF-8] + /// + ErrorResponse = 0xFF, +} + +public enum HandshakeStatus : byte +{ + Ok = 0, + SchemaMismatch = 1, + Error = 2, +} diff --git a/src/Levels.Protocol/ProtocolException.cs b/src/Levels.Protocol/ProtocolException.cs new file mode 100644 index 0000000..ba7df93 --- /dev/null +++ b/src/Levels.Protocol/ProtocolException.cs @@ -0,0 +1,7 @@ +namespace Levels.Protocol; + +public sealed class ProtocolException : Exception +{ + public ProtocolException(string message) : base(message) { } + public ProtocolException(string message, Exception inner) : base(message, inner) { } +} diff --git a/src/Levels.Query/FileIndex.cs b/src/Levels.Query/FileIndex.cs new file mode 100644 index 0000000..38396ef --- /dev/null +++ b/src/Levels.Query/FileIndex.cs @@ -0,0 +1,169 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; + +namespace Levels.Query; + +public sealed class FileIndex +{ + private readonly Dictionary> _entries = new(); + private readonly ReaderWriterLockSlim _lock = new(); + + public void LoadFromDisk(string dataPath) + { + if (!Directory.Exists(dataPath)) + return; + + var extensions = new[] { "*.raw", "*.agg", "*.period", "*.resampled" }; + + foreach (var ext in extensions) + { + foreach (var file in Directory.GetFiles(dataPath, ext, SearchOption.AllDirectories)) + { + var entry = TryReadEntry(dataPath, file); + if (entry is not null) + Register(entry.Value); + } + } + } + + public static FileIndexEntry? TryReadEntryStatic(string dataPath, string filePath) + => TryReadEntry(dataPath, filePath); + + private static FileIndexEntry? TryReadEntry(string dataPath, string filePath) + { + try + { + using var fs = File.OpenRead(filePath); + if (fs.Length < Constants.HeaderSize + Constants.FooterSize) + return null; + + var reader = new BinaryRecordReader(fs); + if (reader.Footer is null) + return null; + + var header = reader.Header; + var footer = reader.Footer.Value; + var venue = ExtractVenue(dataPath, filePath); + + return new FileIndexEntry( + filePath, + new PriceStreamId(header.PriceStreamId), + venue, + header.FileType, + footer.FirstObservedTime, + footer.LastObservedTime, + footer.RecordCount); + } + catch + { + return null; + } + } + + private static string ExtractVenue(string dataPath, string filePath) + { + var relativePath = Path.GetRelativePath(dataPath, filePath); + var parts = relativePath.Split(Path.DirectorySeparatorChar); + return parts.Length >= 3 ? parts[0] : "unknown"; + } + + public void Register(FileIndexEntry entry) + { + _lock.EnterWriteLock(); + try + { + if (!_entries.TryGetValue(entry.PriceStreamId, out var list)) + { + list = new List(); + _entries[entry.PriceStreamId] = list; + } + list.Add(entry); + } + finally + { + _lock.ExitWriteLock(); + } + } + + public void Remove(string filePath) + { + _lock.EnterWriteLock(); + try + { + foreach (var list in _entries.Values) + { + list.RemoveAll(e => e.FilePath == filePath); + } + } + finally + { + _lock.ExitWriteLock(); + } + } + + public IReadOnlyList Query(PriceStreamId streamId, long fromNanos, long toNanos) + { + _lock.EnterReadLock(); + try + { + if (!_entries.TryGetValue(streamId, out var list)) + return []; + + return list + .Where(e => e.FirstObservedTime < toNanos && e.LastObservedTime >= fromNanos) + .ToList(); + } + finally + { + _lock.ExitReadLock(); + } + } + + public IReadOnlyList Query(PriceStreamId streamId, long fromNanos, long toNanos, FileType fileType) + { + _lock.EnterReadLock(); + try + { + if (!_entries.TryGetValue(streamId, out var list)) + return []; + + return list + .Where(e => e.FileType == fileType && e.FirstObservedTime < toNanos && e.LastObservedTime >= fromNanos) + .ToList(); + } + finally + { + _lock.ExitReadLock(); + } + } + + public IReadOnlyList GetAllStreams() + { + _lock.EnterReadLock(); + try + { + return _entries.Keys.ToList(); + } + finally + { + _lock.ExitReadLock(); + } + } + + public bool HasFile(PriceStreamId streamId, FileType fileType, long fromNanos, long toNanos) + { + _lock.EnterReadLock(); + try + { + if (!_entries.TryGetValue(streamId, out var list)) + return false; + + return list.Any(e => e.FileType == fileType && e.FirstObservedTime < toNanos && e.LastObservedTime >= fromNanos); + } + finally + { + _lock.ExitReadLock(); + } + } +} diff --git a/src/Levels.Query/FileIndexEntry.cs b/src/Levels.Query/FileIndexEntry.cs new file mode 100644 index 0000000..a8a708f --- /dev/null +++ b/src/Levels.Query/FileIndexEntry.cs @@ -0,0 +1,13 @@ +using Levels.Core; +using Levels.Core.Format; + +namespace Levels.Query; + +public readonly record struct FileIndexEntry( + string FilePath, + PriceStreamId PriceStreamId, + string Venue, + FileType FileType, + long FirstObservedTime, + long LastObservedTime, + long RecordCount); diff --git a/src/Levels.Query/FileIndexSyncHandler.cs b/src/Levels.Query/FileIndexSyncHandler.cs new file mode 100644 index 0000000..1f0482a --- /dev/null +++ b/src/Levels.Query/FileIndexSyncHandler.cs @@ -0,0 +1,58 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.DataFlow; + +namespace Levels.Query; + +public sealed class FileIndexSyncHandler : IDataFlowHandler +{ + private readonly FileIndex _fileIndex; + private readonly string _dataPath; + + public FileIndexSyncHandler(FileIndex fileIndex, string dataPath) + { + _fileIndex = fileIndex; + _dataPath = dataPath; + } + + public ValueTask OnRecordWritten(PriceStreamId stream, RawRecord record, CancellationToken ct) + => ValueTask.CompletedTask; + + public ValueTask OnFileSealed(SealedFileInfo file, CancellationToken ct) + { + var entry = new FileIndexEntry( + file.FilePath, + file.PriceStreamId, + ExtractVenue(file.FilePath), + FileType.Raw, + file.FirstObservedTime, + file.LastObservedTime, + file.RecordCount); + + _fileIndex.Register(entry); + return ValueTask.CompletedTask; + } + + public ValueTask OnAggCreated(AggFileInfo file, CancellationToken ct) + { + var entry = new FileIndexEntry( + file.FilePath, + file.PriceStreamId, + ExtractVenue(file.FilePath), + FileType.Agg, + file.FirstObservedTime, + file.LastObservedTime, + file.RecordCount); + + _fileIndex.Register(entry); + return ValueTask.CompletedTask; + } + + private string ExtractVenue(string filePath) + { + var relativePath = Path.GetRelativePath(_dataPath, filePath); + var parts = relativePath.Split(Path.DirectorySeparatorChar); + return parts.Length >= 3 ? parts[0] : "unknown"; + } +} diff --git a/src/Levels.Query/Levels.Query.csproj b/src/Levels.Query/Levels.Query.csproj new file mode 100644 index 0000000..11fafa2 --- /dev/null +++ b/src/Levels.Query/Levels.Query.csproj @@ -0,0 +1,23 @@ + + + + net10.0 + enable + enable + preview + + + + + + + + + + + + + + + + diff --git a/src/Levels.Query/Log.cs b/src/Levels.Query/Log.cs new file mode 100644 index 0000000..a252b8f --- /dev/null +++ b/src/Levels.Query/Log.cs @@ -0,0 +1,9 @@ +using Microsoft.Extensions.Logging; + +namespace Levels.Query; + +internal static partial class Log +{ + [LoggerMessage(Level = LogLevel.Debug, Message = "Query executed: StreamId={StreamId}, FromNanos={FromNanos}, ToNanos={ToNanos}, ResultCount={ResultCount}")] + public static partial void QueryExecuted(ILogger logger, string streamId, long fromNanos, long toNanos, int resultCount); +} diff --git a/src/Levels.Query/OrderbookProjection.cs b/src/Levels.Query/OrderbookProjection.cs new file mode 100644 index 0000000..c26b589 --- /dev/null +++ b/src/Levels.Query/OrderbookProjection.cs @@ -0,0 +1,125 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Core.Orderbook; + +namespace Levels.Query; + +public sealed class OrderbookProjection +{ + private readonly QueryLayer _queryLayer; + + public OrderbookProjection(QueryLayer queryLayer) + { + _queryLayer = queryLayer; + } + + public L2Snapshot ProjectL2(PriceStreamId streamId, long fromNanos, long toNanos, bool excludeOwner = false) + { + var bids = new OrderbookSide(); + var asks = new OrderbookSide(); + ReplayRecords(streamId, fromNanos, toNanos, excludeOwner, bids, asks); + + return new L2Snapshot( + bids.Levels.Select(kv => new L2Level(kv.Key, kv.Value)).ToList(), + asks.Levels.Select(kv => new L2Level(kv.Key, kv.Value)).ToList()); + } + + public L1Snapshot ProjectL1(PriceStreamId streamId, long fromNanos, long toNanos, bool excludeOwner = false) + { + var l2 = ProjectL2(streamId, fromNanos, toNanos, excludeOwner); + var bestBid = l2.Bids.Count > 0 ? l2.Bids[^1] : (L2Level?)null; // highest bid + var bestAsk = l2.Asks.Count > 0 ? l2.Asks[0] : (L2Level?)null; // lowest ask + return new L1Snapshot(bestBid, bestAsk); + } + + public L3Snapshot ProjectL3(PriceStreamId streamId, long fromNanos, long toNanos, bool excludeOwner = false) + { + var bids = new L3OrderbookSide(); + var asks = new L3OrderbookSide(); + + var entries = _queryLayer.Resolve(streamId, fromNanos, toNanos); + int orderCounter = 0; + + foreach (var entry in entries) + { + using var fs = File.OpenRead(entry.FilePath); + var reader = new BinaryRecordReader(fs); + + foreach (var record in reader.ReadRecords()) + { + var core = record.Core; + if (core.ObservedTime > toNanos) break; + if (core.ObservedTime < fromNanos && core.Type != RecordType.Snap) continue; + + if (excludeOwner && (core.Flags & Constants.IsOwnerFlag) != 0) + continue; + + if (core.Type is not (RecordType.Snap or RecordType.Delta)) + continue; + + var orderId = TryExtractOrderId(record) ?? $"__synthetic_{orderCounter++}"; + var side = core.Side == RecordSide.Bid ? bids : asks; + + if (core.Quantity == 0) + side.Remove(orderId); + else + side.Apply(orderId, core.Price, core.Quantity); + } + } + + return new L3Snapshot(bids, asks); + } + + private void ReplayRecords(PriceStreamId streamId, long fromNanos, long toNanos, + bool excludeOwner, OrderbookSide bids, OrderbookSide asks) + { + var entries = _queryLayer.Resolve(streamId, fromNanos, toNanos); + + foreach (var entry in entries) + { + using var fs = File.OpenRead(entry.FilePath); + var reader = new BinaryRecordReader(fs); + + foreach (var record in reader.ReadRecords()) + { + var core = record.Core; + if (core.ObservedTime > toNanos) break; + if (core.ObservedTime < fromNanos && core.Type != RecordType.Snap) continue; + + if (excludeOwner && (core.Flags & Constants.IsOwnerFlag) != 0) + continue; + + if (core.Type is not (RecordType.Snap or RecordType.Delta)) + continue; + + var side = core.Side == RecordSide.Bid ? bids : asks; + side.Apply(core.Price, core.Quantity); + } + } + } + + private static string? TryExtractOrderId(RawRecord record) + { + var orderIdMem = record.OrderId; + if (orderIdMem.Length == 0) + return null; + + var span = orderIdMem.Span; + // Trim trailing null bytes + int len = span.Length; + while (len > 0 && span[len - 1] == 0) len--; + if (len == 0) + return null; + + return System.Text.Encoding.UTF8.GetString(span[..len]); + } +} + +public readonly record struct L2Level(long Price, long Quantity); + +public sealed record L2Snapshot(IReadOnlyList Bids, IReadOnlyList Asks); + +public sealed record L1Snapshot(L2Level? BestBid, L2Level? BestAsk); + +public sealed record L3Snapshot(L3OrderbookSide Bids, L3OrderbookSide Asks); diff --git a/src/Levels.Query/QueryConfig.cs b/src/Levels.Query/QueryConfig.cs new file mode 100644 index 0000000..e758870 --- /dev/null +++ b/src/Levels.Query/QueryConfig.cs @@ -0,0 +1,7 @@ +namespace Levels.Query; + +public sealed class QueryConfig +{ + public required string DataPath { get; init; } + public TimeSpan CompactionWindow { get; init; } = TimeSpan.FromHours(1); +} diff --git a/src/Levels.Query/QueryLayer.cs b/src/Levels.Query/QueryLayer.cs new file mode 100644 index 0000000..6028f90 --- /dev/null +++ b/src/Levels.Query/QueryLayer.cs @@ -0,0 +1,85 @@ +using System.Diagnostics; +using Levels.Core; +using Levels.Core.Diagnostics; +using Levels.Core.Format; + +namespace Levels.Query; + +public sealed class QueryLayer +{ + private readonly FileIndex _fileIndex; + private readonly QueryConfig _config; + + public QueryLayer(FileIndex fileIndex, QueryConfig config) + { + _fileIndex = fileIndex; + _config = config; + } + + public IReadOnlyList Resolve(PriceStreamId streamId, long fromNanos, long toNanos) + { + using var activity = StlthLevelsMetrics.ActivitySource.StartActivity("levels.query.resolve"); + var start = Stopwatch.GetTimestamp(); + + var entries = _fileIndex.Query(streamId, fromNanos, toNanos); + if (entries.Count == 0) + { + StlthLevelsMetrics.QueryLatency.Record(Stopwatch.GetElapsedTime(start).TotalMilliseconds); + StlthLevelsMetrics.QueriesExecuted.Add(1); + return []; + } + + var windowNanos = _config.CompactionWindow.Ticks * 100; + + // Bucket by compaction window, excluding Resampled files + var windows = new Dictionary>(); + foreach (var entry in entries) + { + if (entry.FileType == FileType.Resampled) + continue; + + var windowStart = entry.FirstObservedTime / windowNanos * windowNanos; + + if (!windows.TryGetValue(windowStart, out var list)) + { + list = new List(); + windows[windowStart] = list; + } + list.Add(entry); + } + + // Per window: select highest priority type (Period > Agg > Raw) + var result = new List(); + foreach (var (_, windowEntries) in windows) + { + var bestType = windowEntries.Max(e => e.FileType); + result.AddRange(windowEntries.Where(e => e.FileType == bestType)); + } + + result.Sort((a, b) => a.FirstObservedTime.CompareTo(b.FirstObservedTime)); + + StlthLevelsMetrics.QueryLatency.Record(Stopwatch.GetElapsedTime(start).TotalMilliseconds); + StlthLevelsMetrics.QueriesExecuted.Add(1); + + return result; + } + + public FileIndexEntry? ResolveSingle(PriceStreamId streamId, long observedTimeNanos) + { + var entries = Resolve(streamId, observedTimeNanos, observedTimeNanos + 1); + return entries.Count > 0 ? entries[0] : null; + } + + public IReadOnlyList ResolveResampled(PriceStreamId streamId, long fromNanos, long toNanos) + { + var start = Stopwatch.GetTimestamp(); + + var entries = _fileIndex.Query(streamId, fromNanos, toNanos, FileType.Resampled); + var result = entries.OrderBy(e => e.FirstObservedTime).ToList(); + + StlthLevelsMetrics.QueryLatency.Record(Stopwatch.GetElapsedTime(start).TotalMilliseconds); + StlthLevelsMetrics.QueriesExecuted.Add(1); + + return result; + } +} diff --git a/src/Levels.Resampled/BatchResampledProcessor.cs b/src/Levels.Resampled/BatchResampledProcessor.cs new file mode 100644 index 0000000..ee985e9 --- /dev/null +++ b/src/Levels.Resampled/BatchResampledProcessor.cs @@ -0,0 +1,140 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Query; + +namespace Levels.Resampled; + +public sealed class BatchResampledProcessor +{ + private readonly ResampledStreamConfig _config; + private readonly QueryLayer _queryLayer; + private readonly FileIndex _fileIndex; + + public BatchResampledProcessor(ResampledStreamConfig config, QueryLayer queryLayer, FileIndex fileIndex) + { + _config = config; + _queryLayer = queryLayer; + _fileIndex = fileIndex; + } + + public async Task ProcessAsync(long fromNanos, long toNanos, CancellationToken ct = default) + { + var windowNanos = _config.ResamplingWindow.Ticks * 100; + var configHash = _config.ComputeConfigHash(); + var outputStreamId = PriceStreamId.FromSymbol(_config.ConfigVersion); + + var emitter = _config.OutputType switch + { + ResampledOutputType.OhlcvBar => (IResampledEmitter)new OhlcvBarEmitter(), + _ => new TopOfBookEmitter(), + }; + + // Resolve source files per stream + var streamFiles = new Dictionary>(); + foreach (var sourceStream in _config.SourceStreams) + { + var files = _queryLayer.Resolve(sourceStream, fromNanos, toNanos); + if (files.Count > 0) + streamFiles[sourceStream] = files; + } + + if (streamFiles.Count == 0) + return; + + // Build per-stream record iterators + var streamRecords = new Dictionary>(); + foreach (var (streamId, files) in streamFiles) + { + var records = new List(); + foreach (var fileEntry in files) + { + using var fs = File.OpenRead(fileEntry.FilePath); + var reader = new BinaryRecordReader(fs); + foreach (var rec in reader.ReadRecords(validateCrc: true)) + { + if (rec.Core.ObservedTime >= fromNanos && rec.Core.ObservedTime < toNanos) + records.Add(rec); + } + } + records.Sort((a, b) => a.Core.ObservedTime.CompareTo(b.Core.ObservedTime)); + streamRecords[streamId] = records; + } + + // Create per-stream orderbook states + var states = new Dictionary(); + var cursors = new Dictionary(); + foreach (var streamId in _config.SourceStreams) + { + states[streamId] = new OrderbookState(); + cursors[streamId] = 0; + } + + // Compute output path + var utcDate = DateTimeOffset.FromUnixTimeMilliseconds(fromNanos / 1_000_000).UtcDateTime; + var windowSeconds = (int)_config.ResamplingWindow.TotalSeconds; + var dir = Path.Combine(_config.OutputPath, _config.Venue, + outputStreamId.Value.ToString(), "resampled"); + Directory.CreateDirectory(dir); + + var fileName = $"{_config.ConfigVersion}_{windowSeconds}s_{utcDate:yyyyMMdd}_batch.resampled"; + var outputPath = Path.Combine(dir, fileName); + + await using var fileStream = new FileStream( + outputPath, FileMode.Create, FileAccess.Write, FileShare.None, + bufferSize: 4096, FileOptions.WriteThrough | FileOptions.Asynchronous); + + await using var writer = await BinaryRecordWriter.CreateAsync( + fileStream, FileType.Resampled, outputStreamId, + _config.PriceScale, _config.QuantityScale, configHash); + + // Walk through windows + var windowStart = fromNanos / windowNanos * windowNanos; + while (windowStart < toNanos) + { + var windowEnd = windowStart + windowNanos; + + // Apply records in [windowStart, windowEnd) per stream + foreach (var (streamId, records) in streamRecords) + { + if (!cursors.TryGetValue(streamId, out var cursor)) + continue; + + while (cursor < records.Count && records[cursor].Core.ObservedTime < windowEnd) + { + if (records[cursor].Core.ObservedTime >= windowStart) + states[streamId].Apply(records[cursor]); + cursor++; + } + cursors[streamId] = cursor; + } + + // Emit at window boundary + var emitted = emitter.Emit(windowEnd, states).ToList(); + foreach (var record in emitted) + { + await writer.WriteRecordAsync(record.Core, record.OrderId); + } + + windowStart = windowEnd; + } + + await writer.SealAsync(); + fileStream.Dispose(); + + // Register in index + if (writer.RecordCount > 0) + { + var entry = new FileIndexEntry( + outputPath, + outputStreamId, + _config.Venue, + FileType.Resampled, + writer.FirstObservedTime, + writer.LastObservedTime, + writer.RecordCount); + + _fileIndex.Register(entry); + } + } +} diff --git a/src/Levels.Resampled/IResampledEmitter.cs b/src/Levels.Resampled/IResampledEmitter.cs new file mode 100644 index 0000000..77308e8 --- /dev/null +++ b/src/Levels.Resampled/IResampledEmitter.cs @@ -0,0 +1,11 @@ +using Levels.Core; +using Levels.Core.Format; + +namespace Levels.Resampled; + +public interface IResampledEmitter +{ + IEnumerable Emit( + long windowBoundaryNanos, + IReadOnlyDictionary states); +} diff --git a/src/Levels.Resampled/Levels.Resampled.csproj b/src/Levels.Resampled/Levels.Resampled.csproj new file mode 100644 index 0000000..5859284 --- /dev/null +++ b/src/Levels.Resampled/Levels.Resampled.csproj @@ -0,0 +1,24 @@ + + + + net10.0 + enable + enable + preview + + + + + + + + + + + + + + + + + diff --git a/src/Levels.Resampled/LiveResampledHandler.cs b/src/Levels.Resampled/LiveResampledHandler.cs new file mode 100644 index 0000000..bb186ce --- /dev/null +++ b/src/Levels.Resampled/LiveResampledHandler.cs @@ -0,0 +1,210 @@ +using Microsoft.Extensions.Hosting; +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.DataFlow; +using Levels.Query; + +namespace Levels.Resampled; + +public sealed class LiveResampledHandler : IDataFlowHandler, IHostedService, IAsyncDisposable +{ + private readonly ResampledStreamConfig _config; + private readonly FileIndex _fileIndex; + private readonly Dictionary _states = new(); + private readonly HashSet _sourceStreamSet; + private readonly IResampledEmitter _emitter; + private readonly PriceStreamId _outputStreamId; + private readonly uint _configHash; + private readonly object _lock = new(); + + private FileStream? _fileStream; + private BinaryRecordWriter? _writer; + private string? _currentFilePath; + private DateTime _currentFileDate; + private int _fileSequence; + private Task? _timerTask; + private CancellationTokenSource? _cts; + + public LiveResampledHandler(ResampledStreamConfig config, FileIndex fileIndex) + { + _config = config; + _fileIndex = fileIndex; + _sourceStreamSet = new HashSet(config.SourceStreams); + _outputStreamId = PriceStreamId.FromSymbol(config.ConfigVersion); + _configHash = config.ComputeConfigHash(); + + foreach (var stream in config.SourceStreams) + _states[stream] = new OrderbookState(); + + _emitter = config.OutputType switch + { + ResampledOutputType.OhlcvBar => new OhlcvBarEmitter(), + _ => new TopOfBookEmitter(), + }; + } + + public ValueTask OnRecordWritten(PriceStreamId stream, RawRecord record, CancellationToken ct) + { + if (!_sourceStreamSet.Contains(stream)) + return ValueTask.CompletedTask; + + lock (_lock) + { + if (_states.TryGetValue(stream, out var state)) + state.Apply(record); + } + + return ValueTask.CompletedTask; + } + + public ValueTask OnFileSealed(SealedFileInfo file, CancellationToken ct) => ValueTask.CompletedTask; + public ValueTask OnAggCreated(AggFileInfo file, CancellationToken ct) => ValueTask.CompletedTask; + + public Task StartAsync(CancellationToken cancellationToken) + { + _cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + _timerTask = RunTimerAsync(_cts.Token); + return Task.CompletedTask; + } + + private async Task RunTimerAsync(CancellationToken ct) + { + // Align to next window boundary + var windowMs = (long)_config.ResamplingWindow.TotalMilliseconds; + var nowMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var nextBoundary = (nowMs / windowMs + 1) * windowMs; + var initialDelay = (int)(nextBoundary - nowMs); + + if (initialDelay > 0) + await Task.Delay(initialDelay, ct); + + while (!ct.IsCancellationRequested) + { + try + { + var boundaryNanos = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() * 1_000_000L; + await EmitWindowAsync(boundaryNanos, ct); + await Task.Delay(_config.ResamplingWindow, ct); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + break; + } + } + } + + internal async Task EmitWindowAsync(long windowBoundaryNanos, CancellationToken ct) + { + List records; + + lock (_lock) + { + records = _emitter.Emit(windowBoundaryNanos, _states).ToList(); + } + + if (records.Count == 0) + return; + + // Check for UTC day rollover + var utcDate = DateTimeOffset.FromUnixTimeMilliseconds(windowBoundaryNanos / 1_000_000).UtcDateTime.Date; + if (_writer is not null && utcDate != _currentFileDate) + { + await SealCurrentFileAsync(); + } + + if (_writer is null) + { + await OpenNewFileAsync(utcDate); + } + + foreach (var record in records) + { + await _writer!.WriteRecordAsync(record.Core, record.OrderId); + } + } + + private async Task OpenNewFileAsync(DateTime utcDate) + { + var windowSeconds = (int)_config.ResamplingWindow.TotalSeconds; + var dir = Path.Combine(_config.OutputPath, _config.Venue, + _outputStreamId.Value.ToString(), "resampled"); + Directory.CreateDirectory(dir); + + var fileName = $"{_config.ConfigVersion}_{windowSeconds}s_{utcDate:yyyyMMdd}_{_fileSequence:D6}.resampled"; + _fileSequence++; + _currentFilePath = Path.Combine(dir, fileName); + _currentFileDate = utcDate; + + _fileStream = new FileStream( + _currentFilePath, + FileMode.CreateNew, + FileAccess.Write, + FileShare.None, + bufferSize: 4096, + FileOptions.WriteThrough | FileOptions.Asynchronous); + + _writer = await BinaryRecordWriter.CreateAsync( + _fileStream, + FileType.Resampled, + _outputStreamId, + _config.PriceScale, + _config.QuantityScale, + _configHash); + } + + private async Task SealCurrentFileAsync() + { + if (_writer is null) return; + + var writer = _writer; + var stream = _fileStream; + var filePath = _currentFilePath; + + _writer = null; + _fileStream = null; + _currentFilePath = null; + + await writer.SealAsync(); + await writer.DisposeAsync(); + stream!.Dispose(); + + if (filePath is not null) + { + var entry = new FileIndexEntry( + filePath, + _outputStreamId, + _config.Venue, + FileType.Resampled, + writer.FirstObservedTime, + writer.LastObservedTime, + writer.RecordCount); + + _fileIndex.Register(entry); + } + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + if (_cts is not null) + { + await _cts.CancelAsync(); + + if (_timerTask is not null) + { + try { await _timerTask; } + catch (OperationCanceledException) { } + } + + _cts.Dispose(); + _cts = null; + } + + await SealCurrentFileAsync(); + } + + public async ValueTask DisposeAsync() + { + await StopAsync(CancellationToken.None); + } +} diff --git a/src/Levels.Resampled/Log.cs b/src/Levels.Resampled/Log.cs new file mode 100644 index 0000000..c5b6e98 --- /dev/null +++ b/src/Levels.Resampled/Log.cs @@ -0,0 +1,15 @@ +using Microsoft.Extensions.Logging; + +namespace Levels.Resampled; + +internal static partial class Log +{ + [LoggerMessage(Level = LogLevel.Debug, Message = "Window emission: WindowBoundary={WindowBoundary}, RecordCount={RecordCount}")] + public static partial void WindowEmission(ILogger logger, string windowBoundary, int recordCount); + + [LoggerMessage(Level = LogLevel.Information, Message = "Resampled file rollover: FilePath={FilePath}")] + public static partial void ResampledFileRollover(ILogger logger, string filePath); + + [LoggerMessage(Level = LogLevel.Information, Message = "Resampled file sealed: FilePath={FilePath}, RecordCount={RecordCount}")] + public static partial void ResampledFileSeal(ILogger logger, string filePath, long recordCount); +} diff --git a/src/Levels.Resampled/OhlcvBarEmitter.cs b/src/Levels.Resampled/OhlcvBarEmitter.cs new file mode 100644 index 0000000..0c95a97 --- /dev/null +++ b/src/Levels.Resampled/OhlcvBarEmitter.cs @@ -0,0 +1,82 @@ +using Levels.Core; +using Levels.Core.Format; + +namespace Levels.Resampled; + +public sealed class OhlcvBarEmitter : IResampledEmitter +{ + public IEnumerable Emit( + long windowBoundaryNanos, + IReadOnlyDictionary states) + { + uint seq = 0; + foreach (var (streamId, state) in states) + { + if (!state.HasOhlcvData) + continue; + + // Encode OHLCV as 4 SNAP records: Open, High, Low, Close + yield return new RawRecord + { + Core = new CoreRecordLayout + { + ObservedTime = windowBoundaryNanos, + PriceStreamId = streamId.Value, + Price = state.WindowOpen!.Value, + Quantity = 0, + Type = RecordType.Snap, + Side = RecordSide.Bid, // Open + Sequence = seq++, + Level = 0, + }, + }; + + yield return new RawRecord + { + Core = new CoreRecordLayout + { + ObservedTime = windowBoundaryNanos, + PriceStreamId = streamId.Value, + Price = state.WindowHigh, + Quantity = 0, + Type = RecordType.Snap, + Side = RecordSide.Ask, // High + Sequence = seq++, + Level = 1, + }, + }; + + yield return new RawRecord + { + Core = new CoreRecordLayout + { + ObservedTime = windowBoundaryNanos, + PriceStreamId = streamId.Value, + Price = state.WindowLow, + Quantity = 0, + Type = RecordType.Snap, + Side = RecordSide.Bid, // Low + Sequence = seq++, + Level = 2, + }, + }; + + yield return new RawRecord + { + Core = new CoreRecordLayout + { + ObservedTime = windowBoundaryNanos, + PriceStreamId = streamId.Value, + Price = state.WindowClose, + Quantity = state.WindowVolume, + Type = RecordType.Snap, + Side = RecordSide.Ask, // Close + Volume + Sequence = seq++, + Level = 3, + }, + }; + + state.ResetWindow(); + } + } +} diff --git a/src/Levels.Resampled/OrderbookState.cs b/src/Levels.Resampled/OrderbookState.cs new file mode 100644 index 0000000..87bd4dc --- /dev/null +++ b/src/Levels.Resampled/OrderbookState.cs @@ -0,0 +1,92 @@ +using Levels.Core.Format; +using Levels.Core.Orderbook; + +namespace Levels.Resampled; + +public sealed class OrderbookState +{ + private readonly OrderbookSide _bids = new(); + private readonly OrderbookSide _asks = new(); + + // OHLCV tracking + private long? _windowOpen; + private long _windowHigh = long.MinValue; + private long _windowLow = long.MaxValue; + private long _windowClose; + private long _windowVolume; + private bool _hasOhlcvData; + + public (long Price, long Quantity)? BestBid + { + get + { + KeyValuePair? best = null; + foreach (var level in _bids.Levels) + best = level; // SortedDictionary iterates ascending; last = highest bid + return best.HasValue ? (best.Value.Key, best.Value.Value) : null; + } + } + + public (long Price, long Quantity)? BestAsk + { + get + { + foreach (var level in _asks.Levels) + return (level.Key, level.Value); // First = lowest ask + return null; + } + } + + public long? WindowOpen => _windowOpen; + public long WindowHigh => _windowHigh; + public long WindowLow => _windowLow; + public long WindowClose => _windowClose; + public long WindowVolume => _windowVolume; + public bool HasOhlcvData => _hasOhlcvData; + + public void Apply(RawRecord record) + { + var core = record.Core; + if (core.Type is not (RecordType.Snap or RecordType.Delta)) + return; + + var side = core.Side == RecordSide.Bid ? _bids : _asks; + side.Apply(core.Price, core.Quantity); + + // Update OHLCV from mid-price when both sides have data + var bid = BestBid; + var ask = BestAsk; + if (bid.HasValue && ask.HasValue) + { + var mid = (bid.Value.Price + ask.Value.Price) / 2; + if (_windowOpen is null) + _windowOpen = mid; + + if (mid > _windowHigh) _windowHigh = mid; + if (mid < _windowLow) _windowLow = mid; + _windowClose = mid; + _hasOhlcvData = true; + } + + // Track volume from quantity changes + if (core.Type == RecordType.Delta) + _windowVolume += Math.Abs(core.Quantity); + } + + public void ResetWindow() + { + _windowOpen = null; + _windowHigh = long.MinValue; + _windowLow = long.MaxValue; + _windowClose = 0; + _windowVolume = 0; + _hasOhlcvData = false; + } + + public void Clear() + { + _bids.Clear(); + _asks.Clear(); + ResetWindow(); + } +} diff --git a/src/Levels.Resampled/ResampledOutputType.cs b/src/Levels.Resampled/ResampledOutputType.cs new file mode 100644 index 0000000..3a59f75 --- /dev/null +++ b/src/Levels.Resampled/ResampledOutputType.cs @@ -0,0 +1,7 @@ +namespace Levels.Resampled; + +public enum ResampledOutputType : byte +{ + TopOfBook = 0, + OhlcvBar = 1, +} diff --git a/src/Levels.Resampled/ResampledStreamConfig.cs b/src/Levels.Resampled/ResampledStreamConfig.cs new file mode 100644 index 0000000..552c9c0 --- /dev/null +++ b/src/Levels.Resampled/ResampledStreamConfig.cs @@ -0,0 +1,23 @@ +using System.IO.Hashing; +using System.Text; +using Levels.Core; + +namespace Levels.Resampled; + +public sealed class ResampledStreamConfig +{ + public required string ConfigVersion { get; init; } + public required IReadOnlyList SourceStreams { get; init; } + public required TimeSpan ResamplingWindow { get; init; } + public ResampledOutputType OutputType { get; init; } = ResampledOutputType.TopOfBook; + public required string Venue { get; init; } + public required string OutputPath { get; init; } + public int PriceScale { get; init; } + public int QuantityScale { get; init; } + + public uint ComputeConfigHash() + { + var bytes = Encoding.UTF8.GetBytes(ConfigVersion); + return XxHash32.HashToUInt32(bytes); + } +} diff --git a/src/Levels.Resampled/TopOfBookEmitter.cs b/src/Levels.Resampled/TopOfBookEmitter.cs new file mode 100644 index 0000000..edd9272 --- /dev/null +++ b/src/Levels.Resampled/TopOfBookEmitter.cs @@ -0,0 +1,52 @@ +using Levels.Core; +using Levels.Core.Format; + +namespace Levels.Resampled; + +public sealed class TopOfBookEmitter : IResampledEmitter +{ + public IEnumerable Emit( + long windowBoundaryNanos, + IReadOnlyDictionary states) + { + uint seq = 0; + foreach (var (streamId, state) in states) + { + var bid = state.BestBid; + if (bid.HasValue) + { + yield return new RawRecord + { + Core = new CoreRecordLayout + { + ObservedTime = windowBoundaryNanos, + PriceStreamId = streamId.Value, + Price = bid.Value.Price, + Quantity = bid.Value.Quantity, + Type = RecordType.Snap, + Side = RecordSide.Bid, + Sequence = seq++, + }, + }; + } + + var ask = state.BestAsk; + if (ask.HasValue) + { + yield return new RawRecord + { + Core = new CoreRecordLayout + { + ObservedTime = windowBoundaryNanos, + PriceStreamId = streamId.Value, + Price = ask.Value.Price, + Quantity = ask.Value.Quantity, + Type = RecordType.Snap, + Side = RecordSide.Ask, + Sequence = seq++, + }, + }; + } + } + } +} diff --git a/src/Levels.Server/ConfigLoader.cs b/src/Levels.Server/ConfigLoader.cs new file mode 100644 index 0000000..04bc86d --- /dev/null +++ b/src/Levels.Server/ConfigLoader.cs @@ -0,0 +1,39 @@ +using System.Text.Json; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace Levels.Server; + +public static class ConfigLoader +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + }; + + public static ServerConfig Load(string path) + { + var text = File.ReadAllText(path); + var ext = Path.GetExtension(path).ToLowerInvariant(); + + return ext switch + { + ".json" => JsonSerializer.Deserialize(text, JsonOptions) + ?? throw new InvalidOperationException($"Failed to deserialize config from {path}"), + ".yaml" or ".yml" => LoadYaml(text), + _ => throw new InvalidOperationException($"Unsupported config file extension: {ext}. Use .json, .yaml, or .yml"), + }; + } + + private static ServerConfig LoadYaml(string text) + { + var deserializer = new DeserializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .IgnoreUnmatchedProperties() + .Build(); + + return deserializer.Deserialize(text) + ?? throw new InvalidOperationException("Failed to deserialize YAML config"); + } +} diff --git a/src/Levels.Server/Levels.Server.csproj b/src/Levels.Server/Levels.Server.csproj new file mode 100644 index 0000000..abc77d2 --- /dev/null +++ b/src/Levels.Server/Levels.Server.csproj @@ -0,0 +1,26 @@ + + + + Exe + net10.0 + enable + enable + preview + levels-server + false + + + + + + + + + + + + + + + + diff --git a/src/Levels.Server/Program.cs b/src/Levels.Server/Program.cs new file mode 100644 index 0000000..25b37c4 --- /dev/null +++ b/src/Levels.Server/Program.cs @@ -0,0 +1,70 @@ +using Levels.Export; +using Levels.Hosting; +using Levels.Query; +using Levels.Server; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +// Resolve config path +var configPath = args.FirstOrDefault(a => a.StartsWith("--config="))?.Split('=', 2)[1] + ?? Environment.GetEnvironmentVariable("LEVELS_CONFIG") + ?? "levels-server.yaml"; + +if (!File.Exists(configPath)) +{ + Console.Error.WriteLine($"Config file not found: {configPath}"); + Console.Error.WriteLine("Usage: levels-server --config="); + return 1; +} + +var serverConfig = ConfigLoader.Load(configPath); + +// Load schema plugin +var schema = SchemaPluginLoader.Load(serverConfig.Schema); +Console.WriteLine($"Loaded schema: {schema.TypeName} (SchemaId=0x{schema.SchemaId:X8}, RecordSize={schema.RecordSize})"); + +var builder = Host.CreateApplicationBuilder(args); + +// Register schema metadata and config +builder.Services.AddSingleton(schema); +builder.Services.AddSingleton(serverConfig); + +// Register Levels infrastructure using the non-generic overload +builder.Services.AddLevels(opts => +{ + opts.DataPath = serverConfig.DataPath; + opts.SchemaId = schema.SchemaId; + opts.RecordSize = schema.RecordSize; + opts.PriceScale = serverConfig.Levels.PriceScale; + opts.QuantityScale = serverConfig.Levels.QuantityScale; + opts.RolloverSize = serverConfig.Levels.RolloverSize; + opts.BackpressureLimit = serverConfig.Levels.BackpressureLimit; + opts.EnableCompaction = serverConfig.Levels.EnableCompaction; + opts.EnablePeriodPromotion = serverConfig.Levels.EnablePeriodPromotion; + + if (TimeSpan.TryParse(serverConfig.Levels.RolloverInterval, out var rolloverInterval)) + opts.RolloverInterval = rolloverInterval; + if (TimeSpan.TryParse(serverConfig.Levels.CompactionWindow, out var compactionWindow)) + opts.CompactionWindow = compactionWindow; + + opts.EnableMetrics = serverConfig.Telemetry.EnableMetrics; + opts.OtlpEndpoint = serverConfig.Telemetry.OtlpEndpoint; + opts.EnableTracing = serverConfig.Telemetry.EnableTracing; +}); + +// Export infrastructure +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(sp => + new ExportPipeline( + sp.GetRequiredService(), + sp.GetServices())); + +// TCP server +builder.Services.AddHostedService(); + +Console.WriteLine($"Levels server starting on port {serverConfig.Port}"); +var host = builder.Build(); +await host.RunAsync(); +return 0; diff --git a/src/Levels.Server/SchemaPluginLoader.cs b/src/Levels.Server/SchemaPluginLoader.cs new file mode 100644 index 0000000..72a9dd0 --- /dev/null +++ b/src/Levels.Server/SchemaPluginLoader.cs @@ -0,0 +1,42 @@ +using System.Reflection; +using Levels.Core; + +namespace Levels.Server; + +public sealed record SchemaMetadata(uint SchemaId, int RecordSize, string TypeName); + +public static class SchemaPluginLoader +{ + public static SchemaMetadata Load(SchemaConfig config) + { + if (string.IsNullOrEmpty(config.DllPath) || string.IsNullOrEmpty(config.TypeName)) + throw new InvalidOperationException("Schema DllPath and TypeName must be specified in config."); + + var fullPath = Path.GetFullPath(config.DllPath); + if (!File.Exists(fullPath)) + throw new FileNotFoundException($"Schema DLL not found: {fullPath}"); + + var assembly = Assembly.LoadFrom(fullPath); + var type = assembly.GetType(config.TypeName) + ?? throw new InvalidOperationException( + $"Type '{config.TypeName}' not found in '{fullPath}'"); + + if (!typeof(ISchemaDescriptor).IsAssignableFrom(type)) + throw new InvalidOperationException( + $"Type '{config.TypeName}' does not implement ISchemaDescriptor"); + + var schemaIdProp = type.GetProperty("SchemaId", + BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy); + var recordSizeProp = type.GetProperty("RecordSize", + BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy); + + if (schemaIdProp is null || recordSizeProp is null) + throw new InvalidOperationException( + $"Type '{config.TypeName}' does not expose static SchemaId and RecordSize properties"); + + var schemaId = (uint)schemaIdProp.GetValue(null)!; + var recordSize = (int)recordSizeProp.GetValue(null)!; + + return new SchemaMetadata(schemaId, recordSize, config.TypeName); + } +} diff --git a/src/Levels.Server/ServerConfig.cs b/src/Levels.Server/ServerConfig.cs new file mode 100644 index 0000000..a72c20a --- /dev/null +++ b/src/Levels.Server/ServerConfig.cs @@ -0,0 +1,35 @@ +namespace Levels.Server; + +public sealed class ServerConfig +{ + public string DataPath { get; set; } = "./data"; + public int Port { get; set; } = 5050; + public SchemaConfig Schema { get; set; } = new(); + public LevelsConfig Levels { get; set; } = new(); + public TelemetryConfig Telemetry { get; set; } = new(); +} + +public sealed class SchemaConfig +{ + public string DllPath { get; set; } = ""; + public string TypeName { get; set; } = ""; +} + +public sealed class LevelsConfig +{ + public int PriceScale { get; set; } + public int QuantityScale { get; set; } + public long RolloverSize { get; set; } = 256 * 1024 * 1024; + public string RolloverInterval { get; set; } = "01:00:00"; + public int BackpressureLimit { get; set; } = 8192; + public bool EnableCompaction { get; set; } = true; + public bool EnablePeriodPromotion { get; set; } + public string CompactionWindow { get; set; } = "01:00:00"; +} + +public sealed class TelemetryConfig +{ + public bool EnableMetrics { get; set; } + public string? OtlpEndpoint { get; set; } + public bool EnableTracing { get; set; } +} diff --git a/src/Levels.Server/TcpServer.cs b/src/Levels.Server/TcpServer.cs new file mode 100644 index 0000000..f88e5b2 --- /dev/null +++ b/src/Levels.Server/TcpServer.cs @@ -0,0 +1,331 @@ +using System.Diagnostics; +using System.Net; +using System.Net.Sockets; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Levels.Core; +using Levels.Core.Diagnostics; +using Levels.Core.Format; +using Levels.Export; +using Levels.Protocol; +using Levels.Query; +using Levels.Sinks; + +namespace Levels.Server; + +public sealed class TcpServer : BackgroundService +{ + private readonly SchemaMetadata _schema; + private readonly PriceStreamSink _sink; + private readonly FileIndex _fileIndex; + private readonly QueryLayer _queryLayer; + private readonly ExportPipeline _exportPipeline; + private readonly int _port; + private readonly ILogger _logger; + + private const int AckIntervalRecords = 1000; + private const long AckIntervalMs = 100; + + public TcpServer( + SchemaMetadata schema, + PriceStreamSink sink, + FileIndex fileIndex, + QueryLayer queryLayer, + ExportPipeline exportPipeline, + ServerConfig config, + ILogger logger) + { + _schema = schema; + _sink = sink; + _fileIndex = fileIndex; + _queryLayer = queryLayer; + _exportPipeline = exportPipeline; + _port = config.Port; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + var listener = new TcpListener(IPAddress.Any, _port); + listener.Start(); + _logger.LogInformation("TCP server listening on port {Port}", _port); + + stoppingToken.Register(() => listener.Stop()); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + var client = await listener.AcceptTcpClientAsync(stoppingToken); + _ = HandleClientAsync(client, stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (ObjectDisposedException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error accepting TCP connection"); + } + } + } + + private async Task HandleClientAsync(TcpClient client, CancellationToken ct) + { + var endpoint = client.Client.RemoteEndPoint; + _logger.LogInformation("Client connected: {Endpoint}", endpoint); + StlthLevelsMetrics.ActiveConnections.Add(1); + using var activity = StlthLevelsMetrics.ActivitySource.StartActivity("levels.tcp.handle_client"); + activity?.SetTag("endpoint", endpoint?.ToString()); + + try + { + await using var stream = client.GetStream(); + var reader = new FrameReader(stream); + var writer = new FrameWriter(stream); + + // Expect handshake first + var handshakeFrame = await reader.ReadFrameAsync(ct); + if (handshakeFrame is null || handshakeFrame.Value.Type != MessageType.Handshake) + { + _logger.LogWarning("Client {Endpoint} did not send handshake", endpoint); + return; + } + + var handshake = handshakeFrame.Value.ReadHandshake(); + if (handshake.SchemaId != _schema.SchemaId) + { + await writer.WriteHandshakeAckAsync(HandshakeStatus.SchemaMismatch, + $"Expected schema 0x{_schema.SchemaId:X8}, got 0x{handshake.SchemaId:X8}", ct); + await writer.FlushAsync(ct); + return; + } + if (handshake.RecordSize != _schema.RecordSize) + { + await writer.WriteHandshakeAckAsync(HandshakeStatus.SchemaMismatch, + $"Expected record size {_schema.RecordSize}, got {handshake.RecordSize}", ct); + await writer.FlushAsync(ct); + return; + } + + await writer.WriteHandshakeAckAsync(HandshakeStatus.Ok, ct: ct); + await writer.FlushAsync(ct); + + _logger.LogInformation("Client {Endpoint} handshake OK (schema=0x{SchemaId:X8})", endpoint, handshake.SchemaId); + + // Process messages + long accepted = 0; + long rejected = 0; + var lastAckTime = Environment.TickCount64; + + while (!ct.IsCancellationRequested) + { + var frame = await reader.ReadFrameAsync(ct); + if (frame is null) + break; + + StlthLevelsMetrics.TcpMessages.Add(1); + + switch (frame.Value.Type) + { + case MessageType.WriteRecord: + try + { + var record = frame.Value.ReadWriteRecord(); + var evt = ConvertToEvent(record); + await _sink.WriteAsync(evt, ct); + accepted++; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to write record from {Endpoint}", endpoint); + rejected++; + } + + var now = Environment.TickCount64; + if (accepted % AckIntervalRecords == 0 || now - lastAckTime > AckIntervalMs) + { + await writer.WriteAckAsync(accepted, rejected, ct); + await writer.FlushAsync(ct); + lastAckTime = now; + } + break; + + case MessageType.GetStreams: + await HandleGetStreams(writer, ct); + break; + + case MessageType.GetBook: + await HandleGetBook(frame.Value, writer, ct); + break; + + case MessageType.GetBookL1: + await HandleGetBookL1(frame.Value, writer, ct); + break; + + case MessageType.Export: + await HandleExport(frame.Value, writer, ct); + break; + + case MessageType.GetHealth: + await writer.WriteGetHealthResponseAsync("healthy", DateTime.UtcNow.ToString("O"), ct); + await writer.FlushAsync(ct); + break; + + case MessageType.GetSchemaInfo: + await writer.WriteGetSchemaInfoResponseAsync(_schema.SchemaId, _schema.RecordSize, _schema.TypeName, ct); + await writer.FlushAsync(ct); + break; + + default: + await writer.WriteErrorResponseAsync($"Unknown message type: {frame.Value.Type}", ct); + await writer.FlushAsync(ct); + break; + } + } + + // Final ack for any remaining writes + if (accepted > 0 || rejected > 0) + { + await writer.WriteAckAsync(accepted, rejected, ct); + await writer.FlushAsync(ct); + } + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) { } + catch (Exception ex) + { + _logger.LogError(ex, "Error handling client {Endpoint}", endpoint); + } + finally + { + StlthLevelsMetrics.ActiveConnections.Add(-1); + client.Dispose(); + _logger.LogInformation("Client disconnected: {Endpoint}", endpoint); + } + } + + private RawMarketEvent ConvertToEvent(WriteRecordMessage record) + { + var span = record.RecordBytes.Span; + if (span.Length != _schema.RecordSize) + throw new ProtocolException($"Record size {span.Length} does not match expected {_schema.RecordSize}"); + + var observedTime = BitConverter.ToInt64(span[..8]); + var price = BitConverter.ToInt64(span[24..32]); + var quantity = BitConverter.ToInt64(span[32..40]); + var type = (RecordType)span[42]; + var side = (RecordSide)span[43]; + var flags = BitConverter.ToUInt16(span[50..52]); + + ReadOnlyMemory extensionBytes = default; + if (record.RecordBytes.Length > Constants.CoreRecordSize) + extensionBytes = record.RecordBytes[Constants.CoreRecordSize..]; + + return new RawMarketEvent( + record.Venue, + record.Symbol, + observedTime, + price, + quantity, + type, + side, + extensionBytes, + (flags & Constants.IsOwnerFlag) != 0); + } + + private async Task HandleGetStreams(FrameWriter writer, CancellationToken ct) + { + try + { + var streamIds = _fileIndex.GetAllStreams(); + var streams = streamIds.Select(id => (id.ToString(), "")).ToList(); + await writer.WriteGetStreamsResponseAsync(streams, ct); + } + catch (Exception ex) + { + await writer.WriteErrorResponseAsync(ex.Message, ct); + } + await writer.FlushAsync(ct); + } + + private async Task HandleGetBook(Frame frame, FrameWriter writer, CancellationToken ct) + { + try + { + var req = frame.ReadGetBookRequest(); + var streamId = PriceStreamId.FromVenueSymbol(req.Venue, req.Symbol); + var fromNanos = req.FromNanos == 0 ? 0 : req.FromNanos; + var toNanos = req.ToNanos == 0 ? long.MaxValue : req.ToNanos; + + var projection = new OrderbookProjection(_queryLayer); + var l2 = projection.ProjectL2(streamId, fromNanos, toNanos, req.ExcludeOwner); + + var bids = l2.Bids.Select(l => (l.Price, l.Quantity)).ToList(); + var asks = l2.Asks.Select(l => (l.Price, l.Quantity)).ToList(); + await writer.WriteGetBookResponseAsync(bids, asks, ct); + } + catch (Exception ex) + { + await writer.WriteErrorResponseAsync(ex.Message, ct); + } + await writer.FlushAsync(ct); + } + + private async Task HandleGetBookL1(Frame frame, FrameWriter writer, CancellationToken ct) + { + try + { + var req = frame.ReadGetBookRequest(); + var streamId = PriceStreamId.FromVenueSymbol(req.Venue, req.Symbol); + var fromNanos = req.FromNanos == 0 ? 0 : req.FromNanos; + var toNanos = req.ToNanos == 0 ? long.MaxValue : req.ToNanos; + + var projection = new OrderbookProjection(_queryLayer); + var l1 = projection.ProjectL1(streamId, fromNanos, toNanos, req.ExcludeOwner); + + (long, long)? bestBid = l1.BestBid is { } b ? (b.Price, b.Quantity) : null; + (long, long)? bestAsk = l1.BestAsk is { } a ? (a.Price, a.Quantity) : null; + await writer.WriteGetBookL1ResponseAsync(bestBid, bestAsk, ct); + } + catch (Exception ex) + { + await writer.WriteErrorResponseAsync(ex.Message, ct); + } + await writer.FlushAsync(ct); + } + + private async Task HandleExport(Frame frame, FrameWriter writer, CancellationToken ct) + { + try + { + var req = frame.ReadExportRequest(); + var streamId = PriceStreamId.FromVenueSymbol(req.Venue, req.Symbol); + var fromNanos = req.FromNanos == 0 ? 0 : req.FromNanos; + var toNanos = req.ToNanos == 0 ? long.MaxValue : req.ToNanos; + + using var ms = new MemoryStream(); + await _exportPipeline.ExportAsync(streamId, fromNanos, toNanos, req.Format, ms, ct); + + ms.Position = 0; + var buffer = new byte[64 * 1024]; + int bytesRead; + while ((bytesRead = await ms.ReadAsync(buffer, ct)) > 0) + { + await writer.WriteExportChunkAsync(buffer.AsMemory(0, bytesRead), ct); + await writer.FlushAsync(ct); + } + + // End marker: zero-length chunk + await writer.WriteExportChunkAsync(ReadOnlyMemory.Empty, ct); + } + catch (Exception ex) + { + await writer.WriteErrorResponseAsync(ex.Message, ct); + } + await writer.FlushAsync(ct); + } +} diff --git a/src/Levels.Sinks/Levels.Sinks.csproj b/src/Levels.Sinks/Levels.Sinks.csproj new file mode 100644 index 0000000..44edeff --- /dev/null +++ b/src/Levels.Sinks/Levels.Sinks.csproj @@ -0,0 +1,23 @@ + + + + net10.0 + enable + enable + preview + + + + + + + + + + + + + + + + diff --git a/src/Levels.Sinks/Log.cs b/src/Levels.Sinks/Log.cs new file mode 100644 index 0000000..da3f641 --- /dev/null +++ b/src/Levels.Sinks/Log.cs @@ -0,0 +1,24 @@ +using Microsoft.Extensions.Logging; + +namespace Levels.Sinks; + +internal static partial class Log +{ + [LoggerMessage(Level = LogLevel.Debug, Message = "Record written: StreamId={StreamId}, Price={Price}, Quantity={Quantity}")] + public static partial void RecordWritten(ILogger logger, string streamId, decimal price, decimal quantity); + + [LoggerMessage(Level = LogLevel.Information, Message = "File sealed: FilePath={FilePath}, RecordCount={RecordCount}")] + public static partial void FileSealedEvent(ILogger logger, string filePath, long recordCount); + + [LoggerMessage(Level = LogLevel.Information, Message = "File rollover: FilePath={FilePath}")] + public static partial void FileRollover(ILogger logger, string filePath); + + [LoggerMessage(Level = LogLevel.Error, Message = "Write failed for StreamId={StreamId}, consecutive failures={FailureCount}")] + public static partial void WriteFailed(ILogger logger, Exception exception, long streamId, int failureCount); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Circuit breaker tripped for StreamId={StreamId} after {FailureCount} failures, events will be dropped")] + public static partial void CircuitBreakerTripped(ILogger logger, long streamId, int failureCount); + + [LoggerMessage(Level = LogLevel.Information, Message = "Circuit breaker reset for StreamId={StreamId} after cooldown")] + public static partial void CircuitBreakerReset(ILogger logger, long streamId); +} diff --git a/src/Levels.Sinks/PriceStreamSink.cs b/src/Levels.Sinks/PriceStreamSink.cs new file mode 100644 index 0000000..5cc2a9c --- /dev/null +++ b/src/Levels.Sinks/PriceStreamSink.cs @@ -0,0 +1,342 @@ +using System.Collections.Concurrent; +using System.Runtime.InteropServices; +using System.Threading.Channels; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Levels.Core; +using Levels.Core.Diagnostics; +using Levels.Core.Format; +using Levels.Core.IO; + +namespace Levels.Sinks; + +public sealed class PriceStreamSink : IHostedService, IAsyncDisposable +{ + private readonly SinkConfig _config; + private readonly ILogger _logger; + private readonly SinkPartition[] _partitions; + private readonly ConcurrentDictionary<(string Venue, string Symbol), PriceStreamId> _symbolCache = new(); + + public PriceStreamSink(SinkConfig config, ILogger? logger = null) + { + _config = config; + _logger = logger ?? (ILogger)NullLogger.Instance; + + var partitionCount = Math.Max(1, config.ConsumerPartitions); + _partitions = new SinkPartition[partitionCount]; + for (var i = 0; i < partitionCount; i++) + { + _partitions[i] = new SinkPartition(i, config, _logger); + } + } + + /// + /// Routes a RawMarketEvent to the correct partition by venue+symbol hash. + /// + public ValueTask WriteAsync(RawMarketEvent evt, CancellationToken ct = default) + { + var streamId = _symbolCache.GetOrAdd((evt.Venue, evt.Symbol), static k => PriceStreamId.FromVenueSymbol(k.Venue, k.Symbol)); + var partition = _partitions[GetPartitionIndex(streamId)]; + return partition.Channel.Writer.WriteAsync(evt, ct); + } + + /// + /// Exposes the channel writer for the single-partition case (backwards compat). + /// For multi-partition, prefer WriteAsync. + /// + public ChannelWriter Writer + { + get + { + if (_partitions.Length == 1) + return _partitions[0].Channel.Writer; + + throw new InvalidOperationException( + "ChannelWriter is not available with multiple partitions. Use WriteAsync instead."); + } + } + + public Task StartAsync(CancellationToken cancellationToken) + { + foreach (var partition in _partitions) + { + partition.Start(cancellationToken); + } + return Task.CompletedTask; + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + foreach (var partition in _partitions) + { + partition.Channel.Writer.TryComplete(); + } + + foreach (var partition in _partitions) + { + await partition.WaitAsync(); + } + + foreach (var partition in _partitions) + { + await partition.DisposeAsync(); + } + } + + public async ValueTask DisposeAsync() + { + foreach (var partition in _partitions) + { + partition.Channel.Writer.TryComplete(); + } + + foreach (var partition in _partitions) + { + await partition.WaitAsync(); + } + + foreach (var partition in _partitions) + { + await partition.DisposeAsync(); + } + } + + private int GetPartitionIndex(PriceStreamId streamId) + { + // Unsigned modulo to avoid negative index from negative hash values + return (int)((ulong)streamId.Value % (ulong)_partitions.Length); + } + + private sealed class SinkPartition + { + private readonly int _partitionIndex; + private readonly SinkConfig _config; + private readonly ILogger _logger; + private readonly bool _batchingEnabled; + private readonly Dictionary _writers = new(); + private readonly Dictionary _failureCounts = new(); + private readonly Dictionary _trippedAt = new(); + private readonly ConcurrentDictionary<(string Venue, string Symbol), PriceStreamId> _symbolCache = new(); + private WriteAheadLog? _wal; + private Task? _consumeTask; + + public Channel Channel { get; } + + public SinkPartition(int partitionIndex, SinkConfig config, ILogger logger) + { + _partitionIndex = partitionIndex; + _config = config; + _logger = logger; + _batchingEnabled = config.FlushThresholdMs > 0 || config.FlushBufferSize > 0; + Channel = System.Threading.Channels.Channel.CreateBounded( + new BoundedChannelOptions(config.BackpressureLimit) + { + SingleReader = true, + FullMode = BoundedChannelFullMode.Wait, + }); + } + + public void Start(CancellationToken ct) + { + if (_batchingEnabled) + { + var walDir = Path.Combine(_config.OutputPath, ".wal"); + _wal = WriteAheadLog.Open(walDir, _partitionIndex); + } + + _consumeTask = ConsumeAsync(ct); + } + + public async Task WaitAsync() + { + if (_consumeTask is not null) + { + try { await _consumeTask; } + catch (OperationCanceledException) { } + } + } + + private async Task ConsumeAsync(CancellationToken ct) + { + // Replay WAL entries before consuming new events + if (_batchingEnabled) + { + await ReplayWalAsync(); + } + + await foreach (var evt in Channel.Reader.ReadAllAsync(ct)) + { + var streamId = _symbolCache.GetOrAdd((evt.Venue, evt.Symbol), static k => PriceStreamId.FromVenueSymbol(k.Venue, k.Symbol)); + + // Check circuit breaker + if (_trippedAt.TryGetValue(streamId, out var trippedTicks)) + { + var elapsed = Environment.TickCount64 - trippedTicks; + if (elapsed < _config.CircuitBreakerCooldown.TotalMilliseconds) + { + continue; // Drop event — circuit breaker still open + } + + // Cooldown expired — reset circuit breaker + _trippedAt.Remove(streamId); + _failureCounts.Remove(streamId); + Log.CircuitBreakerReset(_logger, streamId.Value); + } + + if (!_writers.TryGetValue(streamId, out var writer)) + { + writer = new PriceStreamWriter(streamId, evt.Venue, _config); + await writer.RecoverStateAsync(); + _writers[streamId] = writer; + StlthLevelsMetrics.ActiveStreams.Add(1); + } + + try + { + // WAL: append before data write for crash durability + if (_wal is not null) + { + var recordBytes = BuildRecordBytes(evt, streamId); + await _wal.AppendAsync(streamId.Value, evt.Venue, evt.Symbol, recordBytes); + } + + await writer.WriteAsync(evt, ct); + + // Reset failure count on success + _failureCounts.Remove(streamId); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; // Propagate cancellation + } + catch (Exception ex) + { + _failureCounts.TryGetValue(streamId, out var count); + count++; + _failureCounts[streamId] = count; + + Log.WriteFailed(_logger, ex, streamId.Value, count); + + if (count >= _config.MaxWriteFailures) + { + Log.CircuitBreakerTripped(_logger, streamId.Value, count); + _trippedAt[streamId] = Environment.TickCount64; + + // Best-effort dispose of the failed writer + _writers.Remove(streamId); + StlthLevelsMetrics.ActiveStreams.Add(-1); + StlthLevelsMetrics.CircuitBreakerTrips.Add(1); + try { await writer.DisposeAsync(); } + catch { /* best effort */ } + } + } + } + + // All events consumed — truncate WAL since data files will be sealed + _wal?.Truncate(); + } + + private async Task ReplayWalAsync() + { + var walDir = Path.Combine(_config.OutputPath, ".wal"); + var entries = WriteAheadLog.Replay(walDir, _partitionIndex); + if (entries.Count == 0) + return; + + foreach (var entry in entries) + { + var streamId = new PriceStreamId(entry.PriceStreamId); + + if (!_writers.TryGetValue(streamId, out var writer)) + { + writer = new PriceStreamWriter(streamId, entry.Venue, _config); + await writer.RecoverStateAsync(); + _writers[streamId] = writer; + _symbolCache.TryAdd((entry.Venue, entry.Symbol), streamId); + } + + // Records from WAL are already formatted — replay by re-writing + if (entry.RecordBytes.Length >= Constants.CoreRecordSize) + { + var core = MemoryMarshal.Read(entry.RecordBytes); + var extensionBytes = entry.RecordBytes.Length > Constants.CoreRecordSize + ? new ReadOnlyMemory(entry.RecordBytes, Constants.CoreRecordSize, + entry.RecordBytes.Length - Constants.CoreRecordSize) + : ReadOnlyMemory.Empty; + + var evt = new RawMarketEvent( + entry.Venue, + entry.Symbol, + core.ObservedTime, + core.Price, + core.Quantity, + core.Type, + core.Side, + extensionBytes, + (core.Flags & Constants.IsOwnerFlag) != 0); + + try + { + await writer.WriteAsync(evt, CancellationToken.None); + } + catch + { + // Best effort replay — data file recovery handles the rest + } + } + } + + // WAL entries replayed — truncate + _wal?.Truncate(); + } + + /// + /// Builds the raw record bytes for WAL persistence from a RawMarketEvent. + /// This creates the core layout + extension bytes that would be written to the data file. + /// + private byte[] BuildRecordBytes(RawMarketEvent evt, PriceStreamId streamId) + { + var recordBytes = new byte[_config.RecordSize]; + var core = new CoreRecordLayout + { + ObservedTime = evt.ObservedTime, + PriceStreamId = streamId.Value, + Price = evt.Price, + Quantity = evt.Quantity, + Type = evt.Type, + Side = evt.Side, + }; + MemoryMarshal.Write(recordBytes, in core); + + if (evt.ExtensionBytes.Length > 0 && _config.RecordSize > Constants.CoreRecordSize) + { + var dest = recordBytes.AsSpan(Constants.CoreRecordSize); + if (evt.ExtensionBytes.Length > dest.Length) + throw new ArgumentException( + $"Extension bytes ({evt.ExtensionBytes.Length}) exceed record extension space ({dest.Length})."); + evt.ExtensionBytes.Span.CopyTo(dest); + } + + return recordBytes; + } + + public async Task DisposeAsync() + { + StlthLevelsMetrics.ActiveStreams.Add(-_writers.Count); + foreach (var writer in _writers.Values) + { + await writer.DisposeAsync(); + } + _writers.Clear(); + _failureCounts.Clear(); + _trippedAt.Clear(); + + if (_wal is not null) + { + await _wal.DisposeAsync(); + _wal = null; + } + } + } +} diff --git a/src/Levels.Sinks/PriceStreamWriter.cs b/src/Levels.Sinks/PriceStreamWriter.cs new file mode 100644 index 0000000..2bceca6 --- /dev/null +++ b/src/Levels.Sinks/PriceStreamWriter.cs @@ -0,0 +1,360 @@ +using System.Diagnostics; +using Levels.Core; +using Levels.Core.Diagnostics; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Core.Orderbook; +using Levels.DataFlow; + +namespace Levels.Sinks; + +internal sealed class PriceStreamWriter : IAsyncDisposable +{ + private readonly PriceStreamId _priceStreamId; + private readonly string _venue; + private readonly SinkConfig _config; + private readonly OrderbookSide _bids; + private readonly OrderbookSide _asks; + + private FileStream? _fileStream; + private BinaryRecordWriter? _writer; + private string? _currentFilePath; + private long _bytesWritten; + private long _fileOpenedAt; + private uint _recordSequence; + private int _fileSequence; + + public PriceStreamWriter(PriceStreamId priceStreamId, string venue, SinkConfig config) + { + _priceStreamId = priceStreamId; + _venue = venue; + _config = config; + _bids = new OrderbookSide(RecordSide.Bid); + _asks = new OrderbookSide(RecordSide.Ask); + } + + public async ValueTask WriteAsync(RawMarketEvent evt, CancellationToken ct) + { + var start = Stopwatch.GetTimestamp(); + + // Open file if needed (leading SNAP uses current orderbook state, before this event) + if (_writer is null) + await OpenNewFileAsync(ct); + + // Check rollover before writing (leading SNAP includes this event's prior state) + if (ShouldRollover()) + { + await RolloverAsync(ct); + } + + var side_book = evt.Side == RecordSide.Bid ? _bids : _asks; + var level = ComputeLevel(side_book, evt.Price, evt.Side); + var flags = evt.IsOwner ? Constants.IsOwnerFlag : (ushort)0; + + var core = new CoreRecordLayout + { + ObservedTime = evt.ObservedTime, + PriceStreamId = _priceStreamId.Value, + Price = evt.Price, + Quantity = evt.Quantity, + Type = evt.Type, + Side = evt.Side, + Sequence = _recordSequence++, + Level = level, + Flags = flags, + }; + + var extensionBytes = evt.ExtensionBytes; + var finalCore = await _writer!.WriteRecordAsync(core, extensionBytes); + + // Build full record bytes for the published RawRecord + var recordBytes = new byte[_config.RecordSize]; + System.Runtime.InteropServices.MemoryMarshal.Write(recordBytes, in finalCore); + if (extensionBytes.Length > 0 && _config.RecordSize > Constants.CoreRecordSize) + { + var dest = recordBytes.AsSpan(Constants.CoreRecordSize); + extensionBytes.Span.CopyTo(dest); + } + + _config.DataFlowBus?.PublishRecordWritten(_priceStreamId, + new RawRecord { Core = finalCore, RecordBytes = recordBytes }); + + _bytesWritten += _config.RecordSize; + + StlthLevelsMetrics.RecordsWritten.Add(1); + StlthLevelsMetrics.RecordWriteLatency.Record(Stopwatch.GetElapsedTime(start).TotalMilliseconds); + + // Update orderbook state after writing (so leading SNAPs on rollover reflect state up to this point) + var side = evt.Side == RecordSide.Bid ? _bids : _asks; + if (evt.Type is RecordType.Snap or RecordType.Delta) + { + side.Apply(evt.Price, evt.Quantity); + } + } + + private async Task OpenNewFileAsync(CancellationToken ct) + { + var now = DateTime.UtcNow; + var dir = Path.Combine(_config.OutputPath, _venue, _priceStreamId.Value.ToString()); + Directory.CreateDirectory(dir); + + var fileName = $"{now:yyyyMMdd}_{_fileSequence:D6}.raw"; + _fileSequence++; + _currentFilePath = Path.Combine(dir, fileName); + + var fileOptions = (_config.FlushThresholdMs > 0 || _config.FlushBufferSize > 0) + ? FileOptions.Asynchronous + : FileOptions.WriteThrough | FileOptions.Asynchronous; + + _fileStream = new FileStream( + _currentFilePath, + FileMode.CreateNew, + FileAccess.Write, + FileShare.None, + bufferSize: 4096, + fileOptions); + + _writer = await BinaryRecordWriter.CreateAsync( + _fileStream, + FileType.Raw, + _priceStreamId, + _config.PriceScale, + _config.QuantityScale, + schemaId: _config.SchemaId, + recordSize: _config.RecordSize, + flushThresholdMs: _config.FlushThresholdMs, + flushBufferSize: _config.FlushBufferSize); + + _bytesWritten = Constants.HeaderSize; + _fileOpenedAt = WriteTimestamp.Now(); + _recordSequence = 0; + + await WriteLeadingSnapAsync(); + } + + private async Task WriteLeadingSnapAsync() + { + var now = WriteTimestamp.Now(); + + foreach (var level in _bids.Levels) + { + var core = new CoreRecordLayout + { + ObservedTime = now, + PriceStreamId = _priceStreamId.Value, + Price = level.Key, + Quantity = level.Value, + Type = RecordType.Snap, + Side = RecordSide.Bid, + Sequence = _recordSequence++, + }; + await _writer!.WriteRecordAsync(core); + _bytesWritten += _config.RecordSize; + } + + foreach (var level in _asks.Levels) + { + var core = new CoreRecordLayout + { + ObservedTime = now, + PriceStreamId = _priceStreamId.Value, + Price = level.Key, + Quantity = level.Value, + Type = RecordType.Snap, + Side = RecordSide.Ask, + Sequence = _recordSequence++, + }; + await _writer!.WriteRecordAsync(core); + _bytesWritten += _config.RecordSize; + } + } + + private bool ShouldRollover() + { + if (_writer is null) return false; + if (_bytesWritten >= _config.RolloverSize) return true; + + var elapsed = WriteTimestamp.Now() - _fileOpenedAt; + var intervalNanos = _config.RolloverInterval.Ticks * 100; + return elapsed >= intervalNanos; + } + + private async Task RolloverAsync(CancellationToken ct) + { + await SealCurrentFileAsync(); + await OpenNewFileAsync(ct); + } + + private async Task SealCurrentFileAsync() + { + if (_writer is null) return; + + var writer = _writer; + var stream = _fileStream; + var filePath = _currentFilePath; + + _writer = null; + _fileStream = null; + _currentFilePath = null; + + await writer.SealAsync(); + await writer.DisposeAsync(); + stream!.Dispose(); + + StlthLevelsMetrics.FilesSealed.Add(1); + + if (filePath is not null && _config.DataFlowBus is { } bus) + { + await bus.PublishFileSealedAsync(new SealedFileInfo( + filePath, + _priceStreamId, + writer.RecordCount, + writer.DeltaCount, + writer.FirstObservedTime, + writer.LastObservedTime)); + } + } + + private static ushort ComputeLevel(OrderbookSide side, long price, RecordSide recordSide) + { + ushort depth = 0; + if (recordSide == RecordSide.Bid) + { + // Bids sorted ascending; best bid is last. Count levels above this price. + foreach (var kv in side.Levels) + { + if (kv.Key > price) + depth++; + } + } + else + { + // Asks sorted ascending; best ask is first. Count levels below this price. + foreach (var kv in side.Levels) + { + if (kv.Key < price) + depth++; + } + } + + return depth; + } + + public async Task RecoverStateAsync() + { + var dir = Path.Combine(_config.OutputPath, _venue, _priceStreamId.Value.ToString()); + if (!Directory.Exists(dir)) + return; + + var rawFiles = Directory.GetFiles(dir, "*.raw").OrderBy(f => f).ToList(); + if (rawFiles.Count == 0) + return; + + // Find most recent file and replay its records to rebuild orderbook state + for (var i = rawFiles.Count - 1; i >= 0; i--) + { + var file = rawFiles[i]; + try + { + // Extract sequence number from filename (yyyyMMdd_NNNNNN.raw) + var fileName = Path.GetFileNameWithoutExtension(file); + var parts = fileName.Split('_'); + if (parts.Length >= 2 && int.TryParse(parts[1], out var seq)) + { + _fileSequence = Math.Max(_fileSequence, seq + 1); + } + + using var fs = File.OpenRead(file); + var reader = new BinaryRecordReader(fs); + + if (reader.IsPartial) + { + // For partial (unsealed) files, replay CRC-valid records then auto-seal + var (records, validDataEnd) = reader.ReadValidRecordsFromPartial(); + foreach (var record in records) + { + ReplayRecord(record.Core); + } + fs.Dispose(); + + // Auto-seal the partial file: truncate to valid data, write footer + await SealPartialFileAsync(file, records, validDataEnd, reader.Header); + continue; + } + + // Replay all records from the most recent sealed file + foreach (var record in reader.ReadRecords(validateCrc: true)) + { + ReplayRecord(record.Core); + } + + // Only need the most recent sealed file for orderbook state + break; + } + catch + { + // Skip corrupt files + } + } + } + + private static async Task SealPartialFileAsync( + string filePath, List records, long validDataEnd, FileHeader header) + { + if (records.Count == 0) + return; + + // Truncate file to valid data boundary and append footer + using var fs = new FileStream(filePath, FileMode.Open, FileAccess.Write, FileShare.None); + fs.SetLength(validDataEnd); + fs.Seek(0, SeekOrigin.End); + + // Compute file-level CRC from all valid records + var fileCrc = Crc32Util.CreateIncremental(); + var recordSize = header.RecordSize > 0 ? header.RecordSize : Constants.CoreRecordSize; + foreach (var record in records) + { + fileCrc.Append(record.RecordBytes.Span[..recordSize]); + } + + long deltaCount = records.Count(r => r.Core.Type == RecordType.Delta); + var firstRecord = records[0].Core; + var lastRecord = records[^1].Core; + + var footer = new FileFooter + { + RecordCount = records.Count, + DeltaCount = deltaCount, + FirstWriteTimestamp = firstRecord.WriteTimestamp, + LastWriteTimestamp = lastRecord.WriteTimestamp, + FirstObservedTime = records.Min(r => r.Core.ObservedTime), + LastObservedTime = records.Max(r => r.Core.ObservedTime), + FileCrc32 = fileCrc.GetCurrentHashAsUInt32(), + MagicEnd = Constants.FooterMagicUInt64, + }; + + Span tmp = stackalloc byte[Constants.FooterSize]; + tmp.Clear(); + FileFooter.WriteTo(tmp, in footer); + footer.FooterCrc32 = Crc32Util.Compute(tmp[..52]); + + var footerBytes = new byte[Constants.FooterSize]; + FileFooter.WriteTo(footerBytes, in footer); + await fs.WriteAsync(footerBytes); + fs.Flush(flushToDisk: true); + } + + private void ReplayRecord(CoreRecordLayout core) + { + if (core.Type is RecordType.Snap or RecordType.Delta) + { + var side = core.Side == RecordSide.Bid ? _bids : _asks; + side.Apply(core.Price, core.Quantity); + } + } + + public async ValueTask DisposeAsync() + { + await SealCurrentFileAsync(); + } +} diff --git a/src/Levels.Sinks/RandomDataAdapter.cs b/src/Levels.Sinks/RandomDataAdapter.cs new file mode 100644 index 0000000..ca6d8cf --- /dev/null +++ b/src/Levels.Sinks/RandomDataAdapter.cs @@ -0,0 +1,57 @@ +using System.Threading.Channels; +using Levels.Core.Format; +using Levels.Core.IO; + +namespace Levels.Sinks; + +public static class RandomDataAdapter +{ + public static async Task GenerateAsync( + ChannelWriter writer, + string venue, + string[] symbols, + int levelsPerSide = 5, + int deltasPerSymbol = 100, + int seed = 42) + { + var rng = new Random(seed); + var baseTime = WriteTimestamp.Now(); + + foreach (var symbol in symbols) + { + var basePrice = 10000L + rng.Next(0, 50000); + + // Write initial SNAP levels + for (var i = 0; i < levelsPerSide; i++) + { + var bidPrice = basePrice - (i + 1) * 10; + var askPrice = basePrice + (i + 1) * 10; + var qty = 100L + rng.Next(0, 900); + + await writer.WriteAsync(new RawMarketEvent( + venue, symbol, baseTime, bidPrice, qty, + RecordType.Snap, RecordSide.Bid)); + + await writer.WriteAsync(new RawMarketEvent( + venue, symbol, baseTime, askPrice, qty, + RecordType.Snap, RecordSide.Ask)); + } + + // Write DELTAs + for (var i = 0; i < deltasPerSymbol; i++) + { + baseTime += 1_000_000; // 1ms + var side = rng.Next(2) == 0 ? RecordSide.Bid : RecordSide.Ask; + var priceOffset = rng.Next(1, levelsPerSide + 1) * 10; + var price = side == RecordSide.Bid + ? basePrice - priceOffset + : basePrice + priceOffset; + var qty = rng.Next(0, 1000); + + await writer.WriteAsync(new RawMarketEvent( + venue, symbol, baseTime, price, qty, + RecordType.Delta, side)); + } + } + } +} diff --git a/src/Levels.Sinks/RawMarketEvent.cs b/src/Levels.Sinks/RawMarketEvent.cs new file mode 100644 index 0000000..c0ca8a9 --- /dev/null +++ b/src/Levels.Sinks/RawMarketEvent.cs @@ -0,0 +1,14 @@ +using Levels.Core.Format; + +namespace Levels.Sinks; + +public readonly record struct RawMarketEvent( + string Venue, + string Symbol, + long ObservedTime, + long Price, + long Quantity, + RecordType Type, + RecordSide Side, + ReadOnlyMemory ExtensionBytes = default, + bool IsOwner = false); diff --git a/src/Levels.Sinks/SinkConfig.cs b/src/Levels.Sinks/SinkConfig.cs new file mode 100644 index 0000000..c366c6c --- /dev/null +++ b/src/Levels.Sinks/SinkConfig.cs @@ -0,0 +1,22 @@ +using Levels.Core.Format; +using Levels.DataFlow; + +namespace Levels.Sinks; + +public sealed class SinkConfig +{ + public required string OutputPath { get; init; } + public int BackpressureLimit { get; init; } = 8192; + public long RolloverSize { get; init; } = 256 * 1024 * 1024; + public TimeSpan RolloverInterval { get; init; } = TimeSpan.FromHours(1); + public int PriceScale { get; init; } + public int QuantityScale { get; init; } + public uint SchemaId { get; init; } + public int RecordSize { get; init; } = Constants.CoreRecordSize; + public int FlushThresholdMs { get; init; } = 0; + public int FlushBufferSize { get; init; } = 0; + public int ConsumerPartitions { get; init; } = 1; + public int MaxWriteFailures { get; init; } = 5; + public TimeSpan CircuitBreakerCooldown { get; init; } = TimeSpan.FromSeconds(60); + public DataFlowBus? DataFlowBus { get; init; } +} diff --git a/src/Levels.SourceGen/CoreRecordGenerator.cs b/src/Levels.SourceGen/CoreRecordGenerator.cs new file mode 100644 index 0000000..8b8efb9 --- /dev/null +++ b/src/Levels.SourceGen/CoreRecordGenerator.cs @@ -0,0 +1,333 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Text; + +namespace Levels.SourceGen +{ + [Generator] + public sealed class CoreRecordGenerator : IIncrementalGenerator + { + private const int CoreRecordSize = 56; + + /// + /// Field names that are part of the core binary layout or ISchemaEvent routing. + /// Recognized in the schema for documentation but do not contribute to extension offsets. + /// + private static readonly HashSet KnownCoreFields = new HashSet(StringComparer.Ordinal) + { + "exchange", "venue", "symbol", + "observed_time", "write_timestamp", "price_stream_id", + "price", "quantity", + "_reserved", "record_type", "record_side", + "sequence", "level", "flags", "crc32", + }; + + /// + /// The 56-byte core record binary layout, always included in RecordAccessor/RecordWriter. + /// + private static readonly List CoreBinaryFields = new List + { + new FieldDef { Name = "observed_time", ClrType = "long", Offset = 0, Size = 8 }, + new FieldDef { Name = "write_timestamp", ClrType = "long", Offset = 8, Size = 8 }, + new FieldDef { Name = "price_stream_id", ClrType = "long", Offset = 16, Size = 8 }, + new FieldDef { Name = "price", ClrType = "long", Offset = 24, Size = 8 }, + new FieldDef { Name = "quantity", ClrType = "long", Offset = 32, Size = 8 }, + new FieldDef { Name = "_reserved", ClrType = "ushort", Offset = 40, Size = 2 }, + new FieldDef { Name = "record_type", ClrType = "byte", Offset = 42, Size = 1 }, + new FieldDef { Name = "record_side", ClrType = "byte", Offset = 43, Size = 1 }, + new FieldDef { Name = "sequence", ClrType = "uint", Offset = 44, Size = 4 }, + new FieldDef { Name = "level", ClrType = "ushort", Offset = 48, Size = 2 }, + new FieldDef { Name = "flags", ClrType = "ushort", Offset = 50, Size = 2 }, + new FieldDef { Name = "crc32", ClrType = "uint", Offset = 52, Size = 4 }, + }; + + public void Initialize(IncrementalGeneratorInitializationContext context) + { + var fieldsPipeline = context.AdditionalTextsProvider + .Where(static file => file.Path.EndsWith(".fbs", StringComparison.OrdinalIgnoreCase)) + .Select(static (file, ct) => + { + string text = file.GetText(ct)?.ToString() ?? string.Empty; + return (Text: text, Fields: FbsParser.Parse(text)); + }); + + context.RegisterSourceOutput(fieldsPipeline, static (ctx, pair) => + { + IReadOnlyList parsedFields = pair.Fields; + uint schemaId = ComputeFnv1a(pair.Text); + string structName = ParseStructName(pair.Text) ?? "Record"; + string ns = ParseNamespace(pair.Text) ?? "Levels.Core.Generated"; + + // Extension fields = parsed fields not in the known core set + var extensionFields = new List(); + int extOffset = 0; + foreach (var f in parsedFields) + { + if (KnownCoreFields.Contains(f.Name)) + continue; + extensionFields.Add(new FieldDef + { + Name = f.Name, + ClrType = f.ClrType, + Offset = CoreRecordSize + extOffset, + Size = f.Size, + IsFixedArray = f.IsFixedArray, + ArrayLength = f.ArrayLength, + }); + extOffset += f.Size; + } + + // Full binary layout = core 56 bytes + extension fields + var allBinaryFields = new List(CoreBinaryFields); + allBinaryFields.AddRange(extensionFields); + + int recordSize = allBinaryFields.Count > 0 + ? allBinaryFields[allBinaryFields.Count - 1].Offset + allBinaryFields[allBinaryFields.Count - 1].Size + : CoreRecordSize; + + var sb = new StringBuilder(); + sb.AppendLine("// "); + sb.AppendLine("#nullable enable"); + sb.AppendLine(); + sb.AppendLine("using System;"); + sb.AppendLine("using System.Buffers.Binary;"); + sb.AppendLine("using Levels.Core;"); + sb.AppendLine("using Levels.Core.Format;"); + sb.AppendLine(); + sb.AppendLine($"namespace {ns}"); + sb.AppendLine("{"); + + EmitRecordStruct(sb, structName, schemaId, recordSize, extensionFields); + EmitRefStruct(sb, "RecordAccessor", allBinaryFields); + EmitRefStruct(sb, "OrderSnapRecord", allBinaryFields); + EmitRefStruct(sb, "OrderDeltaRecord", allBinaryFields); + EmitWriteRecord(sb, allBinaryFields); + + sb.AppendLine("}"); + + ctx.AddSource("Record.g.cs", SourceText.From(sb.ToString(), Encoding.UTF8)); + }); + } + + private static void EmitRecordStruct( + StringBuilder sb, + string structName, + uint schemaId, + int recordSize, + List extensionFields) + { + var parameters = new List + { + "string Venue", + "string Symbol", + "long ObservedTime", + "long Price", + "long Quantity", + "RecordType RecordType", + "RecordSide RecordSide", + }; + + foreach (FieldDef field in extensionFields) + { + string paramName = ToPascalCase(field.Name); + if (field.IsFixedArray) + parameters.Add($"ReadOnlyMemory {paramName} = default"); + else + parameters.Add($"{field.ClrType} {paramName} = 0"); + } + + sb.AppendLine($" public record struct {structName}("); + for (int i = 0; i < parameters.Count; i++) + { + string suffix = i < parameters.Count - 1 ? "," : ") : ISchemaDescriptor, ISchemaEvent"; + sb.AppendLine($" {parameters[i]}{suffix}"); + } + sb.AppendLine(" {"); + sb.AppendLine($" public static uint SchemaId => 0x{schemaId:X8}u;"); + sb.AppendLine($" public static int RecordSize => {recordSize};"); + sb.AppendLine(); + + sb.AppendLine(" public void WriteExtension(Span destination)"); + sb.AppendLine(" {"); + foreach (FieldDef field in extensionFields) + { + string propName = ToPascalCase(field.Name); + int relOffset = field.Offset - CoreRecordSize; + if (field.IsFixedArray) + { + sb.AppendLine($" {propName}.Span.CopyTo(destination.Slice({relOffset}, {field.ArrayLength}));"); + } + else + { + string writeStmt = GetWriteStatement(field, relOffset, propName, "destination"); + sb.AppendLine($" {writeStmt}"); + } + } + sb.AppendLine(" }"); + + sb.AppendLine(" }"); + sb.AppendLine(); + } + + private static void EmitRefStruct( + StringBuilder sb, + string typeName, + IReadOnlyList fields) + { + sb.AppendLine($" public readonly ref struct {typeName}"); + sb.AppendLine(" {"); + sb.AppendLine(" private readonly ReadOnlySpan _span;"); + sb.AppendLine(); + sb.AppendLine($" public {typeName}(ReadOnlySpan span)"); + sb.AppendLine(" {"); + sb.AppendLine(" _span = span;"); + sb.AppendLine(" }"); + sb.AppendLine(); + + foreach (FieldDef field in fields) + { + string propName = ToPascalCase(field.Name); + if (field.IsFixedArray) + { + sb.AppendLine($" public ReadOnlySpan {propName} => _span.Slice({field.Offset}, {field.ArrayLength});"); + } + else + { + string readExpr = GetReadExpression(field); + sb.AppendLine($" public {field.ClrType} {propName} => {readExpr};"); + } + } + + sb.AppendLine(" }"); + sb.AppendLine(); + } + + private static void EmitWriteRecord(StringBuilder sb, IReadOnlyList fields) + { + sb.AppendLine(" public static class RecordWriter"); + sb.AppendLine(" {"); + + var parameters = new List { "Span span" }; + foreach (FieldDef field in fields) + { + if (field.IsFixedArray) + parameters.Add($"ReadOnlySpan {ToCamelCase(field.Name)}"); + else + parameters.Add($"{field.ClrType} {ToCamelCase(field.Name)}"); + } + + sb.AppendLine($" public static void WriteRecord({string.Join(", ", parameters)})"); + sb.AppendLine(" {"); + + foreach (FieldDef field in fields) + { + string paramName = ToCamelCase(field.Name); + if (field.IsFixedArray) + { + sb.AppendLine($" {paramName}.CopyTo(span.Slice({field.Offset}, {field.ArrayLength}));"); + } + else + { + string writeStmt = GetWriteStatement(field, field.Offset, paramName); + sb.AppendLine($" {writeStmt}"); + } + } + + sb.AppendLine(" }"); + sb.AppendLine(" }"); + } + + private static string GetReadExpression(FieldDef field) + { + switch (field.ClrType) + { + case "long": return $"BinaryPrimitives.ReadInt64LittleEndian(_span.Slice({field.Offset}))"; + case "ulong": return $"BinaryPrimitives.ReadUInt64LittleEndian(_span.Slice({field.Offset}))"; + case "int": return $"BinaryPrimitives.ReadInt32LittleEndian(_span.Slice({field.Offset}))"; + case "uint": return $"BinaryPrimitives.ReadUInt32LittleEndian(_span.Slice({field.Offset}))"; + case "short": return $"BinaryPrimitives.ReadInt16LittleEndian(_span.Slice({field.Offset}))"; + case "ushort": return $"BinaryPrimitives.ReadUInt16LittleEndian(_span.Slice({field.Offset}))"; + case "sbyte": return $"(sbyte)_span[{field.Offset}]"; + case "byte": return $"_span[{field.Offset}]"; + case "float": return $"BinaryPrimitives.ReadSingleLittleEndian(_span.Slice({field.Offset}))"; + case "double": return $"BinaryPrimitives.ReadDoubleLittleEndian(_span.Slice({field.Offset}))"; + case "bool": return $"_span[{field.Offset}] != 0"; + default: return $"default /* unknown type: {field.ClrType} */"; + } + } + + private static string GetWriteStatement(FieldDef field, int offset, string paramName, string target = "span") + { + switch (field.ClrType) + { + case "long": return $"BinaryPrimitives.WriteInt64LittleEndian({target}.Slice({offset}), {paramName});"; + case "ulong": return $"BinaryPrimitives.WriteUInt64LittleEndian({target}.Slice({offset}), {paramName});"; + case "int": return $"BinaryPrimitives.WriteInt32LittleEndian({target}.Slice({offset}), {paramName});"; + case "uint": return $"BinaryPrimitives.WriteUInt32LittleEndian({target}.Slice({offset}), {paramName});"; + case "short": return $"BinaryPrimitives.WriteInt16LittleEndian({target}.Slice({offset}), {paramName});"; + case "ushort": return $"BinaryPrimitives.WriteUInt16LittleEndian({target}.Slice({offset}), {paramName});"; + case "sbyte": return $"{target}[{offset}] = (byte){paramName};"; + case "byte": return $"{target}[{offset}] = {paramName};"; + case "float": return $"BinaryPrimitives.WriteSingleLittleEndian({target}.Slice({offset}), {paramName});"; + case "double": return $"BinaryPrimitives.WriteDoubleLittleEndian({target}.Slice({offset}), {paramName});"; + case "bool": return $"{target}[{offset}] = {paramName} ? (byte)1 : (byte)0;"; + default: return $"// unknown type: {field.ClrType}"; + } + } + + private static string? ParseStructName(string fbsText) + { + foreach (string rawLine in fbsText.Split('\n')) + { + string line = rawLine.Trim(); + if (line.StartsWith("struct ", StringComparison.Ordinal) && line.EndsWith("{", StringComparison.Ordinal)) + return line.Substring("struct ".Length, line.Length - "struct ".Length - 1).Trim(); + } + return null; + } + + private static string? ParseNamespace(string fbsText) + { + foreach (string rawLine in fbsText.Split('\n')) + { + string line = rawLine.Trim(); + if (line.StartsWith("namespace ", StringComparison.Ordinal) && line.EndsWith(";", StringComparison.Ordinal)) + return line.Substring("namespace ".Length, line.Length - "namespace ".Length - 1).Trim(); + } + return null; + } + + private static uint ComputeFnv1a(string text) + { + uint hash = 2166136261u; + foreach (char c in text) { hash ^= (byte)c; hash *= 16777619u; } + return hash; + } + + private static string ToPascalCase(string snakeName) + { + var sb = new StringBuilder(); + bool upper = true; + foreach (char c in snakeName) + { + if (c == '_') { upper = true; continue; } + sb.Append(upper ? char.ToUpperInvariant(c) : c); + upper = false; + } + return sb.ToString(); + } + + private static string ToCamelCase(string snakeName) + { + string pascal = ToPascalCase(snakeName); + if (pascal.Length == 0) return pascal; + return char.ToLowerInvariant(pascal[0]) + pascal.Substring(1); + } + } +} diff --git a/src/Levels.SourceGen/FbsParser.cs b/src/Levels.SourceGen/FbsParser.cs new file mode 100644 index 0000000..a2c456d --- /dev/null +++ b/src/Levels.SourceGen/FbsParser.cs @@ -0,0 +1,108 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Text.RegularExpressions; + +namespace Levels.SourceGen +{ + public static class FbsParser + { + private static readonly Dictionary TypeMap = + new Dictionary + { + { "int64", ("long", 8) }, + { "uint64", ("ulong", 8) }, + { "int32", ("int", 4) }, + { "uint32", ("uint", 4) }, + { "int16", ("short", 2) }, + { "uint16", ("ushort", 2) }, + { "int8", ("sbyte", 1) }, + { "uint8", ("byte", 1) }, + { "float32", ("float", 4) }, + { "float64", ("double", 8) }, + { "bool", ("bool", 1) }, + { "string", ("string", 0) }, + }; + + public static List Parse(string fbsText) + { + var fields = new List(); + int offset = 0; + bool inStruct = false; + + foreach (string rawLine in fbsText.Split('\n')) + { + string line = rawLine.Trim(); + + // Strip inline comments + int commentIdx = line.IndexOf("//", StringComparison.Ordinal); + if (commentIdx >= 0) + line = line.Substring(0, commentIdx).Trim(); + + if (line.Length == 0) + continue; + + // Detect struct opening: "struct Name {" + if (line.StartsWith("struct ", StringComparison.Ordinal) && line.EndsWith("{", StringComparison.Ordinal)) + { + inStruct = true; + offset = 0; + continue; + } + + // Detect struct closing + if (inStruct && line == "}") + { + inStruct = false; + continue; + } + + if (!inStruct) + continue; + + // Parse field: "name:type;" + string fieldLine = line.TrimEnd(';'); + int colonIdx = fieldLine.IndexOf(':'); + if (colonIdx < 0) + continue; + + string fieldName = fieldLine.Substring(0, colonIdx).Trim(); + string fbsType = fieldLine.Substring(colonIdx + 1).Trim(); + + if (TypeMap.TryGetValue(fbsType, out var mapped)) + { + fields.Add(new FieldDef + { + Name = fieldName, + ClrType = mapped.ClrType, + Offset = offset, + Size = mapped.Size, + }); + if (mapped.Size > 0) + offset += mapped.Size; + } + else + { + var arrayMatch = Regex.Match(fbsType, @"^byte\[(\d+)\]$"); + if (!arrayMatch.Success) + continue; + + int arrayLen = int.Parse(arrayMatch.Groups[1].Value); + fields.Add(new FieldDef + { + Name = fieldName, + ClrType = "byte", + Offset = offset, + Size = arrayLen, + IsFixedArray = true, + ArrayLength = arrayLen, + }); + offset += arrayLen; + } + } + + return fields; + } + } +} diff --git a/src/Levels.SourceGen/FieldDef.cs b/src/Levels.SourceGen/FieldDef.cs new file mode 100644 index 0000000..022633a --- /dev/null +++ b/src/Levels.SourceGen/FieldDef.cs @@ -0,0 +1,14 @@ +#nullable enable + +namespace Levels.SourceGen +{ + public sealed class FieldDef + { + public string Name { get; set; } = string.Empty; + public string ClrType { get; set; } = string.Empty; + public int Offset { get; set; } + public int Size { get; set; } + public bool IsFixedArray { get; set; } + public int ArrayLength { get; set; } + } +} diff --git a/src/Levels.SourceGen/Levels.SourceGen.csproj b/src/Levels.SourceGen/Levels.SourceGen.csproj new file mode 100644 index 0000000..af0c30d --- /dev/null +++ b/src/Levels.SourceGen/Levels.SourceGen.csproj @@ -0,0 +1,26 @@ + + + + netstandard2.0 + latest + enable + true + + + false + true + true + Source generator for Levels time-series database + + + + + + + + + + + + + diff --git a/src/Levels.Web/Levels.Web.csproj b/src/Levels.Web/Levels.Web.csproj new file mode 100644 index 0000000..0251fad --- /dev/null +++ b/src/Levels.Web/Levels.Web.csproj @@ -0,0 +1,26 @@ + + + + net10.0 + enable + enable + preview + false + + + + + + + + + + + + + + + + + + diff --git a/src/Levels.Web/LevelsEndpoints.cs b/src/Levels.Web/LevelsEndpoints.cs new file mode 100644 index 0000000..64489f4 --- /dev/null +++ b/src/Levels.Web/LevelsEndpoints.cs @@ -0,0 +1,60 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; +using Microsoft.Extensions.DependencyInjection; +using Levels.Core; +using Levels.Export; +using Levels.Query; + +namespace Levels.Web; + +public static class LevelsEndpoints +{ + public static IEndpointRouteBuilder MapLevelsApi(this IEndpointRouteBuilder endpoints) + { + var api = endpoints.MapGroup("/api"); + + api.MapGet("/health", () => Results.Ok(new { status = "healthy", timestamp = DateTime.UtcNow })); + + api.MapGet("/streams", (FileIndex fileIndex) => + { + var allEntries = fileIndex.GetAllStreams(); + return Results.Ok(allEntries); + }); + + api.MapGet("/streams/{venue}/{symbol}/book", (string venue, string symbol, QueryLayer queryLayer, bool? excludeOwner) => + { + var streamId = PriceStreamId.FromVenueSymbol(venue, symbol); + var projection = new OrderbookProjection(queryLayer); + var l2 = projection.ProjectL2(streamId, 0, long.MaxValue, excludeOwner ?? false); + return Results.Ok(l2); + }); + + api.MapGet("/streams/{venue}/{symbol}/book/l1", (string venue, string symbol, QueryLayer queryLayer, bool? excludeOwner) => + { + var streamId = PriceStreamId.FromVenueSymbol(venue, symbol); + var projection = new OrderbookProjection(queryLayer); + var l1 = projection.ProjectL1(streamId, 0, long.MaxValue, excludeOwner ?? false); + return Results.Ok(l1); + }); + + api.MapGet("/streams/{venue}/{symbol}/export", async (string venue, string symbol, string format, QueryLayer queryLayer, + ExportPipeline exportPipeline, HttpContext context) => + { + var streamId = PriceStreamId.FromVenueSymbol(venue, symbol); + var contentType = format.ToLowerInvariant() switch + { + "csv" => "text/csv", + "parquet" => "application/octet-stream", + "avro" => "application/octet-stream", + _ => "application/octet-stream", + }; + + context.Response.ContentType = contentType; + context.Response.Headers.Append("Content-Disposition", $"attachment; filename=\"{symbol}.{format}\""); + await exportPipeline.ExportAsync(streamId, 0, long.MaxValue, format, context.Response.Body, context.RequestAborted); + }); + + return endpoints; + } +} diff --git a/src/Levels.Web/Program.cs b/src/Levels.Web/Program.cs new file mode 100644 index 0000000..940b67a --- /dev/null +++ b/src/Levels.Web/Program.cs @@ -0,0 +1,37 @@ +using Levels.Core; +using Levels.Export; +using Levels.Hosting; +using Levels.Query; +using Levels.Web; +using OpenTelemetry.Metrics; + +var builder = WebApplication.CreateBuilder(args); + +var dataPath = builder.Configuration["Levels:DataPath"] ?? "./data"; + +builder.Services.AddLevels(options => +{ + options.DataPath = dataPath; + options.EnableMetrics = builder.Configuration.GetValue("Levels:EnableMetrics"); + options.OtlpEndpoint = builder.Configuration["Levels:OtlpEndpoint"]; + options.EnableTracing = builder.Configuration.GetValue("Levels:EnableTracing"); +}); + +// Prometheus scraping endpoint (always available for the web host) +builder.Services.AddOpenTelemetry() + .WithMetrics(m => m.AddMeter("Levels").AddPrometheusExporter()); + +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(sp => + new ExportPipeline( + sp.GetRequiredService(), + sp.GetServices())); + +var app = builder.Build(); + +app.MapLevelsApi(); +app.UseOpenTelemetryPrometheusScrapingEndpoint(); + +app.Run(); diff --git a/tests/Levels.Benchmarks/Crc32Benchmarks.cs b/tests/Levels.Benchmarks/Crc32Benchmarks.cs new file mode 100644 index 0000000..1bf44a7 --- /dev/null +++ b/tests/Levels.Benchmarks/Crc32Benchmarks.cs @@ -0,0 +1,46 @@ +using BenchmarkDotNet.Attributes; +using Levels.Core.Format; + +namespace Levels.Benchmarks; + +[MemoryDiagnoser] +public class Crc32Benchmarks +{ + private byte[] _recordData = null!; + private Random _rng = null!; + + [GlobalSetup] + public void Setup() + { + _rng = new Random(42); + _recordData = new byte[Constants.CoreRecordSize]; + _rng.NextBytes(_recordData); + } + + [Benchmark] + public uint ComputePerRecordCrc() + { + Span data = stackalloc byte[Constants.CoreRecordSize]; + _recordData.AsSpan().CopyTo(data); + // CRC is computed over first 52 bytes (core without CRC field) + return Crc32Util.ComputeRecord(data[..52], ReadOnlySpan.Empty); + } + + [Benchmark] + [Arguments(1000)] + [Arguments(10000)] + [Arguments(100000)] + public uint ComputeBatchCrc(int batchSize) + { + uint lastCrc = 0; + Span data = stackalloc byte[Constants.CoreRecordSize]; + _recordData.AsSpan().CopyTo(data); + + for (int i = 0; i < batchSize; i++) + { + lastCrc = Crc32Util.ComputeRecord(data[..52], ReadOnlySpan.Empty); + } + + return lastCrc; + } +} diff --git a/tests/Levels.Benchmarks/FlushBenchmarks.cs b/tests/Levels.Benchmarks/FlushBenchmarks.cs new file mode 100644 index 0000000..b3f0ff7 --- /dev/null +++ b/tests/Levels.Benchmarks/FlushBenchmarks.cs @@ -0,0 +1,90 @@ +using BenchmarkDotNet.Attributes; +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; + +namespace Levels.Benchmarks; + +[MemoryDiagnoser] +public class FlushBenchmarks +{ + private PriceStreamId _streamId; + private string _tempDir = null!; + + [GlobalSetup] + public void Setup() + { + _streamId = PriceStreamId.FromSymbol("BENCH/USD"); + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_bench_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + [GlobalCleanup] + public void Cleanup() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + [Benchmark(Baseline = true)] + public async Task WriteToMemoryStream_10k() + { + using var ms = new MemoryStream(); + var writer = await BinaryRecordWriter.CreateAsync(ms, FileType.Raw, _streamId); + await WriteRecords(writer, 10_000); + await writer.SealAsync(); + } + + [Benchmark] + public async Task WriteToFileStream_WriteThrough_10k() + { + var path = Path.Combine(_tempDir, $"{Guid.NewGuid():N}.raw"); + using var fs = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.None, + bufferSize: 4096, FileOptions.WriteThrough | FileOptions.Asynchronous); + var writer = await BinaryRecordWriter.CreateAsync(fs, FileType.Raw, _streamId); + await WriteRecords(writer, 10_000); + await writer.SealAsync(); + } + + [Benchmark] + public async Task WriteToFileStream_FlushThreshold64_10k() + { + var path = Path.Combine(_tempDir, $"{Guid.NewGuid():N}.raw"); + using var fs = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.None, + bufferSize: 4096, FileOptions.Asynchronous); + var writer = await BinaryRecordWriter.CreateAsync(fs, FileType.Raw, _streamId, flushBufferSize: 64); + await WriteRecords(writer, 10_000); + await writer.SealAsync(); + } + + [Benchmark] + public async Task WriteToFileStream_Buffered_10k() + { + var path = Path.Combine(_tempDir, $"{Guid.NewGuid():N}.raw"); + using var fs = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.None, + bufferSize: 4096, FileOptions.Asynchronous); + var writer = await BinaryRecordWriter.CreateAsync(fs, FileType.Raw, _streamId); + await WriteRecords(writer, 10_000); + await writer.SealAsync(); + } + + private async Task WriteRecords(BinaryRecordWriter writer, int count) + { + var baseTime = WriteTimestamp.Now(); + for (int i = 0; i < count; i++) + { + var core = new CoreRecordLayout + { + ObservedTime = baseTime + i * 1000, + PriceStreamId = _streamId.Value, + Price = 50000_00000000L + i, + Quantity = 1_00000000L, + Type = RecordType.Snap, + Side = RecordSide.Bid, + Sequence = (uint)i, + Level = 0, + }; + await writer.WriteRecordAsync(core); + } + } +} diff --git a/tests/Levels.Benchmarks/Levels.Benchmarks.csproj b/tests/Levels.Benchmarks/Levels.Benchmarks.csproj new file mode 100644 index 0000000..f2a71b8 --- /dev/null +++ b/tests/Levels.Benchmarks/Levels.Benchmarks.csproj @@ -0,0 +1,26 @@ + + + + Exe + net10.0 + enable + enable + preview + false + + + + + + + + + + + + + + + + + diff --git a/tests/Levels.Benchmarks/Program.cs b/tests/Levels.Benchmarks/Program.cs new file mode 100644 index 0000000..c9a0467 --- /dev/null +++ b/tests/Levels.Benchmarks/Program.cs @@ -0,0 +1,3 @@ +using BenchmarkDotNet.Running; + +BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); diff --git a/tests/Levels.Benchmarks/QueryBenchmarks.cs b/tests/Levels.Benchmarks/QueryBenchmarks.cs new file mode 100644 index 0000000..fe451b1 --- /dev/null +++ b/tests/Levels.Benchmarks/QueryBenchmarks.cs @@ -0,0 +1,77 @@ +using BenchmarkDotNet.Attributes; +using Levels.Core; +using Levels.Core.Format; +using Levels.Query; + +namespace Levels.Benchmarks; + +[MemoryDiagnoser] +public class QueryBenchmarks +{ + [Params(10, 100, 1000)] + public int FileCount { get; set; } + + private QueryLayer _queryLayer = null!; + private PriceStreamId _streamId; + private long _queryFrom; + private long _queryTo; + + [GlobalSetup] + public void Setup() + { + _streamId = PriceStreamId.FromSymbol("BENCH/USD"); + var fileIndex = new FileIndex(); + var config = new QueryConfig + { + DataPath = "/tmp/bench", + CompactionWindow = TimeSpan.FromHours(1), + }; + + var windowNanos = config.CompactionWindow.Ticks * 100; + var baseTime = 1_000_000_000_000L; // arbitrary base + + for (int i = 0; i < FileCount; i++) + { + var firstObserved = baseTime + i * windowNanos; + var lastObserved = firstObserved + windowNanos - 1; + + var entry = new FileIndexEntry( + FilePath: $"/tmp/bench/exchange/{_streamId.Value}/{i}.raw", + PriceStreamId: _streamId, + Venue: "exchange", + FileType: FileType.Raw, + FirstObservedTime: firstObserved, + LastObservedTime: lastObserved, + RecordCount: 1000); + + fileIndex.Register(entry); + } + + _queryLayer = new QueryLayer(fileIndex, config); + _queryFrom = baseTime; + _queryTo = baseTime + FileCount * windowNanos; + } + + [Benchmark] + public IReadOnlyList ResolveFullRange() + { + return _queryLayer.Resolve(_streamId, _queryFrom, _queryTo); + } + + [Benchmark] + public IReadOnlyList ResolveNarrowRange() + { + // Query a single window in the middle + var windowNanos = TimeSpan.FromHours(1).Ticks * 100; + var mid = _queryFrom + (FileCount / 2) * windowNanos; + return _queryLayer.Resolve(_streamId, mid, mid + windowNanos); + } + + [Benchmark] + public FileIndexEntry? ResolveSinglePoint() + { + var windowNanos = TimeSpan.FromHours(1).Ticks * 100; + var mid = _queryFrom + (FileCount / 2) * windowNanos; + return _queryLayer.ResolveSingle(_streamId, mid); + } +} diff --git a/tests/Levels.Benchmarks/RecordReadBenchmarks.cs b/tests/Levels.Benchmarks/RecordReadBenchmarks.cs new file mode 100644 index 0000000..df30ac3 --- /dev/null +++ b/tests/Levels.Benchmarks/RecordReadBenchmarks.cs @@ -0,0 +1,71 @@ +using BenchmarkDotNet.Attributes; +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; + +namespace Levels.Benchmarks; + +[MemoryDiagnoser] +public class RecordReadBenchmarks +{ + private byte[] _fileData = null!; + + [GlobalSetup] + public async Task Setup() + { + var streamId = PriceStreamId.FromSymbol("BENCH/USD"); + using var ms = new MemoryStream(); + var writer = await BinaryRecordWriter.CreateAsync(ms, FileType.Raw, streamId); + + var baseTime = WriteTimestamp.Now(); + for (int i = 0; i < 100_000; i++) + { + var core = new CoreRecordLayout + { + ObservedTime = baseTime + i * 1000, + PriceStreamId = streamId.Value, + Price = 50000_00000000L + i, + Quantity = 1_00000000L, + Type = RecordType.Snap, + Side = RecordSide.Bid, + Sequence = (uint)i, + Level = 0, + }; + + await writer.WriteRecordAsync(core); + } + + await writer.SealAsync(); + _fileData = ms.ToArray(); + } + + [Benchmark] + public int ReadAllRecords() + { + using var ms = new MemoryStream(_fileData, writable: false); + var reader = new BinaryRecordReader(ms); + + int count = 0; + foreach (var record in reader.ReadRecords(validateCrc: true)) + { + count++; + } + + return count; + } + + [Benchmark] + public int ReadAllRecordsNoCrc() + { + using var ms = new MemoryStream(_fileData, writable: false); + var reader = new BinaryRecordReader(ms); + + int count = 0; + foreach (var record in reader.ReadRecords(validateCrc: false)) + { + count++; + } + + return count; + } +} diff --git a/tests/Levels.Benchmarks/RecordWriteBenchmarks.cs b/tests/Levels.Benchmarks/RecordWriteBenchmarks.cs new file mode 100644 index 0000000..cc4d42e --- /dev/null +++ b/tests/Levels.Benchmarks/RecordWriteBenchmarks.cs @@ -0,0 +1,48 @@ +using BenchmarkDotNet.Attributes; +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; + +namespace Levels.Benchmarks; + +[MemoryDiagnoser] +public class RecordWriteBenchmarks +{ + [Params(1000, 10000, 100000)] + public int RecordCount { get; set; } + + private PriceStreamId _streamId; + + [GlobalSetup] + public void Setup() + { + _streamId = PriceStreamId.FromSymbol("BENCH/USD"); + } + + [Benchmark] + public async Task WriteRecords() + { + using var ms = new MemoryStream(); + var writer = await BinaryRecordWriter.CreateAsync(ms, FileType.Raw, _streamId); + + var baseTime = WriteTimestamp.Now(); + for (int i = 0; i < RecordCount; i++) + { + var core = new CoreRecordLayout + { + ObservedTime = baseTime + i * 1000, + PriceStreamId = _streamId.Value, + Price = 50000_00000000L + i, + Quantity = 1_00000000L, + Type = RecordType.Snap, + Side = RecordSide.Bid, + Sequence = (uint)i, + Level = 0, + }; + + await writer.WriteRecordAsync(core); + } + + await writer.SealAsync(); + } +} diff --git a/tests/Levels.Tests/Compaction/CompactionTests.cs b/tests/Levels.Tests/Compaction/CompactionTests.cs new file mode 100644 index 0000000..1fa4807 --- /dev/null +++ b/tests/Levels.Tests/Compaction/CompactionTests.cs @@ -0,0 +1,192 @@ +using Levels.Compaction; +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Sinks; + +namespace Levels.Tests.Compaction; + +public class CompactionTests : IDisposable +{ + private readonly string _tempDir; + + public CompactionTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + [Fact] + public async Task SingleRawToAgg_RecordCountMatches() + { + // Create a RAW file via ingestion + var config = new SinkConfig { OutputPath = _tempDir }; + var sink = new PriceStreamSink(config); + await sink.StartAsync(CancellationToken.None); + + await RandomDataAdapter.GenerateAsync( + sink.Writer, "test-exchange", ["BTC-USD"], + levelsPerSide: 3, deltasPerSymbol: 20); + + await sink.StopAsync(CancellationToken.None); + + var rawFiles = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories) + .OrderBy(f => f).ToList(); + Assert.NotEmpty(rawFiles); + + // Count records in raw files + var rawRecordCount = 0L; + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + foreach (var file in rawFiles) + { + using var fs = File.OpenRead(file); + var reader = new BinaryRecordReader(fs); + rawRecordCount += reader.ReadRecords().Count(); + } + + // Compact + var compactionConfig = new CompactionConfig + { + DataPath = _tempDir, + CompactionWindow = TimeSpan.FromHours(24), + SyntheticSnapIntervalDeltas = 10000, // High enough to not trigger + }; + + var engine = new OrderbookReplayEngine(compactionConfig); + var replayRecords = engine.Replay(rawFiles).ToList(); + + // No synthetic snaps should be injected with high interval + Assert.Equal(rawRecordCount, replayRecords.Count); + + // Write AGG file + var aggPath = Path.Combine(_tempDir, "test.agg"); + var aggInfo = await AggFileWriter.WriteAsync( + replayRecords, + aggPath, + streamId, + priceScale: 0, + quantityScale: 0, + rawFiles); + + Assert.Equal(rawRecordCount, aggInfo.RecordCount); + + // Verify AGG header + using var aggFs = File.OpenRead(aggPath); + var aggReader = new BinaryRecordReader(aggFs); + Assert.Equal(FileType.Agg, aggReader.Header.FileType); + Assert.NotNull(aggReader.Footer); + } + + [Fact] + public async Task MultipleRawToAgg_SyntheticSnapsAtCorrectIntervals() + { + // Create RAW files with small rollover to get multiple + var config = new SinkConfig + { + OutputPath = _tempDir, + RolloverSize = 512, + }; + var sink = new PriceStreamSink(config); + await sink.StartAsync(CancellationToken.None); + + await RandomDataAdapter.GenerateAsync( + sink.Writer, "test-exchange", ["ETH-USD"], + levelsPerSide: 3, deltasPerSymbol: 50); + + await sink.StopAsync(CancellationToken.None); + + var rawFiles = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories) + .OrderBy(f => f).ToList(); + Assert.True(rawFiles.Count > 1, $"Expected multiple RAW files, got {rawFiles.Count}"); + + // Compact with synthetic snap every 10 deltas + var compactionConfig = new CompactionConfig + { + DataPath = _tempDir, + CompactionWindow = TimeSpan.FromHours(24), + SyntheticSnapIntervalDeltas = 10, + }; + + var engine = new OrderbookReplayEngine(compactionConfig); + var replayRecords = engine.Replay(rawFiles).ToList(); + + // Check that synthetic SNAPs exist + var syntheticSnaps = replayRecords + .Where(r => r.Core.Type == RecordType.Snap && (r.Core.Flags & Constants.SyntheticSnapFlag) != 0) + .ToList(); + Assert.NotEmpty(syntheticSnaps); + + // Write and verify AGG + var streamId = PriceStreamId.FromSymbol("ETH-USD"); + var aggPath = Path.Combine(_tempDir, "test_multi.agg"); + var aggInfo = await AggFileWriter.WriteAsync( + engine.Replay(rawFiles), + aggPath, + streamId, + priceScale: 0, + quantityScale: 0, + rawFiles); + + Assert.True(aggInfo.SyntheticSnapCount > 0); + + // AGG file readable with CRC validation + using var aggFs = File.OpenRead(aggPath); + var aggReader = new BinaryRecordReader(aggFs); + var aggRecords = aggReader.ReadRecords(validateCrc: true).ToList(); + Assert.Equal(aggInfo.RecordCount, aggRecords.Count); + } + + [Fact] + public async Task AggFile_ReadableByBinaryRecordReader_WithValidCrc() + { + var config = new SinkConfig { OutputPath = _tempDir }; + var sink = new PriceStreamSink(config); + await sink.StartAsync(CancellationToken.None); + + await RandomDataAdapter.GenerateAsync( + sink.Writer, "test-exchange", ["SOL-USD"], + levelsPerSide: 2, deltasPerSymbol: 30); + + await sink.StopAsync(CancellationToken.None); + + var rawFiles = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories) + .OrderBy(f => f).ToList(); + + var compactionConfig = new CompactionConfig + { + DataPath = _tempDir, + CompactionWindow = TimeSpan.FromHours(24), + SyntheticSnapIntervalDeltas = 15, + }; + + var engine = new OrderbookReplayEngine(compactionConfig); + var streamId = PriceStreamId.FromSymbol("SOL-USD"); + var aggPath = Path.Combine(_tempDir, "sol_test.agg"); + + await AggFileWriter.WriteAsync( + engine.Replay(rawFiles), + aggPath, + streamId, + priceScale: 0, + quantityScale: 0, + rawFiles); + + // Verify AGG is fully readable with CRC validation + using var fs = File.OpenRead(aggPath); + var reader = new BinaryRecordReader(fs); + Assert.Equal(FileType.Agg, reader.Header.FileType); + Assert.NotNull(reader.Footer); + Assert.False(reader.IsPartial); + + // Full CRC-validated read — will throw if any record CRC is wrong + var records = reader.ReadRecords(validateCrc: true).ToList(); + Assert.True(records.Count > 0); + Assert.Equal(reader.Footer.Value.RecordCount, records.Count); + } +} diff --git a/tests/Levels.Tests/Compaction/OrderbookReplayTests.cs b/tests/Levels.Tests/Compaction/OrderbookReplayTests.cs new file mode 100644 index 0000000..fefa6c8 --- /dev/null +++ b/tests/Levels.Tests/Compaction/OrderbookReplayTests.cs @@ -0,0 +1,126 @@ +using Levels.Compaction; +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Core.Orderbook; +using Levels.Sinks; + +namespace Levels.Tests.Compaction; + +public class OrderbookReplayTests : IDisposable +{ + private readonly string _tempDir; + + public OrderbookReplayTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + [Fact] + public async Task SyntheticSnapContent_MatchesReconstructedOrderbook() + { + // Generate test data with rollovers + var config = new SinkConfig + { + OutputPath = _tempDir, + RolloverSize = 512, + }; + var sink = new PriceStreamSink(config); + await sink.StartAsync(CancellationToken.None); + + await RandomDataAdapter.GenerateAsync( + sink.Writer, "test-exchange", ["BTC-USD"], + levelsPerSide: 3, deltasPerSymbol: 50, seed: 123); + + await sink.StopAsync(CancellationToken.None); + + var rawFiles = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories) + .OrderBy(f => f).ToList(); + + // Replay with synthetic snaps every 5 deltas + var compactionConfig = new CompactionConfig + { + DataPath = _tempDir, + CompactionWindow = TimeSpan.FromHours(24), + SyntheticSnapIntervalDeltas = 5, + }; + + var engine = new OrderbookReplayEngine(compactionConfig); + var records = engine.Replay(rawFiles).ToList(); + + // Independently reconstruct orderbook to verify synthetic SNAPs + var bids = new OrderbookSide(); + var asks = new OrderbookSide(); + + foreach (var record in records) + { + var core = record.Core; + if ((core.Flags & Constants.SyntheticSnapFlag) != 0) + { + // This is a synthetic SNAP — verify it matches current state + var side = core.Side == RecordSide.Bid ? bids : asks; + var level = side.Levels.FirstOrDefault(l => l.Key == core.Price); + Assert.Equal(core.Quantity, level.Value); + } + else + { + // Regular record — apply to orderbook + if (core.Type is RecordType.Snap or RecordType.Delta) + { + var side = core.Side == RecordSide.Bid ? bids : asks; + side.Apply(core.Price, core.Quantity); + } + } + } + } + + [Fact] + public async Task ReplayWithoutSyntheticSnaps_PreservesAllOriginalRecords() + { + var config = new SinkConfig { OutputPath = _tempDir }; + var sink = new PriceStreamSink(config); + await sink.StartAsync(CancellationToken.None); + + await RandomDataAdapter.GenerateAsync( + sink.Writer, "test-exchange", ["ETH-USD"], + levelsPerSide: 2, deltasPerSymbol: 20, seed: 456); + + await sink.StopAsync(CancellationToken.None); + + var rawFiles = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories) + .OrderBy(f => f).ToList(); + + // Count original records + var originalCount = 0; + foreach (var file in rawFiles) + { + using var fs = File.OpenRead(file); + var reader = new BinaryRecordReader(fs); + originalCount += reader.ReadRecords().Count(); + } + + // Replay without synthetic snaps (high interval) + var compactionConfig = new CompactionConfig + { + DataPath = _tempDir, + CompactionWindow = TimeSpan.FromHours(24), + SyntheticSnapIntervalDeltas = 100000, + }; + + var engine = new OrderbookReplayEngine(compactionConfig); + var replayRecords = engine.Replay(rawFiles).ToList(); + + Assert.Equal(originalCount, replayRecords.Count); + + // No synthetic snaps + var syntheticCount = replayRecords.Count(r => (r.Core.Flags & Constants.SyntheticSnapFlag) != 0); + Assert.Equal(0, syntheticCount); + } +} diff --git a/tests/Levels.Tests/Compaction/StartupRecoveryTests.cs b/tests/Levels.Tests/Compaction/StartupRecoveryTests.cs new file mode 100644 index 0000000..9fd011c --- /dev/null +++ b/tests/Levels.Tests/Compaction/StartupRecoveryTests.cs @@ -0,0 +1,114 @@ +using Levels.Compaction; +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Sinks; + +namespace Levels.Tests.Compaction; + +public class StartupRecoveryTests : IDisposable +{ + private readonly string _tempDir; + + public StartupRecoveryTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + [Fact] + public async Task PartialAgg_DeletedOnStartup() + { + // Create a valid RAW file + var config = new SinkConfig { OutputPath = _tempDir }; + var sink = new PriceStreamSink(config); + await sink.StartAsync(CancellationToken.None); + + await RandomDataAdapter.GenerateAsync( + sink.Writer, "test-exchange", ["BTC-USD"], + levelsPerSide: 2, deltasPerSymbol: 10); + + await sink.StopAsync(CancellationToken.None); + + // Create a partial (unsealed) AGG file in the same directory + var streamId = PriceStreamId.FromVenueSymbol("test-exchange", "BTC-USD"); + var aggDir = Path.Combine(_tempDir, "test-exchange", streamId.Value.ToString()); + var partialAggPath = Path.Combine(aggDir, "partial.agg"); + + // Write a header-only file (no footer = partial) + await using (var fs = new FileStream(partialAggPath, FileMode.CreateNew, FileAccess.Write)) + { + var writer = await BinaryRecordWriter.CreateAsync(fs, FileType.Agg, streamId); + // Don't seal — leave partial + await writer.DisposeAsync(); + } + + Assert.True(File.Exists(partialAggPath)); + + // Start compaction — should delete the partial AGG + var compactionConfig = new CompactionConfig + { + DataPath = _tempDir, + CompactionWindow = TimeSpan.FromHours(1), + }; + + await using var compaction = new EventSourcingCompaction(compactionConfig); + await compaction.StartAsync(CancellationToken.None); + + // Give a moment for startup recovery + await Task.Delay(100); + + Assert.False(File.Exists(partialAggPath), "Partial AGG should have been deleted on startup"); + + await compaction.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task WindowsWithRawButNoAgg_Requeued() + { + // Create sealed RAW files with old timestamps (so window is definitely closed) + var config = new SinkConfig { OutputPath = _tempDir }; + var sink = new PriceStreamSink(config); + await sink.StartAsync(CancellationToken.None); + + await RandomDataAdapter.GenerateAsync( + sink.Writer, "test-exchange", ["BTC-USD"], + levelsPerSide: 2, deltasPerSymbol: 10); + + await sink.StopAsync(CancellationToken.None); + + var rawFiles = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories); + Assert.NotEmpty(rawFiles); + + // No AGG files should exist yet + var aggFilesBefore = Directory.GetFiles(_tempDir, "*.agg", SearchOption.AllDirectories); + Assert.Empty(aggFilesBefore); + + // Start compaction with very short grace period + var compactionConfig = new CompactionConfig + { + DataPath = _tempDir, + CompactionWindow = TimeSpan.FromHours(24), + SyntheticSnapIntervalDeltas = 10000, + WindowGracePeriod = TimeSpan.Zero, + }; + + await using var compaction = new EventSourcingCompaction(compactionConfig); + await compaction.StartAsync(CancellationToken.None); + + // Give time for background processing + await Task.Delay(2000); + + await compaction.StopAsync(CancellationToken.None); + + // AGG file should have been created + var aggFilesAfter = Directory.GetFiles(_tempDir, "*.agg", SearchOption.AllDirectories); + Assert.NotEmpty(aggFilesAfter); + } +} diff --git a/tests/Levels.Tests/CrossLanguage/CrossLanguageRoundTripTests.cs b/tests/Levels.Tests/CrossLanguage/CrossLanguageRoundTripTests.cs new file mode 100644 index 0000000..4e989ae --- /dev/null +++ b/tests/Levels.Tests/CrossLanguage/CrossLanguageRoundTripTests.cs @@ -0,0 +1,71 @@ +using Levels.Core.Format; +using Levels.Core.IO; + +namespace Levels.Tests.CrossLanguage; + +public class CrossLanguageRoundTripTests +{ + private static string GetFixturePath(string language) => + Path.Combine(FindCrossLanguageDir(), "fixtures", $"{language}_test.raw"); + + private static string FindCrossLanguageDir() + { + var dir = AppContext.BaseDirectory; + while (dir is not null) + { + var candidate = Path.Combine(dir, "cross-language"); + if (Directory.Exists(candidate)) return candidate; + dir = Path.GetDirectoryName(dir); + } + throw new DirectoryNotFoundException("cross-language directory not found"); + } + + [Theory] + [InlineData("python")] + [InlineData("cpp")] + [InlineData("typescript")] + public void ReadFixture_MatchesExpectedTestVector(string language) + { + var fixturePath = GetFixturePath(language); + if (!File.Exists(fixturePath)) + { + // Skip if fixture not generated yet + return; + } + + using var fs = File.OpenRead(fixturePath); + var reader = new BinaryRecordReader(fs); + + // Validate header + Assert.Equal(FileType.Raw, reader.Header.FileType); + Assert.Equal(42L, reader.Header.PriceStreamId); + Assert.Equal(2, reader.Header.PriceScale); + Assert.Equal(4, reader.Header.QuantityScale); + + // Validate records + var records = reader.ReadRecords(validateCrc: true).ToList(); + Assert.Equal(3, records.Count); + + // Record 1: SNAP Bid + Assert.Equal(RecordType.Snap, records[0].Core.Type); + Assert.Equal(RecordSide.Bid, records[0].Core.Side); + Assert.Equal(50000L, records[0].Core.Price); + Assert.Equal(100L, records[0].Core.Quantity); + Assert.Equal(1000000L, records[0].Core.ObservedTime); + + // Record 2: DELTA Bid + Assert.Equal(RecordType.Delta, records[1].Core.Type); + Assert.Equal(RecordSide.Bid, records[1].Core.Side); + Assert.Equal(150L, records[1].Core.Quantity); + + // Record 3: DELTA Ask + Assert.Equal(RecordType.Delta, records[2].Core.Type); + Assert.Equal(RecordSide.Ask, records[2].Core.Side); + Assert.Equal(51000L, records[2].Core.Price); + + // Validate footer + Assert.NotNull(reader.Footer); + Assert.Equal(3L, reader.Footer.Value.RecordCount); + Assert.Equal(2L, reader.Footer.Value.DeltaCount); + } +} diff --git a/tests/Levels.Tests/DataFlow/DataFlowBusTests.cs b/tests/Levels.Tests/DataFlow/DataFlowBusTests.cs new file mode 100644 index 0000000..47eaaf8 --- /dev/null +++ b/tests/Levels.Tests/DataFlow/DataFlowBusTests.cs @@ -0,0 +1,152 @@ +using System.Threading.Channels; +using Levels.Core; +using Levels.Core.Format; +using Levels.DataFlow; +using Levels.Sinks; + +namespace Levels.Tests.DataFlow; + +public class DataFlowBusTests : IDisposable +{ + private readonly string _tempDir; + + public DataFlowBusTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + [Fact] + public async Task MultipleHandlers_AllReceivePublishedEvents() + { + var handler1 = new TestDataFlowHandler(); + var handler2 = new TestDataFlowHandler(); + + await using var bus = new DataFlowBus([handler1, handler2]); + await bus.StartAsync(CancellationToken.None); + + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + var record = new RawRecord + { + Core = new CoreRecordLayout + { + ObservedTime = 1000, + PriceStreamId = streamId.Value, + Price = 50000, + Quantity = 100, + Type = RecordType.Delta, + Side = RecordSide.Bid, + }, + RecordBytes = ReadOnlyMemory.Empty, + }; + + bus.PublishRecordWritten(streamId, record); + bus.PublishFileSealed(new SealedFileInfo("test.raw", streamId, 1, 1, 1000, 2000)); + + await bus.StopAsync(CancellationToken.None); + + Assert.Single(handler1.RecordsWritten); + Assert.Single(handler2.RecordsWritten); + Assert.Single(handler1.FilesSealed); + Assert.Single(handler2.FilesSealed); + } + + [Fact] + public async Task FullChannel_RecordWritten_DropsInsteadOfThrowing() + { + var slowHandler = new TestDataFlowHandler(processingDelay: TimeSpan.FromSeconds(10)); + + await using var bus = new DataFlowBus([slowHandler], channelCapacity: 2); + await bus.StartAsync(CancellationToken.None); + + var streamId = PriceStreamId.FromSymbol("ETH-USD"); + var record = new RawRecord + { + Core = new CoreRecordLayout + { + ObservedTime = 1000, + PriceStreamId = streamId.Value, + Type = RecordType.Delta, + Side = RecordSide.Bid, + }, + RecordBytes = ReadOnlyMemory.Empty, + }; + + // Fill the channel — should NOT throw, should drop and increment counter + for (var i = 0; i < 100; i++) + bus.PublishRecordWritten(streamId, record); + + Assert.True(bus.DroppedRecordCount > 0, "Some records should have been dropped"); + + await bus.StopAsync(CancellationToken.None); + } + + [Fact] + public async Task FullChannel_FileSealed_BlocksUntilDrained() + { + var handler = new TestDataFlowHandler(processingDelay: TimeSpan.FromMilliseconds(50)); + + await using var bus = new DataFlowBus([handler], channelCapacity: 1); + await bus.StartAsync(CancellationToken.None); + + var streamId = PriceStreamId.FromSymbol("ETH-USD"); + + // Fill the channel with a record first + var record = new RawRecord + { + Core = new CoreRecordLayout + { + ObservedTime = 1000, + PriceStreamId = streamId.Value, + Type = RecordType.Delta, + Side = RecordSide.Bid, + }, + RecordBytes = ReadOnlyMemory.Empty, + }; + bus.PublishRecordWritten(streamId, record); + + // PublishFileSealedAsync should block until there's room, not throw + await bus.PublishFileSealedAsync(new SealedFileInfo("test.raw", streamId, 1, 1, 1000, 2000)); + + await bus.StopAsync(CancellationToken.None); + + Assert.Single(handler.FilesSealed); + } + + [Fact] + public async Task RecordWritten_DropsDoNotCrashSink() + { + var slowHandler = new TestDataFlowHandler(processingDelay: TimeSpan.FromSeconds(10)); + + await using var bus = new DataFlowBus([slowHandler], channelCapacity: 2); + await bus.StartAsync(CancellationToken.None); + + var config = new SinkConfig + { + OutputPath = _tempDir, + DataFlowBus = bus, + }; + var sink = new PriceStreamSink(config); + await sink.StartAsync(CancellationToken.None); + + // Write enough events — should not crash even with full channel + for (var i = 0; i < 20; i++) + { + await sink.Writer.WriteAsync(new RawMarketEvent( + "binance", "BTC-USD", 1000 + i, 50000, 100, + RecordType.Delta, RecordSide.Bid)); + } + + await sink.StopAsync(CancellationToken.None); + await bus.StopAsync(CancellationToken.None); + + // Records were dropped but ingestion continued + Assert.True(bus.DroppedRecordCount > 0); + } +} diff --git a/tests/Levels.Tests/DataFlow/TestDataFlowHandler.cs b/tests/Levels.Tests/DataFlow/TestDataFlowHandler.cs new file mode 100644 index 0000000..d041121 --- /dev/null +++ b/tests/Levels.Tests/DataFlow/TestDataFlowHandler.cs @@ -0,0 +1,40 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.DataFlow; + +namespace Levels.Tests.DataFlow; + +internal sealed class TestDataFlowHandler : IDataFlowHandler +{ + private readonly TimeSpan _processingDelay; + + public List<(PriceStreamId Stream, RawRecord Record)> RecordsWritten { get; } = []; + public List FilesSealed { get; } = []; + public List AggsCreated { get; } = []; + + public TestDataFlowHandler(TimeSpan? processingDelay = null) + { + _processingDelay = processingDelay ?? TimeSpan.Zero; + } + + public async ValueTask OnRecordWritten(PriceStreamId stream, RawRecord record, CancellationToken ct) + { + if (_processingDelay > TimeSpan.Zero) + await Task.Delay(_processingDelay, ct); + RecordsWritten.Add((stream, record)); + } + + public async ValueTask OnFileSealed(SealedFileInfo file, CancellationToken ct) + { + if (_processingDelay > TimeSpan.Zero) + await Task.Delay(_processingDelay, ct); + FilesSealed.Add(file); + } + + public async ValueTask OnAggCreated(AggFileInfo file, CancellationToken ct) + { + if (_processingDelay > TimeSpan.Zero) + await Task.Delay(_processingDelay, ct); + AggsCreated.Add(file); + } +} diff --git a/tests/Levels.Tests/Diagnostics/MetricsRecordingTests.cs b/tests/Levels.Tests/Diagnostics/MetricsRecordingTests.cs new file mode 100644 index 0000000..2429e93 --- /dev/null +++ b/tests/Levels.Tests/Diagnostics/MetricsRecordingTests.cs @@ -0,0 +1,153 @@ +using System.Diagnostics.Metrics; +using Levels.Core; +using Levels.Core.Diagnostics; +using Levels.Core.Format; +using Levels.DataFlow; +using Levels.Query; +using Levels.Sinks; +using Levels.Tests.DataFlow; + +namespace Levels.Tests.Diagnostics; + +public class MetricsRecordingTests : IDisposable +{ + private readonly string _tempDir; + private readonly MeterListener _listener; + private readonly List<(string Name, long Value)> _longMeasurements = []; + private readonly List<(string Name, double Value)> _doubleMeasurements = []; + + public MetricsRecordingTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_metrics_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + + _listener = new MeterListener(); + _listener.InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name == "Levels") + listener.EnableMeasurementEvents(instrument); + }; + _listener.SetMeasurementEventCallback((instrument, value, tags, state) => + { + lock (_longMeasurements) + _longMeasurements.Add((instrument.Name, value)); + }); + _listener.SetMeasurementEventCallback((instrument, value, tags, state) => + { + lock (_doubleMeasurements) + _doubleMeasurements.Add((instrument.Name, value)); + }); + _listener.SetMeasurementEventCallback((instrument, value, tags, state) => + { + lock (_longMeasurements) + _longMeasurements.Add((instrument.Name, value)); + }); + _listener.Start(); + } + + public void Dispose() + { + _listener.Dispose(); + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + [Fact] + public async Task WriteAsync_RecordsWrittenAndLatency() + { + var config = new SinkConfig { OutputPath = _tempDir }; + var sink = new PriceStreamSink(config); + await sink.StartAsync(CancellationToken.None); + + await sink.Writer.WriteAsync(new RawMarketEvent( + "binance", "BTC-USD", 1000, 50000, 100, + RecordType.Delta, RecordSide.Bid)); + + await sink.StopAsync(CancellationToken.None); + _listener.RecordObservableInstruments(); + + lock (_longMeasurements) + { + Assert.Contains(_longMeasurements, m => m.Name == "pricestorage.records.written" && m.Value > 0); + } + lock (_doubleMeasurements) + { + Assert.Contains(_doubleMeasurements, m => m.Name == "pricestorage.record.write_latency" && m.Value >= 0); + } + } + + [Fact] + public async Task SealFile_FileSealedCounterIncrements() + { + var config = new SinkConfig { OutputPath = _tempDir }; + var sink = new PriceStreamSink(config); + await sink.StartAsync(CancellationToken.None); + + await sink.Writer.WriteAsync(new RawMarketEvent( + "binance", "BTC-USD", 1000, 50000, 100, + RecordType.Delta, RecordSide.Bid)); + + // StopAsync seals all open files + await sink.StopAsync(CancellationToken.None); + _listener.RecordObservableInstruments(); + + lock (_longMeasurements) + { + Assert.Contains(_longMeasurements, m => m.Name == "pricestorage.files.sealed" && m.Value > 0); + } + } + + [Fact] + public async Task DataFlowBackpressure_IncrementedOnDrop() + { + var slowHandler = new TestDataFlowHandler(processingDelay: TimeSpan.FromSeconds(10)); + await using var bus = new DataFlowBus([slowHandler], channelCapacity: 2); + await bus.StartAsync(CancellationToken.None); + + var streamId = PriceStreamId.FromSymbol("ETH-USD"); + var record = new RawRecord + { + Core = new CoreRecordLayout + { + ObservedTime = 1000, + PriceStreamId = streamId.Value, + Type = RecordType.Delta, + Side = RecordSide.Bid, + }, + RecordBytes = ReadOnlyMemory.Empty, + }; + + for (var i = 0; i < 100; i++) + bus.PublishRecordWritten(streamId, record); + + await bus.StopAsync(CancellationToken.None); + _listener.RecordObservableInstruments(); + + lock (_longMeasurements) + { + Assert.Contains(_longMeasurements, m => m.Name == "pricestorage.dataflow.backpressure" && m.Value > 0); + } + } + + [Fact] + public void QueryResolve_RecordsLatencyAndCount() + { + var fileIndex = new FileIndex(); + var queryConfig = new QueryConfig { DataPath = _tempDir, CompactionWindow = TimeSpan.FromHours(1) }; + var queryLayer = new QueryLayer(fileIndex, queryConfig); + + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + queryLayer.Resolve(streamId, 0, long.MaxValue); + + _listener.RecordObservableInstruments(); + + lock (_doubleMeasurements) + { + Assert.Contains(_doubleMeasurements, m => m.Name == "pricestorage.query.latency"); + } + lock (_longMeasurements) + { + Assert.Contains(_longMeasurements, m => m.Name == "pricestorage.queries.executed" && m.Value > 0); + } + } +} diff --git a/tests/Levels.Tests/Export/CsvExportAdapterTests.cs b/tests/Levels.Tests/Export/CsvExportAdapterTests.cs new file mode 100644 index 0000000..fd09fe6 --- /dev/null +++ b/tests/Levels.Tests/Export/CsvExportAdapterTests.cs @@ -0,0 +1,66 @@ +using System.Text; +using Levels.Core.Format; +using Levels.Export; + +namespace Levels.Tests.Export; + +public class CsvExportAdapterTests +{ + [Fact] + public async Task Export_WritesHeaderAndRecords() + { + var adapter = new CsvExportAdapter(); + using var ms = new MemoryStream(); + + var records = CreateTestRecords(3); + await adapter.ExportAsync(records, ms); + + ms.Position = 0; + var csv = Encoding.UTF8.GetString(ms.ToArray()); + var lines = csv.Split('\n', StringSplitOptions.RemoveEmptyEntries); + + Assert.Equal(4, lines.Length); // 1 header + 3 records + Assert.StartsWith("ObservedTime,", lines[0]); + } + + [Fact] + public async Task Export_EmptyRecords_WritesHeaderOnly() + { + var adapter = new CsvExportAdapter(); + using var ms = new MemoryStream(); + + await adapter.ExportAsync(EmptyRecords(), ms); + + ms.Position = 0; + var csv = Encoding.UTF8.GetString(ms.ToArray()); + var lines = csv.Split('\n', StringSplitOptions.RemoveEmptyEntries); + + Assert.Single(lines); // header only + } + + private static async IAsyncEnumerable CreateTestRecords(int count) + { + for (int i = 0; i < count; i++) + { + yield return new ExportRecord( + ObservedTime: 1000 + i, + WriteTimestamp: 2000 + i, + PriceStreamId: 12345, + Price: 50000 + i, + Quantity: 100 + i, + Type: RecordType.Delta, + Side: RecordSide.Bid, + Sequence: (uint)i, + Level: 0, + Flags: 0, + OrderId: null); + await Task.Yield(); + } + } + + private static async IAsyncEnumerable EmptyRecords() + { + await Task.Yield(); + yield break; + } +} diff --git a/tests/Levels.Tests/Export/ExportPipelineTests.cs b/tests/Levels.Tests/Export/ExportPipelineTests.cs new file mode 100644 index 0000000..c15423e --- /dev/null +++ b/tests/Levels.Tests/Export/ExportPipelineTests.cs @@ -0,0 +1,79 @@ +using System.Text; +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Export; +using Levels.Query; +using Levels.Sinks; + +namespace Levels.Tests.Export; + +public class ExportPipelineTests : IDisposable +{ + private readonly string _tempDir; + + public ExportPipelineTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + [Fact] + public async Task ExportCsv_WritesRecords() + { + // Ingest some data + var config = new SinkConfig { OutputPath = _tempDir }; + var sink = new PriceStreamSink(config); + await sink.StartAsync(CancellationToken.None); + await RandomDataAdapter.GenerateAsync(sink.Writer, "test-exchange", ["BTC-USD"], levelsPerSide: 2, deltasPerSymbol: 5); + await sink.StopAsync(CancellationToken.None); + + // Set up export pipeline + var fileIndex = new FileIndex(); + fileIndex.LoadFromDisk(_tempDir); + var queryLayer = new QueryLayer(fileIndex, new QueryConfig { DataPath = _tempDir }); + var pipeline = new ExportPipeline(queryLayer, [new CsvExportAdapter()]); + + var streamId = PriceStreamId.FromVenueSymbol("test-exchange", "BTC-USD"); + using var ms = new MemoryStream(); + await pipeline.ExportAsync(streamId, 0, long.MaxValue, "csv", ms); + + ms.Position = 0; + var csv = Encoding.UTF8.GetString(ms.ToArray()); + var lines = csv.Split('\n', StringSplitOptions.RemoveEmptyEntries); + + // header + at least some records + Assert.True(lines.Length > 1, $"Expected more than 1 line, got {lines.Length}"); + } + + [Fact] + public async Task Export_UnsupportedFormat_Throws() + { + var fileIndex = new FileIndex(); + var queryLayer = new QueryLayer(fileIndex, new QueryConfig { DataPath = _tempDir }); + var pipeline = new ExportPipeline(queryLayer, [new CsvExportAdapter()]); + + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + using var ms = new MemoryStream(); + + await Assert.ThrowsAsync(() => + pipeline.ExportAsync(streamId, 0, long.MaxValue, "xml", ms)); + } + + [Fact] + public void SupportedFormats_ReturnsRegisteredFormats() + { + var fileIndex = new FileIndex(); + var queryLayer = new QueryLayer(fileIndex, new QueryConfig { DataPath = _tempDir }); + var pipeline = new ExportPipeline(queryLayer, [new CsvExportAdapter(), new ParquetExportAdapter()]); + + Assert.Contains("csv", pipeline.SupportedFormats); + Assert.Contains("parquet", pipeline.SupportedFormats); + } +} diff --git a/tests/Levels.Tests/Format/Crc32Tests.cs b/tests/Levels.Tests/Format/Crc32Tests.cs new file mode 100644 index 0000000..9a96134 --- /dev/null +++ b/tests/Levels.Tests/Format/Crc32Tests.cs @@ -0,0 +1,49 @@ +using Levels.Core.Format; + +namespace Levels.Tests.Format; + +public class Crc32Tests +{ + [Fact] + public void Compute_SameInput_ReturnsSameResult() + { + byte[] data = [1, 2, 3, 4, 5]; + Assert.Equal(Crc32Util.Compute(data), Crc32Util.Compute(data)); + } + + [Fact] + public void Compute_DifferentInput_ReturnsDifferentResult() + { + byte[] data1 = [1, 2, 3]; + byte[] data2 = [4, 5, 6]; + Assert.NotEqual(Crc32Util.Compute(data1), Crc32Util.Compute(data2)); + } + + [Fact] + public void ComputeRecord_CorePlusExt_DiffersFromCoreOnly() + { + byte[] core = new byte[52]; + core[0] = 0xAB; + + byte[] ext = [0xDE, 0xAD]; + + var crcCoreOnly = Crc32Util.ComputeRecord(core, ReadOnlySpan.Empty); + var crcWithExt = Crc32Util.ComputeRecord(core, ext); + + Assert.NotEqual(crcCoreOnly, crcWithExt); + } + + [Fact] + public void Incremental_MatchesSingleShot() + { + byte[] data = [10, 20, 30, 40, 50]; + var singleShot = Crc32Util.Compute(data); + + var incremental = Crc32Util.CreateIncremental(); + incremental.Append(data.AsSpan(0, 3)); + incremental.Append(data.AsSpan(3)); + var incrementalResult = incremental.GetCurrentHashAsUInt32(); + + Assert.Equal(singleShot, incrementalResult); + } +} diff --git a/tests/Levels.Tests/Format/FileFooterTests.cs b/tests/Levels.Tests/Format/FileFooterTests.cs new file mode 100644 index 0000000..01550d1 --- /dev/null +++ b/tests/Levels.Tests/Format/FileFooterTests.cs @@ -0,0 +1,47 @@ +using Levels.Core.Format; + +namespace Levels.Tests.Format; + +public class FileFooterTests +{ + [Fact] + public void RoundTrip_WriteThenRead_IsEqual() + { + var original = new FileFooter + { + RecordCount = 1000, + DeltaCount = 500, + FirstWriteTimestamp = 111, + LastWriteTimestamp = 999, + FirstObservedTime = 100, + LastObservedTime = 900, + FileCrc32 = 0xCAFEBABE, + MagicEnd = Constants.FooterMagicUInt64, + }; + + Span buffer = stackalloc byte[Constants.FooterSize]; + FileFooter.WriteTo(buffer, in original); + var readBack = FileFooter.ReadFrom(buffer); + + Assert.Equal(original.RecordCount, readBack.RecordCount); + Assert.Equal(original.DeltaCount, readBack.DeltaCount); + Assert.Equal(original.FirstWriteTimestamp, readBack.FirstWriteTimestamp); + Assert.Equal(original.LastWriteTimestamp, readBack.LastWriteTimestamp); + Assert.Equal(original.FirstObservedTime, readBack.FirstObservedTime); + Assert.Equal(original.LastObservedTime, readBack.LastObservedTime); + Assert.Equal(original.FileCrc32, readBack.FileCrc32); + Assert.Equal(original.MagicEnd, readBack.MagicEnd); + } + + [Fact] + public void FooterSize_Is64Bytes() + { + Assert.Equal(64, System.Runtime.InteropServices.Marshal.SizeOf()); + } + + [Fact] + public void FooterMagic_MatchExpected() + { + Assert.Equal(BitConverter.ToUInt64("LEVEND01"u8), Constants.FooterMagicUInt64); + } +} diff --git a/tests/Levels.Tests/Format/FileHeaderTests.cs b/tests/Levels.Tests/Format/FileHeaderTests.cs new file mode 100644 index 0000000..044d8a0 --- /dev/null +++ b/tests/Levels.Tests/Format/FileHeaderTests.cs @@ -0,0 +1,47 @@ +using Levels.Core.Format; + +namespace Levels.Tests.Format; + +public class FileHeaderTests +{ + [Fact] + public void RoundTrip_WriteThenRead_IsEqual() + { + var original = new FileHeader + { + Magic = Constants.HeaderMagicUInt64, + Version = Constants.FormatVersion, + FileType = FileType.Raw, + SchemaId = 0xDEADBEEF, + PriceStreamId = 42, + CreatedAt = 1234567890L, + PriceScale = 8, + QuantityScale = 2, + }; + + Span buffer = stackalloc byte[Constants.HeaderSize]; + FileHeader.WriteTo(buffer, in original); + var readBack = FileHeader.ReadFrom(buffer); + + Assert.Equal(original.Magic, readBack.Magic); + Assert.Equal(original.Version, readBack.Version); + Assert.Equal(original.FileType, readBack.FileType); + Assert.Equal(original.SchemaId, readBack.SchemaId); + Assert.Equal(original.PriceStreamId, readBack.PriceStreamId); + Assert.Equal(original.CreatedAt, readBack.CreatedAt); + Assert.Equal(original.PriceScale, readBack.PriceScale); + Assert.Equal(original.QuantityScale, readBack.QuantityScale); + } + + [Fact] + public void HeaderSize_Is128Bytes() + { + Assert.Equal(128, System.Runtime.InteropServices.Marshal.SizeOf()); + } + + [Fact] + public void MagicBytes_MatchExpected() + { + Assert.Equal(BitConverter.ToUInt64("LEVELS01"u8), Constants.HeaderMagicUInt64); + } +} diff --git a/tests/Levels.Tests/Format/SchemaIdTests.cs b/tests/Levels.Tests/Format/SchemaIdTests.cs new file mode 100644 index 0000000..a7f0468 --- /dev/null +++ b/tests/Levels.Tests/Format/SchemaIdTests.cs @@ -0,0 +1,30 @@ +using Levels.Core.Format; + +namespace Levels.Tests.Format; + +public class SchemaIdTests +{ + [Fact] + public void Compute_KnownInput_ReturnsExpectedHash() + { + // FNV-1a of empty string + Assert.Equal(2166136261u, SchemaId.Compute(ReadOnlySpan.Empty)); + } + + [Fact] + public void Compute_DifferentInputs_ProduceDifferentHashes() + { + var hash1 = SchemaId.Compute("hello"u8); + var hash2 = SchemaId.Compute("world"u8); + Assert.NotEqual(hash1, hash2); + } + + [Fact] + public void Compute_SameInput_ProducesSameHash() + { + var hash1 = SchemaId.Compute("test data"u8); + var hash2 = SchemaId.Compute("test data"u8); + Assert.Equal(hash1, hash2); + } + +} diff --git a/tests/Levels.Tests/Hints/HintsDbSyncHandlerTests.cs b/tests/Levels.Tests/Hints/HintsDbSyncHandlerTests.cs new file mode 100644 index 0000000..7f88deb --- /dev/null +++ b/tests/Levels.Tests/Hints/HintsDbSyncHandlerTests.cs @@ -0,0 +1,59 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Hints; +using Levels.Query; + +namespace Levels.Tests.Hints; + +public class HintsDbSyncHandlerTests : IDisposable +{ + private readonly string _tempDir; + private readonly string _dbPath; + + public HintsDbSyncHandlerTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + _dbPath = Path.Combine(_tempDir, "hints.db"); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + [Fact] + public async Task OnFileSealed_InsertsHint() + { + using var db = new HintsDb(_dbPath); + var handler = new HintsDbSyncHandler(db, _tempDir); + + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + var filePath = Path.Combine(_tempDir, "exchange", streamId.Value.ToString(), "test.raw"); + + var sealed_ = new SealedFileInfo(filePath, streamId, 10, 5, 100, 200); + await handler.OnFileSealed(sealed_, CancellationToken.None); + + var all = db.GetAll(); + Assert.Single(all); + Assert.Equal(FileType.Raw, all[0].FileType); + } + + [Fact] + public async Task OnAggCreated_InsertsHint() + { + using var db = new HintsDb(_dbPath); + var handler = new HintsDbSyncHandler(db, _tempDir); + + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + var filePath = Path.Combine(_tempDir, "exchange", streamId.Value.ToString(), "test.agg"); + + var aggInfo = new AggFileInfo(filePath, streamId, 20, 10, 2, 100, 200, []); + await handler.OnAggCreated(aggInfo, CancellationToken.None); + + var all = db.GetAll(); + Assert.Single(all); + Assert.Equal(FileType.Agg, all[0].FileType); + } +} diff --git a/tests/Levels.Tests/Hints/HintsDbTests.cs b/tests/Levels.Tests/Hints/HintsDbTests.cs new file mode 100644 index 0000000..a745c74 --- /dev/null +++ b/tests/Levels.Tests/Hints/HintsDbTests.cs @@ -0,0 +1,108 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Hints; +using Levels.Query; +using Levels.Sinks; + +namespace Levels.Tests.Hints; + +public class HintsDbTests : IDisposable +{ + private readonly string _tempDir; + private readonly string _dbPath; + + public HintsDbTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + _dbPath = Path.Combine(_tempDir, "hints.db"); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + [Fact] + public void UpsertAndGetAll_PersistsEntries() + { + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + var entry = new FileIndexEntry("test.raw", streamId, "binance", FileType.Raw, 100, 200, 10); + + using (var db = new HintsDb(_dbPath)) + { + db.Upsert(entry); + } + + using (var db = new HintsDb(_dbPath)) + { + var all = db.GetAll(); + Assert.Single(all); + Assert.Equal("test.raw", all[0].FilePath); + Assert.Equal(streamId, all[0].PriceStreamId); + Assert.Equal(FileType.Raw, all[0].FileType); + } + } + + [Fact] + public void Upsert_UpdatesExistingEntry() + { + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + using var db = new HintsDb(_dbPath); + + db.Upsert(new FileIndexEntry("test.raw", streamId, "binance", FileType.Raw, 100, 200, 10)); + db.Upsert(new FileIndexEntry("test.raw", streamId, "binance", FileType.Raw, 100, 300, 20)); + + var all = db.GetAll(); + Assert.Single(all); + Assert.Equal(300, all[0].LastObservedTime); + Assert.Equal(20, all[0].RecordCount); + } + + [Fact] + public void Remove_DeletesEntry() + { + using var db = new HintsDb(_dbPath); + var streamId = PriceStreamId.FromSymbol("ETH-USD"); + + db.Upsert(new FileIndexEntry("test.raw", streamId, "binance", FileType.Raw, 100, 200, 10)); + db.Remove("test.raw"); + + Assert.Empty(db.GetAll()); + } + + [Fact] + public void LoadInto_PopulatesFileIndex() + { + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + + // Create actual file so LoadInto checks existence + var filePath = Path.Combine(_tempDir, "test.raw"); + File.WriteAllText(filePath, ""); + + using var db = new HintsDb(_dbPath); + db.Upsert(new FileIndexEntry(filePath, streamId, "binance", FileType.Raw, 100, 200, 10)); + + var index = new FileIndex(); + db.LoadInto(index); + + var results = index.Query(streamId, 0, 300); + Assert.Single(results); + } + + [Fact] + public void LoadInto_SkipsMissingFiles() + { + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + + using var db = new HintsDb(_dbPath); + db.Upsert(new FileIndexEntry("/nonexistent/file.raw", streamId, "binance", FileType.Raw, 100, 200, 10)); + + var index = new FileIndex(); + db.LoadInto(index); + + var results = index.Query(streamId, 0, 300); + Assert.Empty(results); + } +} diff --git a/tests/Levels.Tests/Hosting/DataSinkAdapterTests.cs b/tests/Levels.Tests/Hosting/DataSinkAdapterTests.cs new file mode 100644 index 0000000..9e63de2 --- /dev/null +++ b/tests/Levels.Tests/Hosting/DataSinkAdapterTests.cs @@ -0,0 +1,103 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Hosting; +using Levels.Sinks; + +namespace Levels.Tests.Hosting; + +public class DataSinkAdapterTests : IDisposable +{ + private readonly string _tempDir; + + public DataSinkAdapterTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + public record TestMarketEvent( + string Venue, + string Symbol, + long ObservedTime, + long Price, + long Quantity, + RecordType Type, + RecordSide Side); + + [Fact] + public async Task DataSinkAdapter_MapsConventionBasedProperties() + { + var config = new SinkConfig { OutputPath = _tempDir }; + var sink = new PriceStreamSink(config); + await sink.StartAsync(CancellationToken.None); + + var adapter = new DataSinkAdapter(sink); + + var evt = new TestMarketEvent( + "test-exchange", "BTC-USD", + DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() * 1_000_000, + 50000, 100, RecordType.Snap, RecordSide.Bid); + + await adapter.WriteAsync(evt); + + await sink.StopAsync(CancellationToken.None); + + // Verify a file was written + var rawFiles = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories); + Assert.NotEmpty(rawFiles); + } + + [Fact] + public async Task DataSinkAdapter_WritesMultipleRecords() + { + var config = new SinkConfig { OutputPath = _tempDir }; + var sink = new PriceStreamSink(config); + await sink.StartAsync(CancellationToken.None); + + var adapter = new DataSinkAdapter(sink); + var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() * 1_000_000; + + for (int i = 0; i < 10; i++) + { + await adapter.WriteAsync(new TestMarketEvent( + "test-exchange", "ETH-USD", now + i * 1_000_000, + 3000 + i, 50, RecordType.Delta, RecordSide.Ask)); + } + + await sink.StopAsync(CancellationToken.None); + + var rawFiles = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories); + Assert.NotEmpty(rawFiles); + } + + [Fact] + public async Task DataSinkAdapter_WithMultiplePartitions_DoesNotThrow() + { + var config = new SinkConfig { OutputPath = _tempDir, ConsumerPartitions = 2 }; + var sink = new PriceStreamSink(config); + await sink.StartAsync(CancellationToken.None); + + var adapter = new DataSinkAdapter(sink); + var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() * 1_000_000; + + // This should not throw InvalidOperationException + await adapter.WriteAsync(new TestMarketEvent( + "test-exchange", "BTC-USD", now, + 50000, 100, RecordType.Snap, RecordSide.Bid)); + + await adapter.WriteAsync(new TestMarketEvent( + "test-exchange", "ETH-USD", now + 1_000_000, + 3000, 50, RecordType.Snap, RecordSide.Bid)); + + await sink.StopAsync(CancellationToken.None); + + var rawFiles = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories); + Assert.NotEmpty(rawFiles); + } +} diff --git a/tests/Levels.Tests/Hosting/OpenTelemetryExtensionsTests.cs b/tests/Levels.Tests/Hosting/OpenTelemetryExtensionsTests.cs new file mode 100644 index 0000000..ee8940f --- /dev/null +++ b/tests/Levels.Tests/Hosting/OpenTelemetryExtensionsTests.cs @@ -0,0 +1,54 @@ +using Microsoft.Extensions.DependencyInjection; +using OpenTelemetry.Metrics; +using Levels.Core; +using Levels.Hosting; + +namespace Levels.Tests.Hosting; + +public class OpenTelemetryExtensionsTests : IDisposable +{ + private readonly string _tempDir; + + public OpenTelemetryExtensionsTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_otel_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + [Fact] + public void EnableMetrics_RegistersMeterProvider() + { + var services = new ServiceCollection(); + services.AddLevels(opts => + { + opts.DataPath = _tempDir; + opts.EnableMetrics = true; + }); + + using var provider = services.BuildServiceProvider(); + var meterProvider = provider.GetService(); + Assert.NotNull(meterProvider); + } + + [Fact] + public void MetricsDisabled_DoesNotRegisterMeterProvider() + { + var services = new ServiceCollection(); + services.AddLevels(opts => + { + opts.DataPath = _tempDir; + opts.EnableMetrics = false; + opts.EnableTracing = false; + }); + + using var provider = services.BuildServiceProvider(); + var meterProvider = provider.GetService(); + Assert.Null(meterProvider); + } +} diff --git a/tests/Levels.Tests/Hosting/ServiceCollectionExtensionsTests.cs b/tests/Levels.Tests/Hosting/ServiceCollectionExtensionsTests.cs new file mode 100644 index 0000000..d13ff00 --- /dev/null +++ b/tests/Levels.Tests/Hosting/ServiceCollectionExtensionsTests.cs @@ -0,0 +1,72 @@ +using Microsoft.Extensions.DependencyInjection; +using Levels.Core; +using Levels.Core.Format; +using Levels.DataFlow; +using Levels.Hosting; +using Levels.Query; +using Levels.Sinks; + +namespace Levels.Tests.Hosting; + +public class ServiceCollectionExtensionsTests : IDisposable +{ + private readonly string _tempDir; + + public ServiceCollectionExtensionsTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + public record MyEvent( + string Venue, + string Symbol, + long ObservedTime, + long Price, + long Quantity, + RecordType Type, + RecordSide Side); + + [Fact] + public void AddLevels_RegistersCoreServices() + { + var services = new ServiceCollection(); + services.AddLevels(opts => opts.DataPath = _tempDir); + + var provider = services.BuildServiceProvider(); + + Assert.NotNull(provider.GetService()); + Assert.NotNull(provider.GetService()); + Assert.NotNull(provider.GetService()); + Assert.NotNull(provider.GetService()); + Assert.NotNull(provider.GetService()); + } + + [Fact] + public void AddLevelsDataSink_RegistersIDataSink() + { + var services = new ServiceCollection(); + services.AddLevels(opts => opts.DataPath = _tempDir); + services.AddLevelsDataSink(); + + var provider = services.BuildServiceProvider(); + + var sink = provider.GetService>(); + Assert.NotNull(sink); + Assert.IsType>(sink); + } + + [Fact] + public void AddLevels_ThrowsWhenDataPathMissing() + { + var services = new ServiceCollection(); + Assert.Throws(() => + services.AddLevels(opts => { })); + } +} diff --git a/tests/Levels.Tests/IO/FooterCrcTests.cs b/tests/Levels.Tests/IO/FooterCrcTests.cs new file mode 100644 index 0000000..8300f6a --- /dev/null +++ b/tests/Levels.Tests/IO/FooterCrcTests.cs @@ -0,0 +1,104 @@ +using System.Runtime.InteropServices; +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; + +namespace Levels.Tests.IO; + +public class FooterCrcTests +{ + [Fact] + public async Task SealedFile_HasNonZeroFooterCrc32() + { + using var ms = new MemoryStream(); + + await using (var writer = await BinaryRecordWriter.CreateAsync(ms, FileType.Raw, new PriceStreamId(42))) + { + await writer.WriteRecordAsync(MakeSnap()); + await writer.SealAsync(); + } + + ms.Position = 0; + using var reader = new BinaryRecordReader(ms); + + Assert.NotNull(reader.Footer); + Assert.NotEqual(0u, reader.Footer.Value.FooterCrc32); + } + + [Fact] + public async Task CorruptedFooter_ReturnsIsPartial() + { + using var ms = new MemoryStream(); + + await using (var writer = await BinaryRecordWriter.CreateAsync(ms, FileType.Raw, new PriceStreamId(42))) + { + await writer.WriteRecordAsync(MakeSnap()); + await writer.SealAsync(); + } + + // Corrupt a footer metadata byte (RecordCount at offset 0 of footer) + var footerStart = ms.Length - Constants.FooterSize; + ms.Position = footerStart; + var originalByte = (byte)ms.ReadByte(); + ms.Position = footerStart; + ms.WriteByte((byte)(originalByte ^ 0xFF)); + + ms.Position = 0; + using var reader = new BinaryRecordReader(ms); + + Assert.True(reader.IsPartial, "Corrupted footer should be treated as partial (no valid footer)"); + } + + [Fact] + public void BackwardCompat_ZeroFooterCrc32_AcceptedAsValid() + { + // Build a file with FooterCrc32 = 0 (simulating old format) + using var ms = new MemoryStream(); + + // Write minimal header + Span headerBytes = stackalloc byte[Constants.HeaderSize]; + headerBytes.Clear(); + var header = new FileHeader + { + Magic = Constants.HeaderMagicUInt64, + Version = Constants.FormatVersion, + FileType = FileType.Raw, + PriceStreamId = 42, + CreatedAt = 1000, + RecordSize = (ushort)Constants.CoreRecordSize, + }; + FileHeader.WriteTo(headerBytes, in header); + ms.Write(headerBytes); + + // Write footer with FooterCrc32 = 0 (old file format) + var footer = new FileFooter + { + RecordCount = 0, + MagicEnd = Constants.FooterMagicUInt64, + }; + Span footerBytes = stackalloc byte[Constants.FooterSize]; + footerBytes.Clear(); + FileFooter.WriteTo(footerBytes, in footer); + ms.Write(footerBytes); + + ms.Position = 0; + using var reader = new BinaryRecordReader(ms); + + Assert.False(reader.IsPartial, "Old files with FooterCrc32=0 should be accepted"); + Assert.NotNull(reader.Footer); + } + + private static CoreRecordLayout MakeSnap() + { + return new CoreRecordLayout + { + ObservedTime = 1000, + PriceStreamId = 42, + Price = 12345, + Quantity = 100, + Type = RecordType.Snap, + Side = RecordSide.Bid, + Sequence = 1, + }; + } +} diff --git a/tests/Levels.Tests/IO/FsyncTests.cs b/tests/Levels.Tests/IO/FsyncTests.cs new file mode 100644 index 0000000..6226f3d --- /dev/null +++ b/tests/Levels.Tests/IO/FsyncTests.cs @@ -0,0 +1,41 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; + +namespace Levels.Tests.IO; + +public class FsyncTests +{ + [Fact] + public async Task SealAsync_CallsFlushOnUnderlyingStream() + { + var spy = new FlushSpyStream(); + + await using (var writer = await BinaryRecordWriter.CreateAsync(spy, FileType.Raw, new PriceStreamId(42))) + { + await writer.WriteRecordAsync(new CoreRecordLayout + { + ObservedTime = 1000, + PriceStreamId = 42, + Price = 100, + Quantity = 50, + Type = RecordType.Snap, + Side = RecordSide.Bid, + }); + await writer.SealAsync(); + } + + Assert.True(spy.FlushCalled, "Stream.Flush() should be called during SealAsync"); + } + + private sealed class FlushSpyStream : MemoryStream + { + public bool FlushCalled { get; private set; } + + public override void Flush() + { + FlushCalled = true; + base.Flush(); + } + } +} diff --git a/tests/Levels.Tests/IO/RecordSkipTests.cs b/tests/Levels.Tests/IO/RecordSkipTests.cs new file mode 100644 index 0000000..ec0cba0 --- /dev/null +++ b/tests/Levels.Tests/IO/RecordSkipTests.cs @@ -0,0 +1,62 @@ +using System.Runtime.InteropServices; +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; + +namespace Levels.Tests.IO; + +public class RecordSkipTests +{ + [Fact] + public async Task FixedStride_AllRecordsSameSize() + { + const int recordSize = 88; // 56 core + 32 OrderId + using var ms = new MemoryStream(); + + await using (var writer = await BinaryRecordWriter.CreateAsync( + ms, FileType.Raw, new PriceStreamId(42), recordSize: recordSize)) + { + // Record 1: with order id + await writer.WriteRecordAsync(new CoreRecordLayout + { + ObservedTime = 1000, + PriceStreamId = 42, + Price = 111, + Type = RecordType.Snap, + Side = RecordSide.Bid, + Sequence = 1, + }, "order-1"u8.ToArray()); + + // Record 2: no order id + await writer.WriteRecordAsync(new CoreRecordLayout + { + ObservedTime = 2000, + PriceStreamId = 42, + Price = 222, + Type = RecordType.Delta, + Side = RecordSide.Ask, + Sequence = 2, + }); + + await writer.SealAsync(); + } + + // Verify all records have the same stride + long dataSize = ms.Length - Constants.HeaderSize - Constants.FooterSize; + Assert.Equal(recordSize * 2, dataSize); + + // Read back and verify data + ms.Position = 0; + using var reader = new BinaryRecordReader(ms); + var records = reader.ReadRecords().ToList(); + + Assert.Equal(2, records.Count); + Assert.Equal(111, records[0].Core.Price); + Assert.Equal(222, records[1].Core.Price); + Assert.Equal(RecordType.Delta, records[1].Core.Type); + + // Both records have the same RecordBytes length + Assert.Equal(recordSize, records[0].RecordBytes.Length); + Assert.Equal(recordSize, records[1].RecordBytes.Length); + } +} diff --git a/tests/Levels.Tests/IO/RoundTripTests.cs b/tests/Levels.Tests/IO/RoundTripTests.cs new file mode 100644 index 0000000..ce43852 --- /dev/null +++ b/tests/Levels.Tests/IO/RoundTripTests.cs @@ -0,0 +1,342 @@ +using System.Runtime.InteropServices; +using System.Text; +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; + +namespace Levels.Tests.IO; + +public class RoundTripTests +{ + private const int TestRecordSize = 88; // 56-byte core + 32-byte OrderId + + private static CoreRecordLayout MakeSnap(long observedTime = 1000, long price = 12345, long quantity = 100) + { + return new CoreRecordLayout + { + ObservedTime = observedTime, + PriceStreamId = 42, + Price = price, + Quantity = quantity, + Type = RecordType.Snap, + Side = RecordSide.Bid, + Sequence = 1, + Level = 0, + Flags = 0, + }; + } + + private static CoreRecordLayout MakeDelta(long observedTime = 2000, long price = 12350, long quantity = 50) + { + return new CoreRecordLayout + { + ObservedTime = observedTime, + PriceStreamId = 42, + Price = price, + Quantity = quantity, + Type = RecordType.Delta, + Side = RecordSide.Ask, + Sequence = 2, + Level = 1, + Flags = 0, + }; + } + + [Fact] + public async Task WriteSnapRecord_ReadBack_ByteForByteEqual() + { + using var ms = new MemoryStream(); + + await using (var writer = await BinaryRecordWriter.CreateAsync(ms, FileType.Raw, new PriceStreamId(42))) + { + await writer.WriteRecordAsync(MakeSnap()); + await writer.SealAsync(); + } + + ms.Position = 0; + using var reader = new BinaryRecordReader(ms); + var records = reader.ReadRecords().ToList(); + + Assert.Single(records); + + // Verify the writer-mutated fields are present + Assert.True(records[0].Core.WriteTimestamp > 0); + Assert.True(records[0].Core.Crc32 != 0); + Assert.Equal(0, records[0].Core.Reserved); + } + + [Fact] + public async Task WriteDeltaRecord_ReadBack_ByteForByteEqual() + { + using var ms = new MemoryStream(); + + await using (var writer = await BinaryRecordWriter.CreateAsync(ms, FileType.Raw, new PriceStreamId(42))) + { + await writer.WriteRecordAsync(MakeDelta()); + await writer.SealAsync(); + } + + ms.Position = 0; + using var reader = new BinaryRecordReader(ms); + var records = reader.ReadRecords().ToList(); + + Assert.Single(records); + Assert.Equal(RecordType.Delta, records[0].Core.Type); + Assert.Equal(RecordSide.Ask, records[0].Core.Side); + } + + [Fact] + public async Task WriteRecordWithOrderId_ReadBack_OrderIdRoundTrips() + { + using var ms = new MemoryStream(); + var orderId = "test-order-123"u8.ToArray(); + + await using (var writer = await BinaryRecordWriter.CreateAsync( + ms, FileType.Raw, new PriceStreamId(42), recordSize: TestRecordSize)) + { + await writer.WriteRecordAsync(MakeSnap(), orderId); + await writer.SealAsync(); + } + + ms.Position = 0; + using var reader = new BinaryRecordReader(ms); + var records = reader.ReadRecords().ToList(); + + Assert.Single(records); + + // Verify OrderId round-trips + var readOrderId = records[0].OrderId; + Assert.Equal(32, readOrderId.Length); // fixed 32 bytes + // Trim trailing nulls to get the original string + var span = readOrderId.Span; + int len = span.Length; + while (len > 0 && span[len - 1] == 0) len--; + Assert.Equal("test-order-123", Encoding.UTF8.GetString(span[..len])); + } + + [Fact] + public async Task WriteMultipleRecords_ReadAll_CorrectCount() + { + using var ms = new MemoryStream(); + + await using (var writer = await BinaryRecordWriter.CreateAsync(ms, FileType.Raw, new PriceStreamId(42))) + { + await writer.WriteRecordAsync(MakeSnap(1000)); + await writer.WriteRecordAsync(MakeDelta(2000)); + await writer.WriteRecordAsync(MakeDelta(3000)); + await writer.SealAsync(); + } + + ms.Position = 0; + using var reader = new BinaryRecordReader(ms); + var records = reader.ReadRecords().ToList(); + + Assert.Equal(3, records.Count); + Assert.Equal(RecordType.Snap, records[0].Core.Type); + Assert.Equal(RecordType.Delta, records[1].Core.Type); + Assert.Equal(RecordType.Delta, records[2].Core.Type); + } + + [Fact] + public async Task WriteAndSeal_FooterHasCorrectStats() + { + using var ms = new MemoryStream(); + + await using (var writer = await BinaryRecordWriter.CreateAsync(ms, FileType.Raw, new PriceStreamId(42))) + { + await writer.WriteRecordAsync(MakeSnap(1000)); + await writer.WriteRecordAsync(MakeDelta(2000)); + await writer.WriteRecordAsync(MakeDelta(3000)); + await writer.SealAsync(); + } + + ms.Position = 0; + using var reader = new BinaryRecordReader(ms); + + Assert.False(reader.IsPartial); + Assert.NotNull(reader.Footer); + + var footer = reader.Footer!.Value; + Assert.Equal(3, footer.RecordCount); + Assert.Equal(2, footer.DeltaCount); + Assert.Equal(Constants.FooterMagicUInt64, footer.MagicEnd); + Assert.Equal(1000, footer.FirstObservedTime); + Assert.Equal(3000, footer.LastObservedTime); + Assert.True(footer.FirstWriteTimestamp > 0); + Assert.True(footer.LastWriteTimestamp >= footer.FirstWriteTimestamp); + Assert.True(footer.FileCrc32 != 0); + } + + [Fact] + public async Task WriteThenRead_Crc32Validates() + { + using var ms = new MemoryStream(); + var orderId = "order-abc"u8.ToArray(); + + await using (var writer = await BinaryRecordWriter.CreateAsync( + ms, FileType.Raw, new PriceStreamId(42), recordSize: TestRecordSize)) + { + await writer.WriteRecordAsync(MakeSnap(), orderId); + await writer.WriteRecordAsync(MakeDelta()); + await writer.SealAsync(); + } + + ms.Position = 0; + using var reader = new BinaryRecordReader(ms); + + // Should not throw — CRC validation is on by default + var records = reader.ReadRecords().ToList(); + Assert.Equal(2, records.Count); + } + + [Fact] + public async Task WriteThenRead_StreamLengthIsCorrect() + { + using var ms = new MemoryStream(); + + await using (var writer = await BinaryRecordWriter.CreateAsync( + ms, FileType.Raw, new PriceStreamId(42), recordSize: TestRecordSize)) + { + await writer.WriteRecordAsync(MakeSnap(), "abc"u8.ToArray()); + await writer.WriteRecordAsync(MakeDelta()); + await writer.SealAsync(); + } + + // header(128) + 2 records * 88 + footer(64) = 368 + Assert.Equal(Constants.HeaderSize + (TestRecordSize * 2) + Constants.FooterSize, ms.Length); + } + + [Fact] + public async Task WriteWithFlushThreshold_AllRecordsRoundTrip() + { + using var ms = new MemoryStream(); + + await using (var writer = await BinaryRecordWriter.CreateAsync( + ms, FileType.Raw, new PriceStreamId(42), flushBufferSize: 10)) + { + for (int i = 0; i < 100; i++) + { + var core = new CoreRecordLayout + { + ObservedTime = 1000 + i, + PriceStreamId = 42, + Price = 12345 + i, + Quantity = 100, + Type = i % 2 == 0 ? RecordType.Snap : RecordType.Delta, + Side = RecordSide.Bid, + Sequence = (uint)i, + Level = 0, + Flags = 0, + }; + await writer.WriteRecordAsync(core); + } + await writer.SealAsync(); + } + + ms.Position = 0; + using var reader = new BinaryRecordReader(ms); + var records = reader.ReadRecords(validateCrc: true).ToList(); + + Assert.Equal(100, records.Count); + for (int i = 0; i < 100; i++) + { + Assert.Equal(1000 + i, records[i].Core.ObservedTime); + Assert.Equal(12345 + i, records[i].Core.Price); + } + } + + [Fact] + public async Task WriteWithFlushThresholdMs_AllRecordsRoundTrip() + { + using var ms = new MemoryStream(); + + await using (var writer = await BinaryRecordWriter.CreateAsync( + ms, FileType.Raw, new PriceStreamId(42), flushThresholdMs: 1)) + { + for (int i = 0; i < 100; i++) + { + var core = new CoreRecordLayout + { + ObservedTime = 1000 + i, + PriceStreamId = 42, + Price = 12345 + i, + Quantity = 100, + Type = i % 2 == 0 ? RecordType.Snap : RecordType.Delta, + Side = RecordSide.Bid, + Sequence = (uint)i, + Level = 0, + Flags = 0, + }; + await writer.WriteRecordAsync(core); + } + await writer.SealAsync(); + } + + ms.Position = 0; + using var reader = new BinaryRecordReader(ms); + var records = reader.ReadRecords(validateCrc: true).ToList(); + + Assert.Equal(100, records.Count); + for (int i = 0; i < 100; i++) + { + Assert.Equal(1000 + i, records[i].Core.ObservedTime); + Assert.Equal(12345 + i, records[i].Core.Price); + } + } + + [Fact] + public async Task WriteRecordWithOrderId_HeaderHasRecordSize() + { + using var ms = new MemoryStream(); + + await using (var writer = await BinaryRecordWriter.CreateAsync( + ms, FileType.Raw, new PriceStreamId(42), recordSize: TestRecordSize)) + { + await writer.WriteRecordAsync(MakeSnap()); + await writer.SealAsync(); + } + + ms.Position = 0; + using var reader = new BinaryRecordReader(ms); + Assert.Equal(TestRecordSize, reader.Header.RecordSize); + } + + [Fact] + public async Task WriteRecord_ExtensionBytesExceedSpace_Throws() + { + using var ms = new MemoryStream(); + // Record size = CoreRecordSize + 4, so only 4 bytes of extension space + var smallRecordSize = Constants.CoreRecordSize + 4; + + await using var writer = await BinaryRecordWriter.CreateAsync( + ms, FileType.Raw, new PriceStreamId(42), recordSize: smallRecordSize); + + // 8 bytes exceeds the 4 bytes of available extension space + var oversizedExtension = new byte[8]; + Array.Fill(oversizedExtension, (byte)0xAB); + + await Assert.ThrowsAsync(async () => + await writer.WriteRecordAsync(MakeSnap(), oversizedExtension)); + } + + [Fact] + public async Task WriteRecord_ExtensionBytesFitExactly_Succeeds() + { + using var ms = new MemoryStream(); + var smallRecordSize = Constants.CoreRecordSize + 4; + + await using (var writer = await BinaryRecordWriter.CreateAsync( + ms, FileType.Raw, new PriceStreamId(42), recordSize: smallRecordSize)) + { + // Exactly 4 bytes should fit in 4 bytes of extension space + var extension = new byte[] { 0x01, 0x02, 0x03, 0x04 }; + await writer.WriteRecordAsync(MakeSnap(), extension); + await writer.SealAsync(); + } + + ms.Position = 0; + using var reader = new BinaryRecordReader(ms); + var records = reader.ReadRecords(validateCrc: true).ToList(); + Assert.Single(records); + } +} diff --git a/tests/Levels.Tests/IO/TruncationDetectionTests.cs b/tests/Levels.Tests/IO/TruncationDetectionTests.cs new file mode 100644 index 0000000..efaa6e3 --- /dev/null +++ b/tests/Levels.Tests/IO/TruncationDetectionTests.cs @@ -0,0 +1,82 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; + +namespace Levels.Tests.IO; + +public class TruncationDetectionTests +{ + private static CoreRecordLayout MakeSnap() => new() + { + ObservedTime = 1000, + PriceStreamId = 42, + Price = 12345, + Quantity = 100, + Type = RecordType.Snap, + Side = RecordSide.Bid, + Sequence = 1, + }; + + [Fact] + public async Task FileWithoutFooter_IsMarkedPartial() + { + using var ms = new MemoryStream(); + + await using (var writer = await BinaryRecordWriter.CreateAsync(ms, FileType.Raw, new PriceStreamId(42))) + { + await writer.WriteRecordAsync(MakeSnap()); + // No SealAsync — dispose without sealing + } + + ms.Position = 0; + using var reader = new BinaryRecordReader(ms); + + Assert.True(reader.IsPartial); + var records = reader.ReadRecords().ToList(); + Assert.Single(records); + } + + [Fact] + public async Task FileWithTruncatedLastRecord_DiscardsPartialRecord() + { + using var ms = new MemoryStream(); + + await using (var writer = await BinaryRecordWriter.CreateAsync(ms, FileType.Raw, new PriceStreamId(42))) + { + await writer.WriteRecordAsync(MakeSnap()); + await writer.WriteRecordAsync(MakeSnap()); + // No seal + } + + // Truncate: remove last 10 bytes of the second record + ms.SetLength(ms.Length - 10); + + ms.Position = 0; + using var reader = new BinaryRecordReader(ms); + + Assert.True(reader.IsPartial); + var records = reader.ReadRecords().ToList(); + Assert.Single(records); // only first complete record + } + + [Fact] + public async Task FileWithCorruptCrc_ThrowsOnRead() + { + using var ms = new MemoryStream(); + + await using (var writer = await BinaryRecordWriter.CreateAsync(ms, FileType.Raw, new PriceStreamId(42))) + { + await writer.WriteRecordAsync(MakeSnap()); + await writer.SealAsync(); + } + + // Flip a byte in the first record's data area (price field, offset 24 from header end) + var corruptOffset = Constants.HeaderSize + 24; + ms.GetBuffer()[corruptOffset] ^= 0xFF; + + ms.Position = 0; + using var reader = new BinaryRecordReader(ms); + + Assert.Throws(() => reader.ReadRecords().ToList()); + } +} diff --git a/tests/Levels.Tests/IO/WriteAheadLogTests.cs b/tests/Levels.Tests/IO/WriteAheadLogTests.cs new file mode 100644 index 0000000..f784f95 --- /dev/null +++ b/tests/Levels.Tests/IO/WriteAheadLogTests.cs @@ -0,0 +1,85 @@ +using Levels.Core.IO; + +namespace Levels.Tests.IO; + +public class WriteAheadLogTests : IDisposable +{ + private readonly string _tempDir; + + public WriteAheadLogTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + [Fact] + public async Task WalEntry_StoresVenueAndSymbol() + { + var walDir = Path.Combine(_tempDir, ".wal"); + + await using (var wal = WriteAheadLog.Open(walDir, 0)) + { + var recordBytes = new byte[56]; + recordBytes[0] = 0xAB; // marker byte + await wal.AppendAsync(12345, "binance", "BTC-USD", recordBytes); + } + + var entries = WriteAheadLog.Replay(walDir, 0); + Assert.Single(entries); + Assert.Equal(12345, entries[0].PriceStreamId); + Assert.Equal("binance", entries[0].Venue); + Assert.Equal("BTC-USD", entries[0].Symbol); + Assert.Equal(0xAB, entries[0].RecordBytes[0]); + } + + [Fact] + public async Task WalReplay_MultipleEntries_AllParsed() + { + var walDir = Path.Combine(_tempDir, ".wal"); + + await using (var wal = WriteAheadLog.Open(walDir, 0)) + { + await wal.AppendAsync(100, "binance", "BTC-USD", new byte[56]); + await wal.AppendAsync(200, "coinbase", "ETH-USD", new byte[56]); + await wal.AppendAsync(300, "kraken", "SOL-USD", new byte[56]); + } + + var entries = WriteAheadLog.Replay(walDir, 0); + Assert.Equal(3, entries.Count); + + Assert.Equal("binance", entries[0].Venue); + Assert.Equal("BTC-USD", entries[0].Symbol); + Assert.Equal("coinbase", entries[1].Venue); + Assert.Equal("ETH-USD", entries[1].Symbol); + Assert.Equal("kraken", entries[2].Venue); + Assert.Equal("SOL-USD", entries[2].Symbol); + } + + [Fact] + public async Task WalReplay_CorruptEntry_StopsAtCorruption() + { + var walDir = Path.Combine(_tempDir, ".wal"); + Directory.CreateDirectory(walDir); + + // Write a valid entry, then corrupt bytes + await using (var wal = WriteAheadLog.Open(walDir, 0)) + { + await wal.AppendAsync(100, "binance", "BTC-USD", new byte[56]); + } + + // Append garbage to the WAL file + var walPath = Path.Combine(walDir, "wal_0000.log"); + await File.AppendAllTextAsync(walPath, "CORRUPT"); + + var entries = WriteAheadLog.Replay(walDir, 0); + // Should recover the first valid entry, stop at corruption + Assert.Single(entries); + Assert.Equal("binance", entries[0].Venue); + } +} diff --git a/tests/Levels.Tests/Integration/EndToEndPipelineTests.cs b/tests/Levels.Tests/Integration/EndToEndPipelineTests.cs new file mode 100644 index 0000000..55bd85f --- /dev/null +++ b/tests/Levels.Tests/Integration/EndToEndPipelineTests.cs @@ -0,0 +1,293 @@ +using Levels.Compaction; +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.DataFlow; +using Levels.Period; +using Levels.Query; +using Levels.Sinks; + +namespace Levels.Tests.Integration; + +public class EndToEndPipelineTests : IDisposable +{ + private readonly string _tempDir; + + public EndToEndPipelineTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_e2e_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + try { Directory.Delete(_tempDir, recursive: true); } + catch { /* best effort cleanup */ } + } + + [Fact] + public async Task FullPipeline_Write_Compact_Promote_Query() + { + // ── Configuration ───────────────────────────────────────────────────── + var compactionWindow = TimeSpan.FromSeconds(1); + var compactionWindowNanos = compactionWindow.Ticks * 100; + + var compactionConfig = new CompactionConfig + { + DataPath = _tempDir, + CompactionWindow = compactionWindow, + SyntheticSnapIntervalDeltas = 50, + WindowGracePeriod = TimeSpan.Zero, + }; + + var periodConfig = new PeriodConfig + { + DataPath = _tempDir, + CompactionWindow = compactionWindow, + ConfigVersion = "e2e-test-v1", + MissingValueConfig = new MissingValueConfig + { + // Very generous gap threshold so health checks pass — leading SNAPs use + // WriteTimestamp.Now() as ObservedTime, creating large gaps with test data + DefaultGapThresholdNanos = long.MaxValue, + }, + }; + + var queryConfig = new QueryConfig + { + DataPath = _tempDir, + CompactionWindow = compactionWindow, + }; + + // ── Build pipeline ──────────────────────────────────────────────────── + var fileIndex = new FileIndex(); + var fileIndexSync = new FileIndexSyncHandler(fileIndex, _tempDir); + + var healthChecks = new IHealthCheck[] + { + new SnapCoverageChecker(), + new SequenceIntegrityChecker(), + new CrossedBookDetector(), + new MissingValueDetector(periodConfig.MissingValueConfig), + }; + + var compaction = new EventSourcingCompaction(compactionConfig); + var bus = new DataFlowBus(new IDataFlowHandler[] { compaction, fileIndexSync }); + + var sinkConfig = new SinkConfig + { + OutputPath = _tempDir, + RolloverSize = 2048, // Small rollover to force multiple files + RolloverInterval = TimeSpan.FromHours(24), // Won't trigger; size rollover will + PriceScale = 2, + QuantityScale = 4, + BackpressureLimit = 4096, + DataFlowBus = bus, + }; + + var sink = new PriceStreamSink(sinkConfig); + + // ── Start services ──────────────────────────────────────────────────── + var cts = new CancellationTokenSource(); + await bus.StartAsync(cts.Token); + await compaction.StartAsync(cts.Token); + await sink.StartAsync(cts.Token); + + // ── Write 500 market events across 2 symbols ────────────────────────── + // Two price streams: "BTCUSD" and "ETHUSD" + // ObservedTime values span 2 compaction windows (> 1 second in nanos) + // Each event spaced 10ms = 10_000_000 nanos apart + var symbols = new[] { "BTCUSD", "ETHUSD" }; + var baseObservedTime = 1_000_000_000_000L; // 1 second in nanos + var eventSpacingNanos = 10_000_000L; // 10ms + + for (int i = 0; i < 500; i++) + { + var symbol = symbols[i % 2]; + var observedTime = baseObservedTime + (i * eventSpacingNanos); + + // First event per symbol is a SNAP, rest are DELTAs + var isFirst = i < 2; + var recordType = isFirst ? RecordType.Snap : RecordType.Delta; + var side = (i % 4 < 2) ? RecordSide.Bid : RecordSide.Ask; + + // Prices: BTC around 50000, ETH around 3000 + // Ensure bids < asks to avoid crossed book + var basePrice = symbol == "BTCUSD" ? 5000000L : 300000L; + long price; + if (side == RecordSide.Bid) + price = basePrice - 100 - (i % 10); // Bids below midpoint + else + price = basePrice + 100 + (i % 10); // Asks above midpoint + + var quantity = 100L + (i % 50); + + await sink.Writer.WriteAsync(new RawMarketEvent( + Venue: "test-exchange", + Symbol: symbol, + ObservedTime: observedTime, + Price: price, + Quantity: quantity, + Type: recordType, + Side: side + ), cts.Token); + } + + // ── Stop sink to seal files ─────────────────────────────────────────── + await sink.StopAsync(cts.Token); + + // Brief delay for compaction to process sealed file events + await Task.Delay(2000); + + // Stop compaction and bus + await compaction.StopAsync(CancellationToken.None); + await bus.StopAsync(CancellationToken.None); + + // ── Verify RAW files were created ───────────────────────────────────── + var rawFiles = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories); + Assert.True(rawFiles.Length > 0, "Expected at least one RAW file"); + + // ── Verify all RAW files are sealed (have valid footer) ─────────────── + foreach (var rawFile in rawFiles) + { + using var fs = File.OpenRead(rawFile); + var reader = new BinaryRecordReader(fs); + Assert.NotNull(reader.Footer); + Assert.True(reader.Footer.Value.RecordCount > 0, + $"RAW file {rawFile} has no records"); + } + + // ── Check for AGG files ─────────────────────────────────────────────── + var aggFiles = Directory.GetFiles(_tempDir, "*.agg", SearchOption.AllDirectories); + // AGG files should exist if compaction windows closed + // Due to timing, we check but don't hard-fail if zero + + // ── Run PeriodPromotion on any AGG files ────────────────────────────── + var periodPromotion = new PeriodPromotion(periodConfig, fileIndex, healthChecks); + + // Reload file index from disk to pick up all files + fileIndex.LoadFromDisk(_tempDir); + + foreach (var aggFile in aggFiles) + { + using var fs = File.OpenRead(aggFile); + var reader = new BinaryRecordReader(fs); + if (reader.Footer is null) continue; + + var header = reader.Header; + var footer = reader.Footer.Value; + + var aggInfo = new AggFileInfo( + aggFile, + new PriceStreamId(header.PriceStreamId), + footer.RecordCount, + footer.DeltaCount, + 0, // syntheticSnapCount - not tracked during read + footer.FirstObservedTime, + footer.LastObservedTime, + Array.Empty()); + + periodPromotion.Promote(aggInfo); + } + + // ── Verify PERIOD files if AGG files existed ────────────────────────── + var periodFiles = Directory.GetFiles(_tempDir, "*.period", SearchOption.AllDirectories); + + if (aggFiles.Length > 0) + { + // Health checks should pass, so we expect PERIOD files + Assert.True(periodFiles.Length > 0, + "Expected PERIOD files after promotion of AGG files"); + + // Verify PERIOD files have correct FileType + foreach (var periodFile in periodFiles) + { + using var fs = File.OpenRead(periodFile); + var reader = new BinaryRecordReader(fs); + Assert.Equal(FileType.Period, reader.Header.FileType); + Assert.NotNull(reader.Footer); + } + } + + // ── Query via QueryLayer ────────────────────────────────────────────── + // Reload index to include PERIOD files + var freshIndex = new FileIndex(); + freshIndex.LoadFromDisk(_tempDir); + + var queryLayer = new QueryLayer(freshIndex, queryConfig); + + // Query each price stream + foreach (var symbol in symbols) + { + var streamId = PriceStreamId.FromVenueSymbol("test-exchange", symbol); + var results = queryLayer.Resolve( + streamId, + baseObservedTime, + baseObservedTime + 500 * eventSpacingNanos); + + Assert.True(results.Count > 0, + $"Expected query results for {symbol}"); + + // If PERIOD files exist, verify they are preferred over RAW/AGG + if (periodFiles.Length > 0) + { + var periodResults = results.Where(r => r.FileType == FileType.Period).ToList(); + var rawResults = results.Where(r => r.FileType == FileType.Raw).ToList(); + var aggResults = results.Where(r => r.FileType == FileType.Agg).ToList(); + + // For windows that have PERIOD files, RAW and AGG should not appear + foreach (var pr in periodResults) + { + var windowStart = pr.FirstObservedTime / compactionWindowNanos * compactionWindowNanos; + var windowEnd = windowStart + compactionWindowNanos; + + Assert.DoesNotContain(rawResults, + r => r.FirstObservedTime >= windowStart && r.FirstObservedTime < windowEnd); + Assert.DoesNotContain(aggResults, + r => r.FirstObservedTime >= windowStart && r.FirstObservedTime < windowEnd); + } + } + + // Verify total record counts by reading resolved files + long totalRecords = 0; + foreach (var entry in results) + { + totalRecords += entry.RecordCount; + } + + Assert.True(totalRecords > 0, + $"Expected non-zero total records for {symbol}"); + } + + // ── Verify data integrity: read all records from all files ──────────── + var allFilesByType = new Dictionary(); + foreach (var file in Directory.GetFiles(_tempDir, "*.*", SearchOption.AllDirectories)) + { + var ext = Path.GetExtension(file); + if (ext is not ".raw" and not ".agg" and not ".period") continue; + + using var fs = File.OpenRead(file); + try + { + var reader = new BinaryRecordReader(fs); + if (reader.Footer is null) continue; + + // Validate CRC on every record + var records = reader.ReadRecords(validateCrc: true).ToList(); + Assert.Equal(reader.Footer.Value.RecordCount, records.Count); + + var fileType = reader.Header.FileType; + allFilesByType.TryGetValue(fileType, out var count); + allFilesByType[fileType] = count + 1; + } + catch (InvalidDataException) + { + // Partial/corrupt file - skip + } + } + + // We should have at least RAW files + Assert.True(allFilesByType.ContainsKey(FileType.Raw), + "Expected at least some RAW files in the output"); + } +} diff --git a/tests/Levels.Tests/Levels.Tests.csproj b/tests/Levels.Tests/Levels.Tests.csproj new file mode 100644 index 0000000..7fac806 --- /dev/null +++ b/tests/Levels.Tests/Levels.Tests.csproj @@ -0,0 +1,39 @@ + + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/Levels.Tests/Period/CrossedBookDetectorTests.cs b/tests/Levels.Tests/Period/CrossedBookDetectorTests.cs new file mode 100644 index 0000000..667ea1a --- /dev/null +++ b/tests/Levels.Tests/Period/CrossedBookDetectorTests.cs @@ -0,0 +1,64 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Period; + +namespace Levels.Tests.Period; + +public class CrossedBookDetectorTests +{ + private readonly CrossedBookDetector _checker = new(); + private readonly PriceStreamId _streamId = PriceStreamId.FromSymbol("BTC-USD"); + + [Fact] + public void NormalSpread_Passes() + { + var records = new List + { + MakeRecord(RecordType.Snap, RecordSide.Bid, price: 100, quantity: 10), + MakeRecord(RecordType.Snap, RecordSide.Ask, price: 110, quantity: 10), + MakeRecord(RecordType.Delta, RecordSide.Bid, price: 105, quantity: 5), + }; + + var result = _checker.Check(records, _streamId); + Assert.True(result.Passed); + } + + [Fact] + public void CrossedBook_Fails() + { + var records = new List + { + MakeRecord(RecordType.Snap, RecordSide.Bid, price: 100, quantity: 10), + MakeRecord(RecordType.Snap, RecordSide.Ask, price: 110, quantity: 10), + MakeRecord(RecordType.Delta, RecordSide.Bid, price: 115, quantity: 5), // bid > ask + }; + + var result = _checker.Check(records, _streamId); + Assert.False(result.Passed); + Assert.Contains("Crossed book", result.FailureReason); + } + + [Fact] + public void EmptySides_Passes() + { + var records = new List + { + MakeRecord(RecordType.Snap, RecordSide.Bid, price: 100, quantity: 10), + // No ask side records + }; + + var result = _checker.Check(records, _streamId); + Assert.True(result.Passed); + } + + private static RawRecord MakeRecord(RecordType type, RecordSide side, long price, long quantity) => new() + { + Core = new CoreRecordLayout + { + Type = type, + Side = side, + Price = price, + Quantity = quantity, + }, + }; +} diff --git a/tests/Levels.Tests/Period/DemotionTests.cs b/tests/Levels.Tests/Period/DemotionTests.cs new file mode 100644 index 0000000..87eb851 --- /dev/null +++ b/tests/Levels.Tests/Period/DemotionTests.cs @@ -0,0 +1,109 @@ +using Levels.Compaction; +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Period; +using Levels.Query; +using Levels.Sinks; + +namespace Levels.Tests.Period; + +public class DemotionTests : IDisposable +{ + private readonly string _tempDir; + + public DemotionTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + private async Task<(AggFileInfo AggInfo, FileIndex Index, PeriodPromotion Promotion)> SetupPromotedAsync() + { + var sinkConfig = new SinkConfig { OutputPath = _tempDir }; + var sink = new PriceStreamSink(sinkConfig); + await sink.StartAsync(CancellationToken.None); + await RandomDataAdapter.GenerateAsync(sink.Writer, "test-exchange", ["BTC-USD"], levelsPerSide: 2, deltasPerSymbol: 10); + await sink.StopAsync(CancellationToken.None); + + var rawFiles = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories).OrderBy(f => f).ToList(); + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + + var compactionConfig = new CompactionConfig + { + DataPath = _tempDir, + CompactionWindow = TimeSpan.FromHours(24), + SyntheticSnapIntervalDeltas = 10000, + }; + var engine = new OrderbookReplayEngine(compactionConfig); + + using var firstFs = File.OpenRead(rawFiles[0]); + var firstReader = new BinaryRecordReader(firstFs); + var firstFooter = firstReader.Footer!.Value; + firstFs.Close(); + + var aggPath = AggFileWriter.ComputeAggPath(_tempDir, "test-exchange", streamId.Value, firstFooter.FirstObservedTime); + var aggInfo = await AggFileWriter.WriteAsync(engine.Replay(rawFiles), aggPath, streamId, 0, 0, rawFiles); + + var index = new FileIndex(); + index.LoadFromDisk(_tempDir); + + var periodConfig = new PeriodConfig + { + DataPath = _tempDir, + ConfigVersion = "v1", + }; + + var checks = new IHealthCheck[] { new SnapCoverageChecker() }; + var promotion = new PeriodPromotion(periodConfig, index, checks); + promotion.Promote(aggInfo); + + return (aggInfo, index, promotion); + } + + [Fact] + public async Task Demote_RemovesFileManifestAndIndex() + { + var (aggInfo, index, _) = await SetupPromotedAsync(); + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + var periodPath = Path.ChangeExtension(aggInfo.FilePath, ".period"); + var manifestPath = periodPath + ".manifest.json"; + + Assert.True(File.Exists(periodPath)); + Assert.True(File.Exists(manifestPath)); + Assert.True(index.HasFile(streamId, FileType.Period, 0, long.MaxValue)); + + DemotionService.Demote(periodPath, index); + + Assert.False(File.Exists(periodPath)); + Assert.False(File.Exists(manifestPath)); + Assert.False(index.HasFile(streamId, FileType.Period, 0, long.MaxValue)); + } + + [Fact] + public async Task RePromoteAfterDemote_Succeeds() + { + var (aggInfo, index, promotion) = await SetupPromotedAsync(); + var periodPath = Path.ChangeExtension(aggInfo.FilePath, ".period"); + + // Demote + DemotionService.Demote(periodPath, index); + Assert.False(File.Exists(periodPath)); + + // Re-promote should succeed + promotion.Promote(aggInfo); + Assert.True(File.Exists(periodPath)); + + // Verify it's valid + using var fs = File.OpenRead(periodPath); + var reader = new BinaryRecordReader(fs); + Assert.Equal(FileType.Period, reader.Header.FileType); + Assert.NotNull(reader.Footer); + } +} diff --git a/tests/Levels.Tests/Period/MissingValueDetectorTests.cs b/tests/Levels.Tests/Period/MissingValueDetectorTests.cs new file mode 100644 index 0000000..9b28165 --- /dev/null +++ b/tests/Levels.Tests/Period/MissingValueDetectorTests.cs @@ -0,0 +1,73 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Period; + +namespace Levels.Tests.Period; + +public class MissingValueDetectorTests +{ + private readonly PriceStreamId _streamId = PriceStreamId.FromSymbol("BTC-USD"); + + [Fact] + public void GapBelowThreshold_Passes() + { + var config = new MissingValueConfig { DefaultGapThresholdNanos = 1000 }; + var detector = new MissingValueDetector(config); + + var records = new List + { + MakeRecord(100), + MakeRecord(500), + MakeRecord(900), + }; + + var result = detector.Check(records, _streamId); + Assert.True(result.Passed); + } + + [Fact] + public void GapAboveThreshold_Fails() + { + var config = new MissingValueConfig { DefaultGapThresholdNanos = 1000 }; + var detector = new MissingValueDetector(config); + + var records = new List + { + MakeRecord(100), + MakeRecord(500), + MakeRecord(2000), // gap of 1500 > 1000 + }; + + var result = detector.Check(records, _streamId); + Assert.False(result.Passed); + Assert.Contains("exceeds threshold", result.FailureReason); + } + + [Fact] + public void PerStreamOverride_UsesOverrideThreshold() + { + var config = new MissingValueConfig + { + DefaultGapThresholdNanos = 100, + StreamOverrides = new Dictionary + { + [_streamId] = 5000 + } + }; + var detector = new MissingValueDetector(config); + + var records = new List + { + MakeRecord(100), + MakeRecord(4000), // gap 3900 > default 100 but < override 5000 + }; + + var result = detector.Check(records, _streamId); + Assert.True(result.Passed); + } + + private static RawRecord MakeRecord(long observedTime) => new() + { + Core = new CoreRecordLayout { ObservedTime = observedTime }, + }; +} diff --git a/tests/Levels.Tests/Period/PeriodPromotionTests.cs b/tests/Levels.Tests/Period/PeriodPromotionTests.cs new file mode 100644 index 0000000..9393816 --- /dev/null +++ b/tests/Levels.Tests/Period/PeriodPromotionTests.cs @@ -0,0 +1,194 @@ +using Levels.Compaction; +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Period; +using Levels.Query; +using Levels.Sinks; + +namespace Levels.Tests.Period; + +public class PeriodPromotionTests : IDisposable +{ + private readonly string _tempDir; + + public PeriodPromotionTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + private async Task<(AggFileInfo AggInfo, FileIndex Index)> CreateAggFileAsync() + { + // Create RAW files via ingestion + var sinkConfig = new SinkConfig { OutputPath = _tempDir }; + var sink = new PriceStreamSink(sinkConfig); + await sink.StartAsync(CancellationToken.None); + await RandomDataAdapter.GenerateAsync(sink.Writer, "test-exchange", ["BTC-USD"], levelsPerSide: 3, deltasPerSymbol: 20); + await sink.StopAsync(CancellationToken.None); + + var rawFiles = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories).OrderBy(f => f).ToList(); + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + + // Compact to AGG + var compactionConfig = new CompactionConfig + { + DataPath = _tempDir, + CompactionWindow = TimeSpan.FromHours(24), + SyntheticSnapIntervalDeltas = 10000, + }; + var engine = new OrderbookReplayEngine(compactionConfig); + + // Compute AGG path using standard convention + using var firstFs = File.OpenRead(rawFiles[0]); + var firstReader = new BinaryRecordReader(firstFs); + var firstFooter = firstReader.Footer!.Value; + firstFs.Close(); + + var aggPath = AggFileWriter.ComputeAggPath(_tempDir, "test-exchange", streamId.Value, firstFooter.FirstObservedTime); + var aggInfo = await AggFileWriter.WriteAsync(engine.Replay(rawFiles), aggPath, streamId, 0, 0, rawFiles); + + var index = new FileIndex(); + index.LoadFromDisk(_tempDir); + + return (aggInfo, index); + } + + [Fact] + public async Task HappyPath_CreatesPerioFile() + { + var (aggInfo, index) = await CreateAggFileAsync(); + + var periodConfig = new PeriodConfig + { + DataPath = _tempDir, + ConfigVersion = "v1", + CompactionWindow = TimeSpan.FromHours(24), + }; + + var checks = new IHealthCheck[] + { + new SnapCoverageChecker(), + new SequenceIntegrityChecker(), + }; + + var promotion = new PeriodPromotion(periodConfig, index, checks); + promotion.Promote(aggInfo); + + // Verify PERIOD file exists + var periodPath = Path.ChangeExtension(aggInfo.FilePath, ".period"); + Assert.True(File.Exists(periodPath)); + + // Verify PERIOD file has correct FileType header + using var fs = File.OpenRead(periodPath); + var reader = new BinaryRecordReader(fs); + Assert.Equal(FileType.Period, reader.Header.FileType); + Assert.NotNull(reader.Footer); + + // Verify manifest + var manifestPath = periodPath + ".manifest.json"; + Assert.True(File.Exists(manifestPath)); + var manifest = PromotionManifest.ReadFrom(manifestPath); + Assert.Equal(aggInfo.FilePath, manifest.AggFilePath); + Assert.Equal("v1", manifest.ConfigVersion); + Assert.All(manifest.CheckResults, cr => Assert.True(cr.Passed)); + } + + [Fact] + public async Task HealthCheckFailure_BlocksPromotion() + { + var (aggInfo, index) = await CreateAggFileAsync(); + + var periodConfig = new PeriodConfig + { + DataPath = _tempDir, + ConfigVersion = "v1", + }; + + // Use a detector with a very small gap threshold that will fail + var checks = new IHealthCheck[] + { + new SnapCoverageChecker(), + new MissingValueDetector(new MissingValueConfig { DefaultGapThresholdNanos = 1 }), + }; + + var promotion = new PeriodPromotion(periodConfig, index, checks); + promotion.Promote(aggInfo); + + // PERIOD file should NOT exist + var periodPath = Path.ChangeExtension(aggInfo.FilePath, ".period"); + Assert.False(File.Exists(periodPath)); + + // But manifest should exist with failure info + var manifestPath = periodPath + ".manifest.json"; + Assert.True(File.Exists(manifestPath)); + var manifest = PromotionManifest.ReadFrom(manifestPath); + Assert.Contains(manifest.CheckResults, cr => !cr.Passed); + } + + [Fact] + public async Task RePromotion_Rejected() + { + var (aggInfo, index) = await CreateAggFileAsync(); + + var periodConfig = new PeriodConfig + { + DataPath = _tempDir, + ConfigVersion = "v1", + }; + + var checks = new IHealthCheck[] { new SnapCoverageChecker() }; + var promotion = new PeriodPromotion(periodConfig, index, checks); + + // First promotion + promotion.Promote(aggInfo); + var periodPath = Path.ChangeExtension(aggInfo.FilePath, ".period"); + Assert.True(File.Exists(periodPath)); + + // Second promotion should be silently rejected (no error, no overwrite) + var modifiedTime = File.GetLastWriteTimeUtc(periodPath); + await Task.Delay(50); + promotion.Promote(aggInfo); + + // File unchanged + Assert.Equal(modifiedTime, File.GetLastWriteTimeUtc(periodPath)); + } + + [Fact] + public async Task QueryLayer_PrefersPeriodOverAgg() + { + var (aggInfo, index) = await CreateAggFileAsync(); + + var periodConfig = new PeriodConfig + { + DataPath = _tempDir, + ConfigVersion = "v1", + CompactionWindow = TimeSpan.FromHours(24), + }; + + var checks = new IHealthCheck[] { new SnapCoverageChecker() }; + var promotion = new PeriodPromotion(periodConfig, index, checks); + promotion.Promote(aggInfo); + + var queryConfig = new QueryConfig + { + DataPath = _tempDir, + CompactionWindow = TimeSpan.FromHours(24), + }; + var queryLayer = new QueryLayer(index, queryConfig); + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + + var resolved = queryLayer.Resolve(streamId, 0, long.MaxValue); + Assert.NotEmpty(resolved); + + // Should pick PERIOD over RAW/AGG for the same window + var bestEntry = resolved[0]; + Assert.Equal(FileType.Period, bestEntry.FileType); + } +} diff --git a/tests/Levels.Tests/Period/SequenceIntegrityCheckerTests.cs b/tests/Levels.Tests/Period/SequenceIntegrityCheckerTests.cs new file mode 100644 index 0000000..ed55efe --- /dev/null +++ b/tests/Levels.Tests/Period/SequenceIntegrityCheckerTests.cs @@ -0,0 +1,59 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Period; + +namespace Levels.Tests.Period; + +public class SequenceIntegrityCheckerTests +{ + private readonly SequenceIntegrityChecker _checker = new(); + private readonly PriceStreamId _streamId = PriceStreamId.FromSymbol("BTC-USD"); + + [Fact] + public void NonDecreasingTimestamps_Passes() + { + var records = new List + { + MakeRecord(100), + MakeRecord(200), + MakeRecord(300), + }; + + var result = _checker.Check(records, _streamId); + Assert.True(result.Passed); + } + + [Fact] + public void BackwardsJump_Fails() + { + var records = new List + { + MakeRecord(100), + MakeRecord(300), + MakeRecord(200), // backwards + }; + + var result = _checker.Check(records, _streamId); + Assert.False(result.Passed); + Assert.Contains("Backwards", result.FailureReason); + } + + [Fact] + public void DuplicateTimestamps_Passes() + { + var records = new List + { + MakeRecord(100), + MakeRecord(100), + MakeRecord(200), + }; + + var result = _checker.Check(records, _streamId); + Assert.True(result.Passed); + } + + private static RawRecord MakeRecord(long writeTimestamp) => new() + { + Core = new CoreRecordLayout { WriteTimestamp = writeTimestamp }, + }; +} diff --git a/tests/Levels.Tests/Period/SnapCoverageCheckerTests.cs b/tests/Levels.Tests/Period/SnapCoverageCheckerTests.cs new file mode 100644 index 0000000..0c95da9 --- /dev/null +++ b/tests/Levels.Tests/Period/SnapCoverageCheckerTests.cs @@ -0,0 +1,51 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Period; + +namespace Levels.Tests.Period; + +public class SnapCoverageCheckerTests +{ + private readonly SnapCoverageChecker _checker = new(); + private readonly PriceStreamId _streamId = PriceStreamId.FromSymbol("BTC-USD"); + + [Fact] + public void FirstRecordSnap_Passes() + { + var records = new List + { + MakeRecord(RecordType.Snap), + MakeRecord(RecordType.Delta), + }; + + var result = _checker.Check(records, _streamId); + Assert.True(result.Passed); + } + + [Fact] + public void FirstRecordDelta_Fails() + { + var records = new List + { + MakeRecord(RecordType.Delta), + MakeRecord(RecordType.Snap), + }; + + var result = _checker.Check(records, _streamId); + Assert.False(result.Passed); + Assert.Contains("not a SNAP", result.FailureReason); + } + + [Fact] + public void EmptyRecords_Fails() + { + var result = _checker.Check([], _streamId); + Assert.False(result.Passed); + Assert.Contains("No records", result.FailureReason); + } + + private static RawRecord MakeRecord(RecordType type) => new() + { + Core = new CoreRecordLayout { Type = type }, + }; +} diff --git a/tests/Levels.Tests/Projections/L3OrderbookSideTests.cs b/tests/Levels.Tests/Projections/L3OrderbookSideTests.cs new file mode 100644 index 0000000..9ab0ab0 --- /dev/null +++ b/tests/Levels.Tests/Projections/L3OrderbookSideTests.cs @@ -0,0 +1,88 @@ +using Levels.Core.Orderbook; + +namespace Levels.Tests.Projections; + +public class L3OrderbookSideTests +{ + [Fact] + public void Apply_AddsOrder() + { + var side = new L3OrderbookSide(); + side.Apply("order-1", 100, 50); + + Assert.Single(side.Orders); + Assert.Equal((100, 50), side.Orders["order-1"]); + } + + [Fact] + public void Apply_UpdatesExistingOrder() + { + var side = new L3OrderbookSide(); + side.Apply("order-1", 100, 50); + side.Apply("order-1", 101, 60); + + Assert.Single(side.Orders); + Assert.Equal((101, 60), side.Orders["order-1"]); + // Old price level should be removed + Assert.Equal(0, side.AggregatedQuantityAt(100)); + Assert.Equal(60, side.AggregatedQuantityAt(101)); + } + + [Fact] + public void Apply_ZeroQuantity_RemovesOrder() + { + var side = new L3OrderbookSide(); + side.Apply("order-1", 100, 50); + side.Apply("order-1", 100, 0); + + Assert.Empty(side.Orders); + Assert.Equal(0, side.AggregatedQuantityAt(100)); + } + + [Fact] + public void Remove_RemovesOrder() + { + var side = new L3OrderbookSide(); + side.Apply("order-1", 100, 50); + side.Remove("order-1"); + + Assert.Empty(side.Orders); + } + + [Fact] + public void MultipleOrdersSamePrice_AggregateCorrectly() + { + var side = new L3OrderbookSide(); + side.Apply("order-1", 100, 30); + side.Apply("order-2", 100, 70); + + Assert.Equal(100, side.AggregatedQuantityAt(100)); + Assert.Equal(2, side.Orders.Count); + } + + [Fact] + public void ToL2Levels_AggregatesCorrectly() + { + var side = new L3OrderbookSide(); + side.Apply("order-1", 100, 30); + side.Apply("order-2", 100, 70); + side.Apply("order-3", 200, 50); + + var l2 = side.ToL2Levels().ToList(); + Assert.Equal(2, l2.Count); + Assert.Contains(l2, kv => kv.Key == 100 && kv.Value == 100); + Assert.Contains(l2, kv => kv.Key == 200 && kv.Value == 50); + } + + [Fact] + public void Clear_RemovesEverything() + { + var side = new L3OrderbookSide(); + side.Apply("order-1", 100, 50); + side.Apply("order-2", 200, 60); + side.Clear(); + + Assert.Empty(side.Orders); + Assert.Empty(side.PriceLevels); + } +} diff --git a/tests/Levels.Tests/Projections/OrderbookProjectionTests.cs b/tests/Levels.Tests/Projections/OrderbookProjectionTests.cs new file mode 100644 index 0000000..24430ac --- /dev/null +++ b/tests/Levels.Tests/Projections/OrderbookProjectionTests.cs @@ -0,0 +1,109 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Query; +using Levels.Sinks; + +namespace Levels.Tests.Projections; + +public class OrderbookProjectionTests : IDisposable +{ + private readonly string _tempDir; + + public OrderbookProjectionTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + [Fact] + public async Task ProjectL2_ReturnsCorrectBidAskLevels() + { + var config = new SinkConfig { OutputPath = _tempDir }; + var sink = new PriceStreamSink(config); + await sink.StartAsync(CancellationToken.None); + + var now = WriteTimestamp.Now(); + await sink.Writer.WriteAsync(new RawMarketEvent("ex", "BTC-USD", now, 100, 50, RecordType.Snap, RecordSide.Bid)); + await sink.Writer.WriteAsync(new RawMarketEvent("ex", "BTC-USD", now + 1000, 101, 30, RecordType.Snap, RecordSide.Ask)); + await sink.Writer.WriteAsync(new RawMarketEvent("ex", "BTC-USD", now + 2000, 99, 40, RecordType.Delta, RecordSide.Bid)); + await sink.StopAsync(CancellationToken.None); + + var fileIndex = new FileIndex(); + fileIndex.LoadFromDisk(_tempDir); + var queryLayer = new QueryLayer(fileIndex, new QueryConfig { DataPath = _tempDir }); + var projection = new OrderbookProjection(queryLayer); + + var streamId = PriceStreamId.FromVenueSymbol("ex", "BTC-USD"); + var l2 = projection.ProjectL2(streamId, 0, long.MaxValue); + + Assert.NotEmpty(l2.Bids); + Assert.NotEmpty(l2.Asks); + } + + [Fact] + public async Task ProjectL1_ReturnsBestBidAsk() + { + var config = new SinkConfig { OutputPath = _tempDir }; + var sink = new PriceStreamSink(config); + await sink.StartAsync(CancellationToken.None); + + var now = WriteTimestamp.Now(); + await sink.Writer.WriteAsync(new RawMarketEvent("ex", "BTC-USD", now, 100, 50, RecordType.Snap, RecordSide.Bid)); + await sink.Writer.WriteAsync(new RawMarketEvent("ex", "BTC-USD", now + 1000, 102, 30, RecordType.Snap, RecordSide.Ask)); + await sink.StopAsync(CancellationToken.None); + + var fileIndex = new FileIndex(); + fileIndex.LoadFromDisk(_tempDir); + var queryLayer = new QueryLayer(fileIndex, new QueryConfig { DataPath = _tempDir }); + var projection = new OrderbookProjection(queryLayer); + + var streamId = PriceStreamId.FromVenueSymbol("ex", "BTC-USD"); + var l1 = projection.ProjectL1(streamId, 0, long.MaxValue); + + Assert.NotNull(l1.BestBid); + Assert.NotNull(l1.BestAsk); + Assert.Equal(100, l1.BestBid.Value.Price); + Assert.Equal(102, l1.BestAsk.Value.Price); + } + + [Fact] + public async Task ProjectL2_ExcludeOwner_FiltersOwnOrders() + { + var config = new SinkConfig { OutputPath = _tempDir }; + var sink = new PriceStreamSink(config); + await sink.StartAsync(CancellationToken.None); + + var now = WriteTimestamp.Now(); + // Regular order + await sink.Writer.WriteAsync(new RawMarketEvent("ex", "BTC-USD", now, 100, 50, RecordType.Snap, RecordSide.Bid)); + // Owner order + await sink.Writer.WriteAsync(new RawMarketEvent("ex", "BTC-USD", now + 1000, 101, 30, RecordType.Delta, RecordSide.Bid, IsOwner: true)); + // Regular ask + await sink.Writer.WriteAsync(new RawMarketEvent("ex", "BTC-USD", now + 2000, 105, 20, RecordType.Snap, RecordSide.Ask)); + await sink.StopAsync(CancellationToken.None); + + var fileIndex = new FileIndex(); + fileIndex.LoadFromDisk(_tempDir); + var queryLayer = new QueryLayer(fileIndex, new QueryConfig { DataPath = _tempDir }); + var projection = new OrderbookProjection(queryLayer); + + var streamId = PriceStreamId.FromVenueSymbol("ex", "BTC-USD"); + + // Without owner exclusion — both bid levels present + var l2All = projection.ProjectL2(streamId, 0, long.MaxValue, excludeOwner: false); + var allBidPrices = l2All.Bids.Select(b => b.Price).ToList(); + Assert.Contains(101, allBidPrices); + + // With owner exclusion — owner bid at 101 excluded + var l2NoOwner = projection.ProjectL2(streamId, 0, long.MaxValue, excludeOwner: true); + var noOwnerBidPrices = l2NoOwner.Bids.Select(b => b.Price).ToList(); + Assert.DoesNotContain(101, noOwnerBidPrices); + } +} diff --git a/tests/Levels.Tests/Query/FileIndexSyncTests.cs b/tests/Levels.Tests/Query/FileIndexSyncTests.cs new file mode 100644 index 0000000..8759f42 --- /dev/null +++ b/tests/Levels.Tests/Query/FileIndexSyncTests.cs @@ -0,0 +1,83 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.DataFlow; +using Levels.Query; + +namespace Levels.Tests.Query; + +public class FileIndexSyncTests +{ + [Fact] + public async Task OnFileSealed_RegistersRawEntry() + { + var index = new FileIndex(); + var handler = new FileIndexSyncHandler(index, "/data"); + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + + var sealedFile = new SealedFileInfo( + "/data/binance/12345/20240315_000001.raw", + streamId, 100, 50, 1000, 2000); + + await handler.OnFileSealed(sealedFile, CancellationToken.None); + + var entries = index.Query(streamId, 0, 3000); + Assert.Single(entries); + Assert.Equal(FileType.Raw, entries[0].FileType); + Assert.Equal("binance", entries[0].Venue); + Assert.Equal(1000, entries[0].FirstObservedTime); + Assert.Equal(2000, entries[0].LastObservedTime); + } + + [Fact] + public async Task OnAggCreated_RegistersAggEntry() + { + var index = new FileIndex(); + var handler = new FileIndexSyncHandler(index, "/data"); + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + + var aggFile = new AggFileInfo( + "/data/binance/12345/20240315_120000.agg", + streamId, 200, 100, 10, 1000, 2000, []); + + await handler.OnAggCreated(aggFile, CancellationToken.None); + + var entries = index.Query(streamId, 0, 3000); + Assert.Single(entries); + Assert.Equal(FileType.Agg, entries[0].FileType); + Assert.Equal("binance", entries[0].Venue); + } + + [Fact] + public async Task OnRecordWritten_IsNoOp() + { + var index = new FileIndex(); + var handler = new FileIndexSyncHandler(index, "/data"); + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + + await handler.OnRecordWritten(streamId, new RawRecord(), CancellationToken.None); + + var entries = index.Query(streamId, 0, long.MaxValue); + Assert.Empty(entries); + } + + [Fact] + public async Task DataFlowBus_PropagatesEventsToFileIndex() + { + var index = new FileIndex(); + var handler = new FileIndexSyncHandler(index, "/data"); + var bus = new DataFlowBus([handler]); + await bus.StartAsync(CancellationToken.None); + + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + bus.PublishFileSealed(new SealedFileInfo( + "/data/ex/12345/test.raw", streamId, 10, 5, 100, 200)); + + // Give the bus consumer time to process + await Task.Delay(100); + + var entries = index.Query(streamId, 0, 300); + Assert.Single(entries); + + await bus.StopAsync(CancellationToken.None); + } +} diff --git a/tests/Levels.Tests/Query/FileIndexTests.cs b/tests/Levels.Tests/Query/FileIndexTests.cs new file mode 100644 index 0000000..7124b92 --- /dev/null +++ b/tests/Levels.Tests/Query/FileIndexTests.cs @@ -0,0 +1,117 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Query; +using Levels.Sinks; + +namespace Levels.Tests.Query; + +public class FileIndexTests : IDisposable +{ + private readonly string _tempDir; + + public FileIndexTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + [Fact] + public async Task LoadFromDisk_FindsSealedFiles() + { + // Create RAW files via ingestion + var config = new SinkConfig { OutputPath = _tempDir }; + var sink = new PriceStreamSink(config); + await sink.StartAsync(CancellationToken.None); + await RandomDataAdapter.GenerateAsync(sink.Writer, "test-exchange", ["BTC-USD"], levelsPerSide: 2, deltasPerSymbol: 10); + await sink.StopAsync(CancellationToken.None); + + var index = new FileIndex(); + index.LoadFromDisk(_tempDir); + + var streamId = PriceStreamId.FromVenueSymbol("test-exchange", "BTC-USD"); + var entries = index.Query(streamId, 0, long.MaxValue); + Assert.NotEmpty(entries); + Assert.All(entries, e => Assert.Equal(FileType.Raw, e.FileType)); + } + + [Fact] + public void Register_AddsEntry() + { + var index = new FileIndex(); + var streamId = PriceStreamId.FromSymbol("ETH-USD"); + var entry = new FileIndexEntry("test.raw", streamId, "binance", FileType.Raw, 100, 200, 10); + + index.Register(entry); + + var results = index.Query(streamId, 0, 300); + Assert.Single(results); + Assert.Equal("test.raw", results[0].FilePath); + } + + [Fact] + public void Remove_RemovesEntry() + { + var index = new FileIndex(); + var streamId = PriceStreamId.FromSymbol("ETH-USD"); + var entry = new FileIndexEntry("test.raw", streamId, "binance", FileType.Raw, 100, 200, 10); + + index.Register(entry); + index.Remove("test.raw"); + + var results = index.Query(streamId, 0, 300); + Assert.Empty(results); + } + + [Fact] + public void Query_ReturnsOverlappingEntries() + { + var index = new FileIndex(); + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + + index.Register(new FileIndexEntry("a.raw", streamId, "ex", FileType.Raw, 100, 200, 5)); + index.Register(new FileIndexEntry("b.raw", streamId, "ex", FileType.Raw, 300, 400, 5)); + index.Register(new FileIndexEntry("c.raw", streamId, "ex", FileType.Raw, 500, 600, 5)); + + // Query overlapping [150, 350) should return a and b + var results = index.Query(streamId, 150, 350); + Assert.Equal(2, results.Count); + } + + [Fact] + public void Query_WithFileType_FiltersCorrectly() + { + var index = new FileIndex(); + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + + index.Register(new FileIndexEntry("a.raw", streamId, "ex", FileType.Raw, 100, 200, 5)); + index.Register(new FileIndexEntry("a.agg", streamId, "ex", FileType.Agg, 100, 200, 5)); + + var rawResults = index.Query(streamId, 0, 300, FileType.Raw); + Assert.Single(rawResults); + Assert.Equal(FileType.Raw, rawResults[0].FileType); + + var aggResults = index.Query(streamId, 0, 300, FileType.Agg); + Assert.Single(aggResults); + Assert.Equal(FileType.Agg, aggResults[0].FileType); + } + + [Fact] + public void HasFile_ReturnsTrueWhenExists() + { + var index = new FileIndex(); + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + + index.Register(new FileIndexEntry("a.period", streamId, "ex", FileType.Period, 100, 200, 5)); + + Assert.True(index.HasFile(streamId, FileType.Period, 100, 201)); + Assert.False(index.HasFile(streamId, FileType.Period, 300, 400)); + Assert.False(index.HasFile(streamId, FileType.Agg, 100, 201)); + } +} diff --git a/tests/Levels.Tests/Query/QueryLayerTests.cs b/tests/Levels.Tests/Query/QueryLayerTests.cs new file mode 100644 index 0000000..4be29e7 --- /dev/null +++ b/tests/Levels.Tests/Query/QueryLayerTests.cs @@ -0,0 +1,113 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Query; + +namespace Levels.Tests.Query; + +public class QueryLayerTests +{ + private readonly PriceStreamId _streamId = PriceStreamId.FromSymbol("BTC-USD"); + private const long WindowNanos = 3_600_000_000_000; // 1 hour in nanos + + private static QueryLayer CreateLayer(FileIndex index) => new(index, new QueryConfig + { + DataPath = "/data", + CompactionWindow = TimeSpan.FromHours(1), + }); + + [Fact] + public void Resolve_PrefersPeriodOverAggOverRaw() + { + var index = new FileIndex(); + var streamId = _streamId; + + // All in same window (window start = 0) + index.Register(new FileIndexEntry("a.raw", streamId, "ex", FileType.Raw, 100, 200, 5)); + index.Register(new FileIndexEntry("a.agg", streamId, "ex", FileType.Agg, 100, 200, 5)); + index.Register(new FileIndexEntry("a.period", streamId, "ex", FileType.Period, 100, 200, 5)); + + var layer = CreateLayer(index); + var result = layer.Resolve(streamId, 0, 300); + + Assert.Single(result); + Assert.Equal(FileType.Period, result[0].FileType); + } + + [Fact] + public void Resolve_PrefersAggOverRaw_WhenNoPeriod() + { + var index = new FileIndex(); + var streamId = _streamId; + + index.Register(new FileIndexEntry("a.raw", streamId, "ex", FileType.Raw, 100, 200, 5)); + index.Register(new FileIndexEntry("a.agg", streamId, "ex", FileType.Agg, 100, 200, 5)); + + var layer = CreateLayer(index); + var result = layer.Resolve(streamId, 0, 300); + + Assert.Single(result); + Assert.Equal(FileType.Agg, result[0].FileType); + } + + [Fact] + public void Resolve_MixedWindowTypes_SelectsBestPerWindow() + { + var index = new FileIndex(); + var streamId = _streamId; + + // Window 0: has Period + index.Register(new FileIndexEntry("w0.raw", streamId, "ex", FileType.Raw, 100, 200, 5)); + index.Register(new FileIndexEntry("w0.period", streamId, "ex", FileType.Period, 100, 200, 5)); + + // Window 1: only has Raw + index.Register(new FileIndexEntry("w1.raw", streamId, "ex", FileType.Raw, + WindowNanos + 100, WindowNanos + 200, 5)); + + var layer = CreateLayer(index); + var result = layer.Resolve(streamId, 0, WindowNanos * 2); + + Assert.Equal(2, result.Count); + Assert.Equal(FileType.Period, result[0].FileType); + Assert.Equal(FileType.Raw, result[1].FileType); + } + + [Fact] + public void Resolve_SortedByFirstObservedTime() + { + var index = new FileIndex(); + var streamId = _streamId; + + // Register in reverse order + index.Register(new FileIndexEntry("b.raw", streamId, "ex", FileType.Raw, + WindowNanos + 100, WindowNanos + 200, 5)); + index.Register(new FileIndexEntry("a.raw", streamId, "ex", FileType.Raw, 100, 200, 5)); + + var layer = CreateLayer(index); + var result = layer.Resolve(streamId, 0, WindowNanos * 2); + + Assert.Equal(2, result.Count); + Assert.True(result[0].FirstObservedTime < result[1].FirstObservedTime); + } + + [Fact] + public void ResolveSingle_ReturnsMatchingEntry() + { + var index = new FileIndex(); + var streamId = _streamId; + index.Register(new FileIndexEntry("a.raw", streamId, "ex", FileType.Raw, 100, 200, 5)); + + var layer = CreateLayer(index); + var result = layer.ResolveSingle(streamId, 150); + Assert.NotNull(result); + Assert.Equal("a.raw", result.Value.FilePath); + } + + [Fact] + public void ResolveSingle_ReturnsNullWhenNoMatch() + { + var index = new FileIndex(); + var layer = CreateLayer(index); + var result = layer.ResolveSingle(_streamId, 150); + Assert.Null(result); + } +} diff --git a/tests/Levels.Tests/Resampled/BatchResampledProcessorTests.cs b/tests/Levels.Tests/Resampled/BatchResampledProcessorTests.cs new file mode 100644 index 0000000..f31e677 --- /dev/null +++ b/tests/Levels.Tests/Resampled/BatchResampledProcessorTests.cs @@ -0,0 +1,189 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Query; +using Levels.Resampled; + +namespace Levels.Tests.Resampled; + +public class BatchResampledProcessorTests : IDisposable +{ + private readonly string _dataPath; + private readonly PriceStreamId _stream1 = PriceStreamId.FromSymbol("BTC-USD"); + private readonly PriceStreamId _stream2 = PriceStreamId.FromSymbol("ETH-USD"); + + public BatchResampledProcessorTests() + { + _dataPath = Path.Combine(Path.GetTempPath(), $"levels_batch_resampled_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_dataPath); + } + + private async Task WriteTestRawFile(PriceStreamId streamId, string venue, + (RecordType Type, RecordSide Side, long Price, long Quantity, long ObservedTime)[] events) + { + var dir = Path.Combine(_dataPath, venue, streamId.Value.ToString()); + Directory.CreateDirectory(dir); + var filePath = Path.Combine(dir, $"{Guid.NewGuid():N}.raw"); + + await using var fs = new FileStream(filePath, FileMode.CreateNew, FileAccess.Write, FileShare.None); + await using var writer = await BinaryRecordWriter.CreateAsync(fs, FileType.Raw, streamId); + + uint seq = 0; + foreach (var (type, side, price, qty, time) in events) + { + var core = new CoreRecordLayout + { + ObservedTime = time, + PriceStreamId = streamId.Value, + Price = price, + Quantity = qty, + Type = type, + Side = side, + Sequence = seq++, + }; + await writer.WriteRecordAsync(core); + } + + await writer.SealAsync(); + return filePath; + } + + [Fact] + public async Task ProcessAsync_ProducesExpectedOutput() + { + var venue = "testex"; + var fromNanos = 1_000_000_000L; + var toNanos = 3_000_000_000L; + + + var rawFile = await WriteTestRawFile(_stream1, venue, [ + (RecordType.Snap, RecordSide.Bid, 100, 10, 1_000_000_000), + (RecordType.Snap, RecordSide.Ask, 110, 5, 1_000_000_000), + (RecordType.Delta, RecordSide.Bid, 105, 15, 1_500_000_000), + (RecordType.Delta, RecordSide.Bid, 108, 20, 2_500_000_000), + ]); + + var fileIndex = new FileIndex(); + fileIndex.LoadFromDisk(_dataPath); + + var queryConfig = new QueryConfig + { + DataPath = _dataPath, + CompactionWindow = TimeSpan.FromSeconds(10), + }; + var queryLayer = new QueryLayer(fileIndex, queryConfig); + + var config = new ResampledStreamConfig + { + ConfigVersion = "batch_v1", + SourceStreams = [_stream1], + ResamplingWindow = TimeSpan.FromSeconds(1), + OutputType = ResampledOutputType.TopOfBook, + Venue = venue, + OutputPath = _dataPath, + }; + + var processor = new BatchResampledProcessor(config, queryLayer, fileIndex); + await processor.ProcessAsync(fromNanos, toNanos); + + var resampledFiles = Directory.GetFiles(_dataPath, "*.resampled", SearchOption.AllDirectories); + Assert.Single(resampledFiles); + + using var fs = File.OpenRead(resampledFiles[0]); + var reader = new BinaryRecordReader(fs); + Assert.Equal(FileType.Resampled, reader.Header.FileType); + Assert.NotNull(reader.Footer); + + // Should have records at window boundaries + var records = reader.ReadRecords(validateCrc: true).ToList(); + Assert.True(records.Count > 0); + } + + [Fact] + public async Task ProcessAsync_MultipleSourceStreams() + { + var venue = "testex"; + var fromNanos = 1_000_000_000L; + var toNanos = 2_000_000_000L; + + await WriteTestRawFile(_stream1, venue, [ + (RecordType.Snap, RecordSide.Bid, 100, 10, 1_000_000_000), + (RecordType.Snap, RecordSide.Ask, 110, 5, 1_000_000_000), + ]); + + await WriteTestRawFile(_stream2, venue, [ + (RecordType.Snap, RecordSide.Bid, 200, 20, 1_000_000_000), + (RecordType.Snap, RecordSide.Ask, 210, 15, 1_000_000_000), + ]); + + var fileIndex = new FileIndex(); + fileIndex.LoadFromDisk(_dataPath); + + var queryConfig = new QueryConfig + { + DataPath = _dataPath, + CompactionWindow = TimeSpan.FromSeconds(10), + }; + var queryLayer = new QueryLayer(fileIndex, queryConfig); + + var config = new ResampledStreamConfig + { + ConfigVersion = "batch_multi_v1", + SourceStreams = [_stream1, _stream2], + ResamplingWindow = TimeSpan.FromSeconds(1), + OutputType = ResampledOutputType.TopOfBook, + Venue = venue, + OutputPath = _dataPath, + }; + + var processor = new BatchResampledProcessor(config, queryLayer, fileIndex); + await processor.ProcessAsync(fromNanos, toNanos); + + var resampledFiles = Directory.GetFiles(_dataPath, "*.resampled", SearchOption.AllDirectories); + Assert.Single(resampledFiles); + + using var fs = File.OpenRead(resampledFiles[0]); + var reader = new BinaryRecordReader(fs); + var records = reader.ReadRecords(validateCrc: true).ToList(); + // 2 streams * 2 sides = 4 records per window + Assert.Equal(4, records.Count); + } + + [Fact] + public async Task ProcessAsync_ConfigHashEmbedded() + { + var venue = "testex"; + await WriteTestRawFile(_stream1, venue, [ + (RecordType.Snap, RecordSide.Bid, 100, 10, 1_000_000_000), + (RecordType.Snap, RecordSide.Ask, 110, 5, 1_000_000_000), + ]); + + var fileIndex = new FileIndex(); + fileIndex.LoadFromDisk(_dataPath); + + var queryConfig = new QueryConfig { DataPath = _dataPath, CompactionWindow = TimeSpan.FromSeconds(10) }; + var queryLayer = new QueryLayer(fileIndex, queryConfig); + + var config = new ResampledStreamConfig + { + ConfigVersion = "hash_test_v1", + SourceStreams = [_stream1], + ResamplingWindow = TimeSpan.FromSeconds(1), + Venue = venue, + OutputPath = _dataPath, + }; + + var processor = new BatchResampledProcessor(config, queryLayer, fileIndex); + await processor.ProcessAsync(1_000_000_000, 2_000_000_000); + + var resampledFiles = Directory.GetFiles(_dataPath, "*.resampled", SearchOption.AllDirectories); + using var fs = File.OpenRead(resampledFiles[0]); + var reader = new BinaryRecordReader(fs); + Assert.Equal(config.ComputeConfigHash(), reader.Header.ResampledConfigHash); + } + + public void Dispose() + { + try { Directory.Delete(_dataPath, recursive: true); } catch { } + } +} diff --git a/tests/Levels.Tests/Resampled/LiveResampledHandlerTests.cs b/tests/Levels.Tests/Resampled/LiveResampledHandlerTests.cs new file mode 100644 index 0000000..0b6f5c3 --- /dev/null +++ b/tests/Levels.Tests/Resampled/LiveResampledHandlerTests.cs @@ -0,0 +1,163 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Query; +using Levels.Resampled; + +namespace Levels.Tests.Resampled; + +public class LiveResampledHandlerTests : IDisposable +{ + private readonly string _dataPath; + private readonly PriceStreamId _sourceStream = PriceStreamId.FromSymbol("BTC-USD"); + private readonly PriceStreamId _otherStream = PriceStreamId.FromSymbol("DOGE-USD"); + + public LiveResampledHandlerTests() + { + _dataPath = Path.Combine(Path.GetTempPath(), $"levels_live_resampled_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_dataPath); + } + + private (LiveResampledHandler Handler, FileIndex Index) CreateHandler( + ResampledOutputType outputType = ResampledOutputType.TopOfBook, + int windowMs = 100) + { + var fileIndex = new FileIndex(); + var config = new ResampledStreamConfig + { + ConfigVersion = "test_v1", + SourceStreams = [_sourceStream], + ResamplingWindow = TimeSpan.FromMilliseconds(windowMs), + OutputType = outputType, + Venue = "testex", + OutputPath = _dataPath, + PriceScale = 2, + QuantityScale = 4, + }; + return (new LiveResampledHandler(config, fileIndex), fileIndex); + } + + private static RawRecord MakeRecord(RecordType type, RecordSide side, long price, long quantity, long streamId = 0) + { + return new RawRecord + { + Core = new CoreRecordLayout + { + ObservedTime = 1000, + PriceStreamId = streamId, + Price = price, + Quantity = quantity, + Type = type, + Side = side, + }, + }; + } + + [Fact] + public async Task NonSourceRecords_AreIgnored() + { + var (handler, _) = CreateHandler(); + var record = MakeRecord(RecordType.Snap, RecordSide.Bid, 100, 10); + + // Should not throw for non-source streams + await handler.OnRecordWritten(_otherStream, record, CancellationToken.None); + } + + [Fact] + public async Task SourceRecords_UpdateState_AndEmitProducesFile() + { + var (handler, fileIndex) = CreateHandler(); + + // Feed records + await handler.OnRecordWritten(_sourceStream, + MakeRecord(RecordType.Snap, RecordSide.Bid, 100, 10), CancellationToken.None); + await handler.OnRecordWritten(_sourceStream, + MakeRecord(RecordType.Snap, RecordSide.Ask, 105, 5), CancellationToken.None); + + // Emit directly + var boundaryNanos = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() * 1_000_000L; + await handler.EmitWindowAsync(boundaryNanos, CancellationToken.None); + + // Seal to finalize the file + await handler.StopAsync(CancellationToken.None); + + // Check that a .resampled file was created + var files = Directory.GetFiles(_dataPath, "*.resampled", SearchOption.AllDirectories); + Assert.Single(files); + + // Verify header + using var fs = File.OpenRead(files[0]); + var reader = new BinaryRecordReader(fs); + Assert.Equal(FileType.Resampled, reader.Header.FileType); + Assert.NotEqual(0u, reader.Header.ResampledConfigHash); + } + + [Fact] + public async Task ConfigHash_EmbeddedCorrectly() + { + var (handler, _) = CreateHandler(); + + await handler.OnRecordWritten(_sourceStream, + MakeRecord(RecordType.Snap, RecordSide.Bid, 100, 10), CancellationToken.None); + await handler.OnRecordWritten(_sourceStream, + MakeRecord(RecordType.Snap, RecordSide.Ask, 105, 5), CancellationToken.None); + + await handler.EmitWindowAsync(1_000_000_000L, CancellationToken.None); + await handler.StopAsync(CancellationToken.None); + + var files = Directory.GetFiles(_dataPath, "*.resampled", SearchOption.AllDirectories); + using var fs = File.OpenRead(files[0]); + var reader = new BinaryRecordReader(fs); + + var expectedHash = new ResampledStreamConfig + { + ConfigVersion = "test_v1", + SourceStreams = [_sourceStream], + ResamplingWindow = TimeSpan.FromMilliseconds(100), + Venue = "testex", + OutputPath = _dataPath, + }.ComputeConfigHash(); + + Assert.Equal(expectedHash, reader.Header.ResampledConfigHash); + } + + [Fact] + public async Task StopAsync_SealsOpenFile() + { + var (handler, _) = CreateHandler(); + + await handler.OnRecordWritten(_sourceStream, + MakeRecord(RecordType.Snap, RecordSide.Bid, 100, 10), CancellationToken.None); + await handler.OnRecordWritten(_sourceStream, + MakeRecord(RecordType.Snap, RecordSide.Ask, 105, 5), CancellationToken.None); + + await handler.EmitWindowAsync(1_000_000_000L, CancellationToken.None); + await handler.StopAsync(CancellationToken.None); + + var files = Directory.GetFiles(_dataPath, "*.resampled", SearchOption.AllDirectories); + Assert.Single(files); + + // File should have valid footer + using var fs = File.OpenRead(files[0]); + var reader = new BinaryRecordReader(fs); + Assert.NotNull(reader.Footer); + } + + [Fact] + public async Task EmitWindow_NoRecords_NoFileCreated() + { + var (handler, _) = CreateHandler(); + + // Emit without any records + await handler.EmitWindowAsync(1_000_000_000L, CancellationToken.None); + await handler.StopAsync(CancellationToken.None); + + var files = Directory.GetFiles(_dataPath, "*.resampled", SearchOption.AllDirectories); + Assert.Empty(files); + } + + public void Dispose() + { + try { Directory.Delete(_dataPath, recursive: true); } catch { } + } +} diff --git a/tests/Levels.Tests/Resampled/OhlcvBarEmitterTests.cs b/tests/Levels.Tests/Resampled/OhlcvBarEmitterTests.cs new file mode 100644 index 0000000..8ec95d3 --- /dev/null +++ b/tests/Levels.Tests/Resampled/OhlcvBarEmitterTests.cs @@ -0,0 +1,84 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Resampled; + +namespace Levels.Tests.Resampled; + +public class OhlcvBarEmitterTests +{ + private static readonly PriceStreamId Stream1 = PriceStreamId.FromSymbol("BTC-USD"); + + private static RawRecord MakeRecord(RecordType type, RecordSide side, long price, long quantity) + { + return new RawRecord + { + Core = new CoreRecordLayout + { + ObservedTime = 1000, + Price = price, + Quantity = quantity, + Type = type, + Side = side, + }, + }; + } + + [Fact] + public void Emit_WithOhlcvData_ProducesFourRecords() + { + var emitter = new OhlcvBarEmitter(); + var state = new OrderbookState(); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Bid, 100, 10)); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Ask, 110, 5)); + // Mid = 105 + + state.Apply(MakeRecord(RecordType.Delta, RecordSide.Bid, 108, 10)); + // Mid = (108+110)/2 = 109 + + state.Apply(MakeRecord(RecordType.Delta, RecordSide.Bid, 96, 10)); + // Mid = (96+110)/2 = 103 (but best bid is 108, not 96, since 108 still there) + // Actually, bid 108 with qty 10 replaces previous, and we also have bid 96 + // Best bid = 108, mid = (108+110)/2 = 109 + + var states = new Dictionary { [Stream1] = state }; + var results = emitter.Emit(2000, states).ToList(); + + Assert.Equal(4, results.Count); + // Level 0 = Open, Level 1 = High, Level 2 = Low, Level 3 = Close + Assert.Equal(0, results[0].Core.Level); + Assert.Equal(1, results[1].Core.Level); + Assert.Equal(2, results[2].Core.Level); + Assert.Equal(3, results[3].Core.Level); + + Assert.Equal(105L, results[0].Core.Price); // Open (first mid) + Assert.Equal(2000L, results[0].Core.ObservedTime); + } + + [Fact] + public void Emit_NoData_SkipsStream() + { + var emitter = new OhlcvBarEmitter(); + var states = new Dictionary + { + [Stream1] = new OrderbookState(), + }; + + var results = emitter.Emit(2000, states).ToList(); + Assert.Empty(results); + } + + [Fact] + public void Emit_ResetsWindowAfterEmission() + { + var emitter = new OhlcvBarEmitter(); + var state = new OrderbookState(); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Bid, 100, 10)); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Ask, 110, 5)); + + var states = new Dictionary { [Stream1] = state }; + emitter.Emit(2000, states).ToList(); + + // After emission, OHLCV should be reset + Assert.False(state.HasOhlcvData); + } +} diff --git a/tests/Levels.Tests/Resampled/OrderbookStateTests.cs b/tests/Levels.Tests/Resampled/OrderbookStateTests.cs new file mode 100644 index 0000000..c90b66f --- /dev/null +++ b/tests/Levels.Tests/Resampled/OrderbookStateTests.cs @@ -0,0 +1,142 @@ +using Levels.Core.Format; +using Levels.Resampled; + +namespace Levels.Tests.Resampled; + +public class OrderbookStateTests +{ + private static RawRecord MakeRecord(RecordType type, RecordSide side, long price, long quantity) + { + return new RawRecord + { + Core = new CoreRecordLayout + { + ObservedTime = 1000, + PriceStreamId = 42, + Price = price, + Quantity = quantity, + Type = type, + Side = side, + }, + RecordBytes = ReadOnlyMemory.Empty, + }; + } + + [Fact] + public void Apply_Snap_BuildsCorrectState() + { + var state = new OrderbookState(); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Bid, 100, 10)); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Ask, 105, 5)); + + Assert.Equal((100L, 10L), state.BestBid); + Assert.Equal((105L, 5L), state.BestAsk); + } + + [Fact] + public void Apply_Delta_UpdatesExistingLevel() + { + var state = new OrderbookState(); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Bid, 100, 10)); + state.Apply(MakeRecord(RecordType.Delta, RecordSide.Bid, 100, 20)); + + Assert.Equal((100L, 20L), state.BestBid); + } + + [Fact] + public void Apply_ZeroQuantity_RemovesLevel() + { + var state = new OrderbookState(); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Bid, 100, 10)); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Bid, 90, 5)); + state.Apply(MakeRecord(RecordType.Delta, RecordSide.Bid, 100, 0)); + + Assert.Equal((90L, 5L), state.BestBid); + } + + [Fact] + public void BestBid_MultipleLevels_ReturnsHighest() + { + var state = new OrderbookState(); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Bid, 90, 5)); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Bid, 100, 10)); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Bid, 95, 8)); + + Assert.Equal((100L, 10L), state.BestBid); + } + + [Fact] + public void BestAsk_MultipleLevels_ReturnsLowest() + { + var state = new OrderbookState(); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Ask, 110, 5)); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Ask, 105, 10)); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Ask, 115, 8)); + + Assert.Equal((105L, 10L), state.BestAsk); + } + + [Fact] + public void EmptyState_ReturnsNull() + { + var state = new OrderbookState(); + Assert.Null(state.BestBid); + Assert.Null(state.BestAsk); + } + + [Fact] + public void Clear_ResetsAllState() + { + var state = new OrderbookState(); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Bid, 100, 10)); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Ask, 105, 5)); + state.Clear(); + + Assert.Null(state.BestBid); + Assert.Null(state.BestAsk); + Assert.False(state.HasOhlcvData); + } + + [Fact] + public void OhlcvTracking_UpdatesOnMidPriceChange() + { + var state = new OrderbookState(); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Bid, 100, 10)); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Ask, 110, 5)); + + Assert.True(state.HasOhlcvData); + Assert.Equal(105L, state.WindowOpen); + Assert.Equal(105L, state.WindowClose); + + // Move bid up + state.Apply(MakeRecord(RecordType.Delta, RecordSide.Bid, 104, 10)); + + Assert.Equal(105L, state.WindowOpen); // Open stays the same + Assert.Equal(107L, state.WindowClose); // (104 + 110) / 2 + Assert.Equal(107L, state.WindowHigh); + Assert.Equal(105L, state.WindowLow); + } + + [Fact] + public void ResetWindow_ClearsOhlcvButKeepsBook() + { + var state = new OrderbookState(); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Bid, 100, 10)); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Ask, 110, 5)); + state.ResetWindow(); + + Assert.False(state.HasOhlcvData); + Assert.Null(state.WindowOpen); + // Book state preserved + Assert.Equal((100L, 10L), state.BestBid); + Assert.Equal((110L, 5L), state.BestAsk); + } + + [Fact] + public void Tombstone_IsIgnored() + { + var state = new OrderbookState(); + state.Apply(MakeRecord(RecordType.Tombstone, RecordSide.Bid, 100, 10)); + Assert.Null(state.BestBid); + } +} diff --git a/tests/Levels.Tests/Resampled/ResampledQueryTests.cs b/tests/Levels.Tests/Resampled/ResampledQueryTests.cs new file mode 100644 index 0000000..c058288 --- /dev/null +++ b/tests/Levels.Tests/Resampled/ResampledQueryTests.cs @@ -0,0 +1,143 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Query; + +namespace Levels.Tests.Resampled; + +public class ResampledQueryTests : IDisposable +{ + private readonly string _dataPath; + + public ResampledQueryTests() + { + _dataPath = Path.Combine(Path.GetTempPath(), $"levels_resampled_query_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_dataPath); + } + + private async Task WriteTestFile(PriceStreamId streamId, FileType fileType, string venue, + long firstTime, long lastTime, string ext) + { + var dir = Path.Combine(_dataPath, venue, streamId.Value.ToString()); + Directory.CreateDirectory(dir); + var filePath = Path.Combine(dir, $"{Guid.NewGuid():N}.{ext}"); + + await using var fs = new FileStream(filePath, FileMode.CreateNew, FileAccess.Write, FileShare.None); + await using var writer = await BinaryRecordWriter.CreateAsync(fs, fileType, streamId); + + var core = new CoreRecordLayout + { + ObservedTime = firstTime, + PriceStreamId = streamId.Value, + Price = 100, + Quantity = 10, + Type = RecordType.Snap, + Side = RecordSide.Bid, + }; + await writer.WriteRecordAsync(core); + + if (lastTime != firstTime) + { + core.ObservedTime = lastTime; + core.Sequence = 1; + await writer.WriteRecordAsync(core); + } + + await writer.SealAsync(); + return filePath; + } + + [Fact] + public async Task Resolve_ExcludesResampledFiles() + { + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + + await WriteTestFile(streamId, FileType.Raw, "testex", 1000, 2000, "raw"); + await WriteTestFile(streamId, FileType.Resampled, "testex", 1000, 2000, "resampled"); + + var fileIndex = new FileIndex(); + fileIndex.LoadFromDisk(_dataPath); + + var queryConfig = new QueryConfig { DataPath = _dataPath, CompactionWindow = TimeSpan.FromHours(1) }; + var queryLayer = new QueryLayer(fileIndex, queryConfig); + + var results = queryLayer.Resolve(streamId, 0, 3000); + Assert.All(results, r => Assert.NotEqual(FileType.Resampled, r.FileType)); + Assert.Single(results); // Only the RAW file + } + + [Fact] + public async Task ResolveResampled_ReturnsOnlyResampledFiles() + { + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + + await WriteTestFile(streamId, FileType.Raw, "testex", 1000, 2000, "raw"); + await WriteTestFile(streamId, FileType.Resampled, "testex", 1000, 2000, "resampled"); + + var fileIndex = new FileIndex(); + fileIndex.LoadFromDisk(_dataPath); + + var queryConfig = new QueryConfig { DataPath = _dataPath, CompactionWindow = TimeSpan.FromHours(1) }; + var queryLayer = new QueryLayer(fileIndex, queryConfig); + + var results = queryLayer.ResolveResampled(streamId, 0, 3000); + Assert.Single(results); + Assert.Equal(FileType.Resampled, results[0].FileType); + } + + [Fact] + public async Task ResolveResampled_SortedByFirstObservedTime() + { + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + + await WriteTestFile(streamId, FileType.Resampled, "testex", 2000, 3000, "resampled"); + await WriteTestFile(streamId, FileType.Resampled, "testex", 1000, 1500, "resampled"); + + var fileIndex = new FileIndex(); + fileIndex.LoadFromDisk(_dataPath); + + var queryConfig = new QueryConfig { DataPath = _dataPath, CompactionWindow = TimeSpan.FromHours(1) }; + var queryLayer = new QueryLayer(fileIndex, queryConfig); + + var results = queryLayer.ResolveResampled(streamId, 0, 4000); + Assert.Equal(2, results.Count); + Assert.True(results[0].FirstObservedTime <= results[1].FirstObservedTime); + } + + [Fact] + public async Task ResampledFiles_IndexedOnLoadFromDisk() + { + // Create a resampled file manually + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + var dir = Path.Combine(_dataPath, "testex", streamId.Value.ToString()); + Directory.CreateDirectory(dir); + var filePath = Path.Combine(dir, "test.resampled"); + + await using (var fs = new FileStream(filePath, FileMode.CreateNew, FileAccess.Write, FileShare.None)) + { + var writer = await BinaryRecordWriter.CreateAsync(fs, FileType.Resampled, streamId); + var core = new CoreRecordLayout + { + ObservedTime = 1000, + PriceStreamId = streamId.Value, + Price = 100, + Quantity = 10, + Type = RecordType.Snap, + Side = RecordSide.Bid, + }; + await writer.WriteRecordAsync(core); + await writer.SealAsync(); + } + + var fileIndex = new FileIndex(); + fileIndex.LoadFromDisk(_dataPath); + + var entries = fileIndex.Query(streamId, 0, 2000, FileType.Resampled); + Assert.Single(entries); + } + + public void Dispose() + { + try { Directory.Delete(_dataPath, recursive: true); } catch { } + } +} diff --git a/tests/Levels.Tests/Resampled/TopOfBookEmitterTests.cs b/tests/Levels.Tests/Resampled/TopOfBookEmitterTests.cs new file mode 100644 index 0000000..f7392c3 --- /dev/null +++ b/tests/Levels.Tests/Resampled/TopOfBookEmitterTests.cs @@ -0,0 +1,94 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Resampled; + +namespace Levels.Tests.Resampled; + +public class TopOfBookEmitterTests +{ + private static readonly PriceStreamId Stream1 = PriceStreamId.FromSymbol("BTC-USD"); + private static readonly PriceStreamId Stream2 = PriceStreamId.FromSymbol("ETH-USD"); + + private static RawRecord MakeRecord(RecordType type, RecordSide side, long price, long quantity) + { + return new RawRecord + { + Core = new CoreRecordLayout + { + ObservedTime = 1000, + Price = price, + Quantity = quantity, + Type = type, + Side = side, + }, + }; + } + + [Fact] + public void Emit_WithData_ProducesBidAndAskSnaps() + { + var emitter = new TopOfBookEmitter(); + var states = new Dictionary(); + var state = new OrderbookState(); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Bid, 100, 10)); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Ask, 105, 5)); + states[Stream1] = state; + + var results = emitter.Emit(2000, states).ToList(); + + Assert.Equal(2, results.Count); + Assert.Equal(RecordSide.Bid, results[0].Core.Side); + Assert.Equal(100L, results[0].Core.Price); + Assert.Equal(10L, results[0].Core.Quantity); + Assert.Equal(RecordSide.Ask, results[1].Core.Side); + Assert.Equal(105L, results[1].Core.Price); + Assert.Equal(2000L, results[0].Core.ObservedTime); + } + + [Fact] + public void Emit_EmptyOrderbook_SkipsStream() + { + var emitter = new TopOfBookEmitter(); + var states = new Dictionary + { + [Stream1] = new OrderbookState(), + }; + + var results = emitter.Emit(2000, states).ToList(); + Assert.Empty(results); + } + + [Fact] + public void Emit_MultipleStreams_EmitsAll() + { + var emitter = new TopOfBookEmitter(); + var states = new Dictionary(); + + var state1 = new OrderbookState(); + state1.Apply(MakeRecord(RecordType.Snap, RecordSide.Bid, 100, 10)); + state1.Apply(MakeRecord(RecordType.Snap, RecordSide.Ask, 105, 5)); + states[Stream1] = state1; + + var state2 = new OrderbookState(); + state2.Apply(MakeRecord(RecordType.Snap, RecordSide.Bid, 200, 20)); + state2.Apply(MakeRecord(RecordType.Snap, RecordSide.Ask, 210, 15)); + states[Stream2] = state2; + + var results = emitter.Emit(2000, states).ToList(); + Assert.Equal(4, results.Count); + } + + [Fact] + public void Emit_OnlyBidSide_EmitsOnlyBid() + { + var emitter = new TopOfBookEmitter(); + var states = new Dictionary(); + var state = new OrderbookState(); + state.Apply(MakeRecord(RecordType.Snap, RecordSide.Bid, 100, 10)); + states[Stream1] = state; + + var results = emitter.Emit(2000, states).ToList(); + Assert.Single(results); + Assert.Equal(RecordSide.Bid, results[0].Core.Side); + } +} diff --git a/tests/Levels.Tests/Sinks/IngestionTests.cs b/tests/Levels.Tests/Sinks/IngestionTests.cs new file mode 100644 index 0000000..8d7b8ab --- /dev/null +++ b/tests/Levels.Tests/Sinks/IngestionTests.cs @@ -0,0 +1,142 @@ +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Sinks; + +namespace Levels.Tests.Sinks; + +public class IngestionTests : IDisposable +{ + private readonly string _tempDir; + + public IngestionTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + [Fact] + public async Task Ingestion_MultipleSymbols_CreatesFilesPerSymbol() + { + var config = new SinkConfig { OutputPath = _tempDir }; + var sink = new PriceStreamSink(config); + + await sink.StartAsync(CancellationToken.None); + + var symbols = new[] { "BTC-USD", "ETH-USD", "SOL-USD" }; + var eventsPerSymbol = 1000 / symbols.Length; // ~333 per symbol + + // Write snap + deltas for each symbol + foreach (var symbol in symbols) + { + // Initial snap (bid + ask) + await sink.Writer.WriteAsync(new RawMarketEvent( + "binance", symbol, 1000, 50000, 100, + RecordType.Snap, RecordSide.Bid)); + await sink.Writer.WriteAsync(new RawMarketEvent( + "binance", symbol, 1000, 50010, 100, + RecordType.Snap, RecordSide.Ask)); + + for (var i = 0; i < eventsPerSymbol; i++) + { + await sink.Writer.WriteAsync(new RawMarketEvent( + "binance", symbol, 2000 + i, 50000 + i, 10 + i, + RecordType.Delta, RecordSide.Bid)); + } + } + + await sink.StopAsync(CancellationToken.None); + + // Should have one file per symbol + var files = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories); + Assert.Equal(3, files.Length); + + // Verify each file + foreach (var file in files) + { + using var fs = File.OpenRead(file); + var reader = new BinaryRecordReader(fs); + Assert.NotNull(reader.Footer); + Assert.False(reader.IsPartial); + + var records = reader.ReadRecords(validateCrc: true).ToList(); + Assert.True(records.Count > 0); + + // First record should be a SNAP + Assert.Equal(RecordType.Snap, records[0].Core.Type); + + // All CRCs are valid (would throw if not, since validateCrc: true) + } + } + + [Fact] + public async Task Ingestion_TwoVenues_SameSymbol_CreatesSeparateWriters() + { + var config = new SinkConfig { OutputPath = _tempDir }; + var sink = new PriceStreamSink(config); + + await sink.StartAsync(CancellationToken.None); + + // Write events from two different venues with the same symbol + await sink.WriteAsync(new RawMarketEvent( + "binance", "BTC-USD", 1000, 50000, 100, + RecordType.Snap, RecordSide.Bid)); + await sink.WriteAsync(new RawMarketEvent( + "coinbase", "BTC-USD", 1000, 50100, 200, + RecordType.Snap, RecordSide.Bid)); + + await sink.StopAsync(CancellationToken.None); + + // Should have TWO separate directories (one per venue) + var venueDirs = Directory.GetDirectories(_tempDir); + Assert.Equal(2, venueDirs.Length); + + var venueNames = venueDirs.Select(d => Path.GetFileName(d)).OrderBy(x => x).ToArray(); + Assert.Contains("binance", venueNames); + Assert.Contains("coinbase", venueNames); + + // Each venue directory should have a raw file + var rawFiles = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories); + Assert.Equal(2, rawFiles.Length); + } + + [Fact] + public async Task Ingestion_RandomDataAdapter_ProducesReadableFiles() + { + var config = new SinkConfig { OutputPath = _tempDir }; + var sink = new PriceStreamSink(config); + + await sink.StartAsync(CancellationToken.None); + + await RandomDataAdapter.GenerateAsync( + sink.Writer, + "test-exchange", + ["SYM-A", "SYM-B", "SYM-C"], + levelsPerSide: 5, + deltasPerSymbol: 100); + + await sink.StopAsync(CancellationToken.None); + + var files = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories); + Assert.Equal(3, files.Length); + + foreach (var file in files) + { + using var fs = File.OpenRead(file); + var reader = new BinaryRecordReader(fs); + Assert.NotNull(reader.Footer); + + // Header is valid + Assert.Equal(Constants.HeaderMagicUInt64, reader.Header.Magic); + + // Records are readable with CRC validation + var records = reader.ReadRecords(validateCrc: true).ToList(); + Assert.True(records.Count > 0); + } + } +} diff --git a/tests/Levels.Tests/Sinks/IsOwnerFlagTests.cs b/tests/Levels.Tests/Sinks/IsOwnerFlagTests.cs new file mode 100644 index 0000000..c7f5564 --- /dev/null +++ b/tests/Levels.Tests/Sinks/IsOwnerFlagTests.cs @@ -0,0 +1,59 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Sinks; + +namespace Levels.Tests.Sinks; + +public class IsOwnerFlagTests : IDisposable +{ + private readonly string _tempDir; + + public IsOwnerFlagTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + [Fact] + public async Task IsOwner_SetsFlag() + { + var config = new SinkConfig { OutputPath = _tempDir }; + var sink = new PriceStreamSink(config); + await sink.StartAsync(CancellationToken.None); + + var now = WriteTimestamp.Now(); + await sink.Writer.WriteAsync(new RawMarketEvent("ex", "BTC-USD", now, 100, 50, RecordType.Snap, RecordSide.Bid)); + await sink.Writer.WriteAsync(new RawMarketEvent("ex", "BTC-USD", now + 1000, 101, 30, RecordType.Delta, RecordSide.Bid, IsOwner: true)); + await sink.Writer.WriteAsync(new RawMarketEvent("ex", "BTC-USD", now + 2000, 102, 20, RecordType.Delta, RecordSide.Ask, IsOwner: false)); + + await sink.StopAsync(CancellationToken.None); + + var rawFiles = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories); + Assert.NotEmpty(rawFiles); + + using var fs = File.OpenRead(rawFiles[0]); + var reader = new BinaryRecordReader(fs); + var records = reader.ReadRecords().ToList(); + + // Find the owner record (second event, but there may be leading snap records) + var ownerRecord = records.FirstOrDefault(r => r.Core.Price == 101 && r.Core.Type == RecordType.Delta); + Assert.Equal(Constants.IsOwnerFlag, (ushort)(ownerRecord.Core.Flags & Constants.IsOwnerFlag)); + + // Non-owner record + var nonOwnerRecord = records.FirstOrDefault(r => r.Core.Price == 102 && r.Core.Type == RecordType.Delta); + Assert.Equal(0, nonOwnerRecord.Core.Flags & Constants.IsOwnerFlag); + } + + [Fact] + public void IsOwnerFlag_HasExpectedValue() + { + Assert.Equal(0x0002, Constants.IsOwnerFlag); + } +} diff --git a/tests/Levels.Tests/Sinks/OrderbookSideTests.cs b/tests/Levels.Tests/Sinks/OrderbookSideTests.cs new file mode 100644 index 0000000..d627484 --- /dev/null +++ b/tests/Levels.Tests/Sinks/OrderbookSideTests.cs @@ -0,0 +1,86 @@ +using Levels.Core.Format; +using Levels.Core.Orderbook; + +namespace Levels.Tests.Sinks; + +public class OrderbookSideTests +{ + [Fact] + public void Apply_AddsLevel() + { + var side = new OrderbookSide(); + side.Apply(100, 50); + Assert.Single(side.Levels); + Assert.Equal(50, side.Levels.First().Value); + } + + [Fact] + public void Apply_UpdatesExistingLevel() + { + var side = new OrderbookSide(); + side.Apply(100, 50); + side.Apply(100, 75); + Assert.Single(side.Levels); + Assert.Equal(75, side.Levels.First().Value); + } + + [Fact] + public void Apply_ZeroQuantity_RemovesLevel() + { + var side = new OrderbookSide(); + side.Apply(100, 50); + side.Apply(100, 0); + Assert.Empty(side.Levels); + } + + [Fact] + public void Clear_RemovesAllLevels() + { + var side = new OrderbookSide(); + side.Apply(100, 50); + side.Apply(200, 75); + side.Clear(); + Assert.Empty(side.Levels); + } + + [Fact] + public void Levels_AreSortedByPrice() + { + var side = new OrderbookSide(); + side.Apply(300, 10); + side.Apply(100, 30); + side.Apply(200, 20); + + var prices = side.Levels.Select(l => l.Key).ToList(); + Assert.Equal([100L, 200L, 300L], prices); + } + + [Fact] + public void Unlimited_Depth_AcceptsAllLevels() + { + var side = new OrderbookSide(RecordSide.Bid); + for (var i = 0; i < 1000; i++) + side.Apply(i, 10); + + Assert.Equal(1000, side.Levels.Count()); + } + + [Fact] + public void Removal_ReducesCount() + { + var side = new OrderbookSide(RecordSide.Bid); + side.Apply(100, 10); + side.Apply(200, 20); + side.Apply(300, 30); + + side.Apply(200, 0); + Assert.Equal(2, side.Levels.Count()); + + side.Apply(50, 5); + Assert.Equal(3, side.Levels.Count()); + var prices = side.Levels.Select(l => l.Key).ToList(); + Assert.Contains(50L, prices); + Assert.Contains(100L, prices); + Assert.Contains(300L, prices); + } +} diff --git a/tests/Levels.Tests/Sinks/PriceStreamIdTests.cs b/tests/Levels.Tests/Sinks/PriceStreamIdTests.cs new file mode 100644 index 0000000..464cd83 --- /dev/null +++ b/tests/Levels.Tests/Sinks/PriceStreamIdTests.cs @@ -0,0 +1,54 @@ +using Levels.Core; + +namespace Levels.Tests.Sinks; + +public class PriceStreamIdTests +{ + [Fact] + public void FromSymbol_IsDeterministic() + { + var a = PriceStreamId.FromSymbol("BTC-USD"); + var b = PriceStreamId.FromSymbol("BTC-USD"); + Assert.Equal(a, b); + } + + [Fact] + public void FromSymbol_DifferentSymbols_ProduceDifferentIds() + { + var a = PriceStreamId.FromSymbol("BTC-USD"); + var b = PriceStreamId.FromSymbol("ETH-USD"); + Assert.NotEqual(a, b); + } + + [Fact] + public void FromSymbol_EmptyString_DoesNotThrow() + { + var id = PriceStreamId.FromSymbol(""); + Assert.NotEqual(0, id.Value); + } + + [Fact] + public void FromVenueSymbol_DifferentVenues_SameSymbol_ProduceDifferentIds() + { + var a = PriceStreamId.FromVenueSymbol("binance", "BTC-USD"); + var b = PriceStreamId.FromVenueSymbol("coinbase", "BTC-USD"); + Assert.NotEqual(a, b); + } + + [Fact] + public void FromVenueSymbol_IsDeterministic() + { + var a = PriceStreamId.FromVenueSymbol("binance", "BTC-USD"); + var b = PriceStreamId.FromVenueSymbol("binance", "BTC-USD"); + Assert.Equal(a, b); + } + + [Fact] + public void FromVenueSymbol_SeparatorPreventsCollisions() + { + // "ab" + "cd" should not equal "a" + "bcd" even though concatenated they look similar + var a = PriceStreamId.FromVenueSymbol("ab", "cd"); + var b = PriceStreamId.FromVenueSymbol("a", "bcd"); + Assert.NotEqual(a, b); + } +} diff --git a/tests/Levels.Tests/Sinks/PriceStreamWriterTests.cs b/tests/Levels.Tests/Sinks/PriceStreamWriterTests.cs new file mode 100644 index 0000000..bef4f4b --- /dev/null +++ b/tests/Levels.Tests/Sinks/PriceStreamWriterTests.cs @@ -0,0 +1,130 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Sinks; + +namespace Levels.Tests.Sinks; + +public class PriceStreamWriterTests : IDisposable +{ + private readonly string _tempDir; + + public PriceStreamWriterTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + [Fact] + public async Task WriteSingleEvent_CreatesFileWithHeaderAndRecord() + { + var config = new SinkConfig { OutputPath = _tempDir }; + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + + await using (var writer = new PriceStreamWriter(streamId, "binance", config)) + { + await writer.WriteAsync(new RawMarketEvent( + "binance", "BTC-USD", 1000, 50000, 100, + RecordType.Snap, RecordSide.Bid), CancellationToken.None); + } + + var files = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories); + Assert.Single(files); + + // Read back and verify + using var fs = File.OpenRead(files[0]); + var reader = new BinaryRecordReader(fs); + Assert.Equal(FileType.Raw, reader.Header.FileType); + Assert.Equal(streamId.Value, reader.Header.PriceStreamId); + Assert.NotNull(reader.Footer); + + var records = reader.ReadRecords().ToList(); + Assert.Single(records); + Assert.Equal(RecordType.Snap, records[0].Core.Type); + } + + [Fact] + public async Task DeltaWriting_UpdatesOrderbookAndWritesRecords() + { + var config = new SinkConfig { OutputPath = _tempDir }; + var streamId = PriceStreamId.FromSymbol("ETH-USD"); + + await using (var writer = new PriceStreamWriter(streamId, "binance", config)) + { + // Initial snap + await writer.WriteAsync(new RawMarketEvent( + "binance", "ETH-USD", 1000, 3000, 10, + RecordType.Snap, RecordSide.Bid), CancellationToken.None); + + // Delta + await writer.WriteAsync(new RawMarketEvent( + "binance", "ETH-USD", 2000, 3001, 5, + RecordType.Delta, RecordSide.Ask), CancellationToken.None); + } + + var files = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories); + using var fs = File.OpenRead(files[0]); + var reader = new BinaryRecordReader(fs); + var records = reader.ReadRecords().ToList(); + + Assert.Equal(2, records.Count); + Assert.Equal(RecordType.Snap, records[0].Core.Type); + Assert.Equal(RecordType.Delta, records[1].Core.Type); + } + + [Fact] + public async Task Seal_WritesFooterWithCorrectStats() + { + var config = new SinkConfig { OutputPath = _tempDir }; + var streamId = PriceStreamId.FromSymbol("SOL-USD"); + + await using (var writer = new PriceStreamWriter(streamId, "binance", config)) + { + await writer.WriteAsync(new RawMarketEvent( + "binance", "SOL-USD", 1000, 100, 50, + RecordType.Snap, RecordSide.Bid), CancellationToken.None); + await writer.WriteAsync(new RawMarketEvent( + "binance", "SOL-USD", 2000, 101, 25, + RecordType.Delta, RecordSide.Ask), CancellationToken.None); + } + + var files = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories); + using var fs = File.OpenRead(files[0]); + var reader = new BinaryRecordReader(fs); + + Assert.NotNull(reader.Footer); + Assert.Equal(2, reader.Footer.Value.RecordCount); + Assert.Equal(1, reader.Footer.Value.DeltaCount); + } + + [Fact] + public async Task FileNaming_FollowsConvention() + { + var config = new SinkConfig { OutputPath = _tempDir }; + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + + await using (var writer = new PriceStreamWriter(streamId, "binance", config)) + { + await writer.WriteAsync(new RawMarketEvent( + "binance", "BTC-USD", 1000, 50000, 100, + RecordType.Snap, RecordSide.Bid), CancellationToken.None); + } + + var files = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories); + var file = files[0]; + + // Should contain exchange and stream id in path + Assert.Contains("binance", file); + Assert.Contains(streamId.Value.ToString(), file); + + // Filename should match pattern yyyyMMdd_NNNNNN.raw + var fileName = Path.GetFileName(file); + Assert.Matches(@"^\d{8}_\d{6}\.raw$", fileName); + } +} diff --git a/tests/Levels.Tests/Sinks/RecoveryTests.cs b/tests/Levels.Tests/Sinks/RecoveryTests.cs new file mode 100644 index 0000000..4d654e7 --- /dev/null +++ b/tests/Levels.Tests/Sinks/RecoveryTests.cs @@ -0,0 +1,191 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.Sinks; + +namespace Levels.Tests.Sinks; + +public class RecoveryTests : IDisposable +{ + private readonly string _tempDir; + + public RecoveryTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + [Fact] + public async Task RecoverState_RebuildOrderbookFromSealedFile() + { + var config = new SinkConfig { OutputPath = _tempDir }; + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + + // Write some events and seal the file + await using (var writer = new PriceStreamWriter(streamId, "binance", config)) + { + await writer.WriteAsync(new RawMarketEvent( + "binance", "BTC-USD", 1000, 50000, 100, + RecordType.Snap, RecordSide.Bid), CancellationToken.None); + await writer.WriteAsync(new RawMarketEvent( + "binance", "BTC-USD", 2000, 50100, 200, + RecordType.Snap, RecordSide.Ask), CancellationToken.None); + await writer.WriteAsync(new RawMarketEvent( + "binance", "BTC-USD", 3000, 50000, 150, + RecordType.Delta, RecordSide.Bid), CancellationToken.None); + } + + // Create new writer and recover state + await using (var writer2 = new PriceStreamWriter(streamId, "binance", config)) + { + await writer2.RecoverStateAsync(); + + // Write another event — this will trigger a new file with leading SNAPs + await writer2.WriteAsync(new RawMarketEvent( + "binance", "BTC-USD", 4000, 50200, 50, + RecordType.Delta, RecordSide.Ask), CancellationToken.None); + } + + // Read back the second file + var files = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories) + .OrderBy(f => f) + .ToList(); + Assert.Equal(2, files.Count); + + using var fs = File.OpenRead(files[1]); + var reader = new BinaryRecordReader(fs); + var records = reader.ReadRecords().ToList(); + + // Should have leading SNAPs (from recovered orderbook) + the new delta + // Recovered state: bid@50000=150, ask@50100=200 + var snaps = records.Where(r => r.Core.Type == RecordType.Snap).ToList(); + Assert.Equal(2, snaps.Count); + + var bidSnap = snaps.First(r => r.Core.Side == RecordSide.Bid); + Assert.Equal(50000, bidSnap.Core.Price); + Assert.Equal(150, bidSnap.Core.Quantity); + + var askSnap = snaps.First(r => r.Core.Side == RecordSide.Ask); + Assert.Equal(50100, askSnap.Core.Price); + Assert.Equal(200, askSnap.Core.Quantity); + } + + [Fact] + public async Task RecoverState_FileSequenceContinuesFromLastFile() + { + var config = new SinkConfig { OutputPath = _tempDir }; + var streamId = PriceStreamId.FromSymbol("ETH-USD"); + + // Write and seal a file + await using (var writer = new PriceStreamWriter(streamId, "binance", config)) + { + await writer.WriteAsync(new RawMarketEvent( + "binance", "ETH-USD", 1000, 3000, 10, + RecordType.Snap, RecordSide.Bid), CancellationToken.None); + } + + // Create new writer, recover, and write + await using (var writer2 = new PriceStreamWriter(streamId, "binance", config)) + { + await writer2.RecoverStateAsync(); + + await writer2.WriteAsync(new RawMarketEvent( + "binance", "ETH-USD", 2000, 3001, 5, + RecordType.Delta, RecordSide.Ask), CancellationToken.None); + } + + var files = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories) + .OrderBy(f => f) + .ToList(); + Assert.Equal(2, files.Count); + + // Second file should have sequence > first file + var firstSeq = int.Parse(Path.GetFileNameWithoutExtension(files[0]).Split('_')[1]); + var secondSeq = int.Parse(Path.GetFileNameWithoutExtension(files[1]).Split('_')[1]); + Assert.True(secondSeq > firstSeq, $"Second file sequence ({secondSeq}) should be greater than first ({firstSeq})"); + } + + [Fact] + public async Task RecoverState_NoExistingFiles_WorksNormally() + { + var config = new SinkConfig { OutputPath = _tempDir }; + var streamId = PriceStreamId.FromSymbol("SOL-USD"); + + await using var writer = new PriceStreamWriter(streamId, "binance", config); + await writer.RecoverStateAsync(); + + await writer.WriteAsync(new RawMarketEvent( + "binance", "SOL-USD", 1000, 100, 50, + RecordType.Snap, RecordSide.Bid), CancellationToken.None); + + // Should work fine with no prior files + var files = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories); + Assert.Single(files); + } + + [Fact] + public async Task WalReplay_EmptyWriters_RecreatesWriterFromV2Entry() + { + // Enable WAL by setting FlushBufferSize > 0 + var config = new SinkConfig + { + OutputPath = _tempDir, + FlushBufferSize = 100, + }; + + // Write events through the sink (WAL enabled) + var sink = new PriceStreamSink(config); + await sink.StartAsync(CancellationToken.None); + + await sink.WriteAsync(new RawMarketEvent( + "binance", "BTC-USD", 1000, 50000, 100, + RecordType.Snap, RecordSide.Bid)); + await sink.WriteAsync(new RawMarketEvent( + "binance", "BTC-USD", 2000, 50000, 150, + RecordType.Delta, RecordSide.Bid)); + + // Graceful stop — this seals files and truncates WAL + await sink.StopAsync(CancellationToken.None); + + // Now manually write a WAL entry (simulating a crash before seal) + var walDir = Path.Combine(_tempDir, ".wal"); + var streamId = PriceStreamId.FromVenueSymbol("binance", "BTC-USD"); + + await using (var wal = Core.IO.WriteAheadLog.Open(walDir, 0)) + { + var recordBytes = new byte[config.RecordSize]; + var core = new CoreRecordLayout + { + ObservedTime = 3000, + PriceStreamId = streamId.Value, + Price = 50100, + Quantity = 200, + Type = RecordType.Delta, + Side = RecordSide.Ask, + }; + System.Runtime.InteropServices.MemoryMarshal.Write(recordBytes, in core); + await wal.AppendAsync(streamId.Value, "binance", "BTC-USD", recordBytes); + } + + // Start a fresh sink — WAL should be replayed, creating writers from scratch + var sink2 = new PriceStreamSink(config); + await sink2.StartAsync(CancellationToken.None); + + // Write one more event to ensure the writer was recreated and is functional + await sink2.WriteAsync(new RawMarketEvent( + "binance", "BTC-USD", 4000, 50200, 50, + RecordType.Delta, RecordSide.Bid)); + + await sink2.StopAsync(CancellationToken.None); + + // Verify files were created (at least the original + recovered file) + var rawFiles = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories); + Assert.True(rawFiles.Length >= 2, $"Expected at least 2 raw files, got {rawFiles.Length}"); + } +} diff --git a/tests/Levels.Tests/Sinks/RolloverTests.cs b/tests/Levels.Tests/Sinks/RolloverTests.cs new file mode 100644 index 0000000..f433174 --- /dev/null +++ b/tests/Levels.Tests/Sinks/RolloverTests.cs @@ -0,0 +1,161 @@ +using Levels.Core; +using Levels.Core.Format; +using Levels.Core.IO; +using Levels.DataFlow; +using Levels.Sinks; +using Levels.Tests.DataFlow; + +namespace Levels.Tests.Sinks; + +public class RolloverTests : IDisposable +{ + private readonly string _tempDir; + + public RolloverTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"levels_test_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, recursive: true); + } + + [Fact] + public async Task SizeTriggeredRollover_CreatesMultipleFiles() + { + // Very small rollover size to trigger quickly + var handler = new TestDataFlowHandler(); + await using var bus = new DataFlowBus([handler]); + await bus.StartAsync(CancellationToken.None); + + var config = new SinkConfig + { + OutputPath = _tempDir, + RolloverSize = 512, // Very small — header alone is 128 bytes + DataFlowBus = bus, + }; + var streamId = PriceStreamId.FromSymbol("BTC-USD"); + + await using (var writer = new PriceStreamWriter(streamId, "binance", config)) + { + for (var i = 0; i < 50; i++) + { + await writer.WriteAsync(new RawMarketEvent( + "binance", "BTC-USD", 1000 + i, 50000 + i, 100, + i == 0 ? RecordType.Snap : RecordType.Delta, + RecordSide.Bid), CancellationToken.None); + } + } + + await bus.StopAsync(CancellationToken.None); + + var files = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories); + Assert.True(files.Length > 1, $"Expected multiple files but got {files.Length}"); + + // Each file should be sealed with a footer + foreach (var file in files) + { + using var fs = File.OpenRead(file); + var reader = new BinaryRecordReader(fs); + Assert.NotNull(reader.Footer); + Assert.Equal(Constants.FooterMagicUInt64, reader.Footer.Value.MagicEnd); + } + + // DataFlowBus handler should have received sealed events for all files + Assert.Equal(files.Length, handler.FilesSealed.Count); + } + + [Fact] + public async Task TimeTriggeredRollover_CreatesNewFile() + { + var config = new SinkConfig + { + OutputPath = _tempDir, + RolloverInterval = TimeSpan.Zero, // Always rollover + }; + var streamId = PriceStreamId.FromSymbol("ETH-USD"); + + await using (var writer = new PriceStreamWriter(streamId, "binance", config)) + { + // First event opens a file + await writer.WriteAsync(new RawMarketEvent( + "binance", "ETH-USD", 1000, 3000, 10, + RecordType.Snap, RecordSide.Bid), CancellationToken.None); + + // Second event should trigger rollover (interval = zero) + await writer.WriteAsync(new RawMarketEvent( + "binance", "ETH-USD", 2000, 3001, 5, + RecordType.Delta, RecordSide.Ask), CancellationToken.None); + } + + var files = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories); + Assert.True(files.Length >= 2, $"Expected at least 2 files but got {files.Length}"); + + // Verify new files start with leading SNAP reflecting orderbook state + // Sort by name to get chronological order + var sorted = files.OrderBy(f => f).ToArray(); + + // Second (and subsequent) files should start with SNAP + for (var i = 1; i < sorted.Length; i++) + { + using var fs = File.OpenRead(sorted[i]); + var reader = new BinaryRecordReader(fs); + var records = reader.ReadRecords().ToList(); + Assert.True(records.Count > 0); + Assert.Equal(RecordType.Snap, records[0].Core.Type); + } + } + + [Fact] + public async Task Rollover_LeadingSnapMatchesOrderbookState() + { + var config = new SinkConfig + { + OutputPath = _tempDir, + RolloverSize = 512, + }; + var streamId = PriceStreamId.FromSymbol("SOL-USD"); + + await using (var writer = new PriceStreamWriter(streamId, "binance", config)) + { + // Build up orderbook state + await writer.WriteAsync(new RawMarketEvent( + "binance", "SOL-USD", 1000, 100, 50, + RecordType.Snap, RecordSide.Bid), CancellationToken.None); + await writer.WriteAsync(new RawMarketEvent( + "binance", "SOL-USD", 1000, 200, 30, + RecordType.Snap, RecordSide.Ask), CancellationToken.None); + + // Write enough deltas to trigger rollover + for (var i = 0; i < 30; i++) + { + await writer.WriteAsync(new RawMarketEvent( + "binance", "SOL-USD", 2000 + i, 100, 50 + i, + RecordType.Delta, RecordSide.Bid), CancellationToken.None); + } + } + + var files = Directory.GetFiles(_tempDir, "*.raw", SearchOption.AllDirectories) + .OrderBy(f => f).ToArray(); + + if (files.Length < 2) + return; // If no rollover happened, skip (shouldn't happen with 512 byte limit) + + // Read the second file — leading SNAPs should contain bid=100 and ask=200 + using var fs = File.OpenRead(files[1]); + var reader = new BinaryRecordReader(fs); + var records = reader.ReadRecords().ToList(); + + var snaps = records.Where(r => r.Core.Type == RecordType.Snap).ToList(); + Assert.True(snaps.Count >= 2, "Should have at least bid + ask SNAP levels"); + + var bidSnap = snaps.FirstOrDefault(r => r.Core.Side == RecordSide.Bid); + var askSnap = snaps.FirstOrDefault(r => r.Core.Side == RecordSide.Ask); + + Assert.Equal(100, bidSnap.Core.Price); + Assert.Equal(200, askSnap.Core.Price); + } +}