Upgrade to C++20, update lc0 integration, and add one-game-per-file output

.github/workflows/cmake-multi-platform.yml

@@ -0,0 +1,127 @@
+name: CMake on multiple platforms
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        build_type: [Release]
+        c_compiler: [gcc, clang, cl]
+        include:
+          - os: windows-latest
+            c_compiler: gcc
+            cpp_compiler: g++
+          - os: ubuntu-latest
+            c_compiler: gcc
+            cpp_compiler: g++
+          - os: ubuntu-latest
+            c_compiler: clang
+            cpp_compiler: clang++
+        exclude:
+          - os: windows-latest
+            c_compiler: cl
+          - os: windows-latest
+            c_compiler: clang
+          - os: ubuntu-latest
+            c_compiler: cl
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+
+    - name: Update submodules
+      run: |
+        git submodule sync --recursive
+        git submodule update --init --recursive
+
+    - name: Set reusable strings
+      id: strings
+      shell: bash
+      run: |
+        echo "build-output-dir=${{ github.workspace }}/build" >> "$GITHUB_OUTPUT"
+
+    - name: Install dependencies (Linux)
+      if: runner.os == 'Linux'
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y cmake zlib1g-dev ninja-build
+
+    - name: Install dependencies (Windows)
+      if: runner.os == 'Windows'
+      run: |
+        choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' -y
+        choco install ninja -y
+
+    - name: Configure CMake
+      run: >
+        cmake -B ${{ steps.strings.outputs.build-output-dir }}
+        -G Ninja
+        -DCMAKE_CXX_COMPILER=${{ matrix.cpp_compiler }}
+        -DCMAKE_C_COMPILER=${{ matrix.c_compiler }}
+        -DCMAKE_BUILD_TYPE=${{ matrix.build_type }}
+        -S ${{ github.workspace }}
+
+    - name: Build
+      run: cmake --build ${{ steps.strings.outputs.build-output-dir }} --config ${{ matrix.build_type }} --verbose
+
+    - name: Test
+      working-directory: ${{ steps.strings.outputs.build-output-dir }}
+      run: ctest --build-config ${{ matrix.build_type }}
+
+    - name: Upload Build Artifact
+      uses: actions/upload-artifact@v4.4.3
+      with:
+        name: trainingdata-tool-${{ matrix.os }}-${{ matrix.c_compiler }}
+        path: |
+          ${{ steps.strings.outputs.build-output-dir }}/trainingdata-tool
+          ${{ steps.strings.outputs.build-output-dir }}/trainingdata-tool.exe
+        if-no-files-found: ignore
+
+  release:
+    needs: build
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+    permissions:
+      contents: write
+
+    steps:
+    - name: Download all artifacts
+      uses: actions/download-artifact@v4
+      with:
+        path: artifacts
+
+    - name: Create timestamp
+      id: timestamp
+      run: echo "time=$(date +'%Y%m%d-%H%M%S')" >> "$GITHUB_OUTPUT"
+
+    - name: Prepare release assets
+      shell: bash
+      run: |
+        mkdir -p release-assets
+        for file in artifacts/*/*; do
+          [ -f "$file" ] || continue
+          artifact_name="$(basename "$(dirname "$file")")"
+          basename_file="$(basename "$file")"
+          cp "$file" "release-assets/${artifact_name}-${basename_file}"
+        done
+
+    - name: Create Pre-release
+      uses: softprops/action-gh-release@v1
+      with:
+        tag_name: pre-release-${{ steps.timestamp.outputs.time }}
+        name: Pre-release ${{ steps.timestamp.outputs.time }}
+        prerelease: true
+        files: release-assets/*
+        generate_release_notes: true
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -50,3 +50,9 @@ Makefile
 trainingdata-tool
 
 Project\.cbp
+
+__pycache__/
+*.pyc
+
+# Analysis output, not source
+graphify-out/

.gitmodules

@@ -6,5 +6,5 @@
 	url = https://github.com/DanielUranga/polyglot.git
 [submodule "lc0"]
 	path = lc0
-	url = https://github.com/CallOn84/lc0.git
-	branch = trainingdata-tool
+	url = https://github.com/LeelaChessZero/lc0.git
+	branch = master

CMakeLists.txt

@@ -2,21 +2,22 @@
 # project specific logic here.
 #
 cmake_minimum_required (VERSION 3.8)
+project(trainingdata-tool)
 
-set(CMAKE_REQUIRED_FLAGS -std=c++17)
+set(CMAKE_REQUIRED_FLAGS -std=c++20)
 
 file(GLOB_RECURSE sources src/*.cpp src/*.h)
 
 set (
     lc0
-    "lc0/src/chess/bitboard.cc"
     "lc0/src/chess/board.cc"
     "lc0/src/chess/position.cc"
     "lc0/src/neural/encoder.cc"
-    "lc0/src/neural/writer.cc"
+    "lc0/src/trainingdata/writer.cc"
     "lc0/src/utils/commandline.cc"
     "lc0/src/utils/logging.cc"
     "lc0/src/utils/random.cc"
+    "lc0/src/utils/string.cc"
 )
 
 if (WIN32)
@@ -26,32 +27,81 @@ else (WIN32)
 endif (WIN32)
 
 AUX_SOURCE_DIRECTORY(polyglot/src polyglot)
-AUX_SOURCE_DIRECTORY(zlib zlib)
+
+# Use system zlib on Unix to avoid old bundled zlib issues with modern compilers
+if (UNIX)
+    find_package(ZLIB REQUIRED)
+    set(ZLIB_LIBS ZLIB::ZLIB)
+    set(ZLIB_INCLUDE ${ZLIB_INCLUDE_DIRS})
+else()
+    AUX_SOURCE_DIRECTORY(zlib zlib_sources)
+    set(ZLIB_LIBS "")
+    set(ZLIB_INCLUDE "zlib")
+endif()
 
 # Add source to this project's executable.
-add_executable(trainingdata-tool ${sources} ${lc0} ${lc0_filesystem} ${polyglot} ${zlib})
+if (UNIX)
+    add_executable(trainingdata-tool ${sources} ${lc0} ${lc0_filesystem} ${polyglot})
+else()
+    add_executable(trainingdata-tool ${sources} ${lc0} ${lc0_filesystem} ${polyglot} ${zlib_sources})
+endif()
 
 set_target_properties(trainingdata-tool PROPERTIES
-    CXX_STANDARD 17
+    CXX_STANDARD 20
     CXX_STANDARD_REQUIRED ON
     CXX_EXTENSIONS ON
 )
 
 if (UNIX)
-    target_link_libraries(trainingdata-tool -lpthread -lstdc++fs)
+    target_link_libraries(trainingdata-tool -lpthread -lstdc++fs ${ZLIB_LIBS})
 endif(UNIX)
 
-find_package(Boost 1.65.0)
+set(CMAKE_BUILD_TYPE Release)
 
 include_directories(
     "lc0/src"
-    "lc0/src/chess"
-    "lc0/src/neural"
+    "src"
     "polyglot/src"
-    "zlib"
-    ${Boost_INCLUDE_DIRS}
+    ${ZLIB_INCLUDE}
+    "."
 )
 
+# For MinGW: pre-define __GETOPT_H__ to prevent polyglot's getopt.h from
+# being used when system unistd.h includes <getopt.h>. This avoids the
+# "undefined reference to getopt_internal" error.
+if (MINGW)
+    add_compile_definitions(__GETOPT_H__)
+endif()
+
 add_compile_definitions(NO_PEXT)
 
+# MSVC-specific settings for compatibility with legacy C code in polyglot and lc0 quirks
+# MSVC-specific settings for compatibility with legacy C code in polyglot and lc0 quirks
+if (MSVC)
+    # Disable deprecation warnings for unsafe CRT functions and other noisy warnings
+    # 4996: unsafe function (strcpy vs strcpy_s)
+    # 4267, 4244: conversion loss of data
+    # 4390: empty controlled statement
+    # 4018: signed/unsigned mismatch
+    add_compile_definitions(_CRT_SECURE_NO_WARNINGS)
+    add_compile_options(/wd4996 /wd4267 /wd4244 /wd4390 /wd4018)
+
+    # Relax conformance mode to allow some legacy C++ constructs
+    add_compile_options(/permissive)
+
+    # Force include <array> because lc0/src/neural/encoder.cc uses std::array but doesn't include it
+    add_compile_options(/FIarray)
+
+    # Specific flags for polyglot files to handle const char* conversions
+    # We try to force them to use older C++ standard semantics where possible
+    set_source_files_properties(${polyglot} PROPERTIES 
+        COMPILE_OPTIONS "/Zc:strictStrings-;/permissive"
+    )
+elseif (MINGW)
+    # MinGW/GCC also needs lenient handling for legacy C code
+    set_source_files_properties(${polyglot} PROPERTIES 
+        COMPILE_OPTIONS "-fpermissive;-Wno-write-strings"
+    )
+endif()
+
 # TODO: Add tests and install targets if needed.

PULL_REQUEST.md

@@ -0,0 +1,40 @@
+# PR Title
+Upgrade to C++20, update lc0 integration, and add one-game-per-file output
+
+# PR Description
+
+## Summary
+This PR modernizes the trainingdata-tool by upgrading to C++20, updating the lc0 integration to work with the latest lc0 codebase, and improving the output format to write one game per chunk file.
+
+## Changes
+
+### Build System Updates
+- Upgraded C++ standard from C++17 to C++20
+- Set explicit Release build type
+- Simplified include directories
+- Added `lc0/src/utils/string.cc` to fix linker error for `StrSplit` function
+
+### lc0 Integration Updates
+- Updated lc0 source file paths to match latest lc0 structure:
+  - Removed `lc0/src/chess/bitboard.cc` (no longer needed)
+  - Changed `lc0/src/neural/writer.cc` to `lc0/src/trainingdata/writer.cc`
+- Updated `.gitmodules` for lc0 submodule
+- Added `absl/` library dependency
+
+### Training Data Format Upgrade
+- Upgraded from V4 to V6 training data format
+- Replaced `V4TrainingDataHashUtil.h` with `V6TrainingDataHashUtil.h`
+- Updated related source files for V6 compatibility
+
+### Output Format Improvement
+- Modified `TrainingDataWriter` to write one game per chunk file
+- Each game's training positions are now isolated in their own `.gz` file
+- Removed batching logic that previously combined multiple games into one file
+
+## Testing
+- Successfully built on Linux with GCC
+- Verified output: Converting 5 games produces 5 separate files with varying sizes reflecting different game lengths
+
+---
+
+*Made with Gemini and Claude*