From e045db10e618c5ca97f1ad7794ecb457a71836a5 Mon Sep 17 00:00:00 2001 From: Claude Opus 5 Date: Wed, 2 Sep 2026 04:24:17 +0000 Subject: [PATCH 1/4] Rebuild the scanner around an explainable verdict, and stop calling unchecked files safe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v0.2.0 reported `Safe` for files it had not managed to inspect: an oversized file, a file it could not `stat`, and a `zipfile.BadZipFile` were all swallowed and reported as clean. Eleven of twenty-two structurally risky corpus samples came back Safe, including ZIP path traversal, a 48 KB → 48 MB bomb, an encrypted archive, and a PDF with a ZIP appended after %%EOF. For a tool a teacher trusts with a folder of student submissions, "I did not look" reported as "this is fine" is the worst failure it can have. The scoring was equally indefensible: an unsourced weight table that double- and triple-counted a single fact (one macro produced four findings and saturated at 100) and escalated a twenty-link bibliography to High. What replaces it: - scanner/findings.py, verdict.py — severity and confidence as separate axes, and a verdict from four stated rules. COULD_NOT_INSPECT is a first-class outcome that can never round down to safe. The risk score is capped per finding code and used only for sorting. - scanner/detectors/ — archive, office, pdf, image and general detectors replacing heuristics.py and the four *_rules modules. - scanner/limits.py — explicit bounds on size, archive members, nesting depth, total uncompressed bytes and compression ratio. - scanner/quarantine.py — a JSONL manifest, 0600/0700, hash-verified restore, and refusal to follow symlinks. - scanner/reporters.py — a self-contained HTML report with no script and no network, and sanitize_display() to neutralise the bidi and zero-width characters an attacker uses to make `invoicegpj.exe` read as a JPEG. The report was rendering the attack as its own disguise. - scanner/gui_model.py + gui.py — the Tk-free model is tested; PySimpleGUI is gone. Defects found and fixed while building this, each with a regression test: every valid PNG was flagged (the IEND CRC was not counted); every real .docx was flagged (a settings.xml rule); /EmbeddedFile was missed when it straddled the 4096-byte read boundary; a symlink to /dev/zero never finished; quarantining two files of the same name destroyed the first; a mistyped scan path exited 0; and .rar/.7z/.tar/.rtf returned LIKELY_SAFE rather than COULD_NOT_INSPECT. Deleting ruff.toml activated the stricter pyproject config, which surfaced a real B023 loop-variable closure in office.py and a B904 in quarantine.py. Verified on Linux/CPython 3.11: pytest 117 passed 1 skipped; ruff clean over scanner tests examples scripts; mypy clean over 19 source files; corpus gate 31 samples, 15 correctly blocked, 9 correctly clean, no false positives. Not verified: the PyInstaller binary, Windows, macOS, the Tk window, and the optional YARA path. See docs/REVIVAL_AUDIT.md. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CKVgvkabikjdEvThAqweov --- .github/workflows/ci.yml | 89 +- .github/workflows/release.yml | 128 +- BEGINNERS_GUIDE.md | 130 +- CHANGELOG.md | 130 +- README.md | 432 ++++--- SAFETY.md | 83 +- docs/ARCHITECTURE.md | 91 ++ docs/NEXT_20_COMMITS.md | 1149 +++++++++++++++++ docs/README.md | 3 - docs/REVIVAL_AUDIT.md | 556 ++++++++ docs/REVIVAL_CHANGELOG.md | 525 ++++++++ docs/SCORING.md | 127 ++ examples/benign_samples/Assignment.pdf.exe | 1 + examples/benign_samples/README.md | 37 + .../archive_double_extension.zip | Bin 0 -> 199 bytes .../benign_samples/archive_nested_deep.zip | Bin 0 -> 255 bytes .../archive_password_protected.zip | Bin 0 -> 197 bytes .../benign_samples/archive_path_traversal.zip | Bin 0 -> 293 bytes .../benign_samples/archive_with_program.zip | Bin 0 -> 290 bytes .../benign_samples/archive_zip_bomb_shape.zip | Bin 0 -> 49044 bytes examples/benign_samples/broken_upload.zip | 1 + examples/benign_samples/clean_diagram.png | Bin 0 -> 69 bytes examples/benign_samples/clean_essay.txt | 2 + examples/benign_samples/clean_homework.zip | Bin 0 -> 239 bytes examples/benign_samples/clean_photo.jpg | Bin 0 -> 91 bytes examples/benign_samples/clean_report.docx | Bin 0 -> 1185 bytes examples/benign_samples/clean_worksheet.pdf | Bin 0 -> 226 bytes examples/benign_samples/coursework.7z | 4 + examples/benign_samples/empty_submission.docx | 0 examples/benign_samples/essay.rtf | 1 + .../image_is_really_a_program.jpg | Bin 0 -> 583 bytes .../benign_samples/image_large_appended.jpg | Bin 0 -> 614491 bytes examples/benign_samples/image_polyglot.png | Bin 0 -> 248 bytes .../invoice\342\200\256gpj.exe" | 1 + examples/benign_samples/links_suspicious.txt | 4 + examples/benign_samples/office_dde_field.docx | Bin 0 -> 1215 bytes .../office_embedded_object.docx | Bin 0 -> 1449 bytes .../office_remote_template.docx | Bin 0 -> 1625 bytes .../office_renamed_program.docx | Bin 0 -> 327 bytes .../benign_samples/office_with_macro.docm | Bin 0 -> 1811 bytes .../benign_samples/pdf_appended_payload.pdf | Bin 0 -> 4326 bytes examples/benign_samples/pdf_javascript.pdf | Bin 0 -> 281 bytes examples/benign_samples/pdf_launch_action.pdf | Bin 0 -> 269 bytes examples/benign_samples/sample_text.txt | 1 - examples/generate_benign_samples.py | 368 ++++-- examples/sample_report.json | 924 ++++++++++++- mypy.ini | 5 - pyproject.toml | 39 +- requirements-dev.txt | 5 + requirements-optional.txt | 3 - requirements.txt | 17 +- ruff.toml | 2 - scanner/__init__.py | 10 +- scanner/detectors/__init__.py | 224 +--- scanner/detectors/archive.py | 500 +++++++ scanner/detectors/base.py | 250 ++++ scanner/detectors/general.py | 344 +++++ scanner/detectors/image.py | 303 +++++ scanner/detectors/image_rules.py | 33 - scanner/detectors/office.py | 476 +++++++ scanner/detectors/office_rules.py | 32 - scanner/detectors/pdf.py | 293 +++++ scanner/detectors/pdf_rules.py | 40 - scanner/detectors/zip_rules.py | 54 - scanner/findings.py | 216 ++++ scanner/gui.py | 460 +++++-- scanner/gui_model.py | 221 ++++ scanner/heuristics.py | 55 - scanner/limits.py | 63 + scanner/main.py | 556 +++++--- scanner/quarantine.py | 332 ++++- scanner/reporters.py | 595 ++++++--- scanner/reporting/html_theme.css | 6 - scanner/scanner_core.py | 639 ++++++--- scanner/triage.py | 203 +++ scanner/utils.py | 141 -- scanner/verdict.py | 159 +++ scripts/build_binary.py | 64 + scripts/build_pyinstaller.ps1 | 8 - scripts/build_pyinstaller.sh | 10 - scripts/check_corpus.py | 111 ++ setup.cfg | 8 - teacher-safe-scan.spec | 78 ++ tests/conftest.py | 36 +- tests/samples.py | 108 ++ tests/test_detectors.py | 36 - tests/test_detectors_archive.py | 119 ++ tests/test_detectors_documents.py | 186 +++ tests/test_heuristics.py | 20 - tests/test_image_rules.py | 13 - tests/test_integration.py | 12 - tests/test_office_rules.py | 13 - tests/test_pdf_rules.py | 14 - tests/test_quarantine.py | 160 +++ tests/test_reporting_and_cli.py | 216 ++++ tests/test_scanner_core.py | 183 +++ tests/test_verdict.py | 99 ++ tests/test_zip_rules.py | 14 - tests/utils_make_samples.py | 35 - 99 files changed, 10676 insertions(+), 1930 deletions(-) create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/NEXT_20_COMMITS.md delete mode 100644 docs/README.md create mode 100644 docs/REVIVAL_AUDIT.md create mode 100644 docs/REVIVAL_CHANGELOG.md create mode 100644 docs/SCORING.md create mode 100644 examples/benign_samples/Assignment.pdf.exe create mode 100644 examples/benign_samples/README.md create mode 100644 examples/benign_samples/archive_double_extension.zip create mode 100644 examples/benign_samples/archive_nested_deep.zip create mode 100644 examples/benign_samples/archive_password_protected.zip create mode 100644 examples/benign_samples/archive_path_traversal.zip create mode 100644 examples/benign_samples/archive_with_program.zip create mode 100644 examples/benign_samples/archive_zip_bomb_shape.zip create mode 100644 examples/benign_samples/broken_upload.zip create mode 100644 examples/benign_samples/clean_diagram.png create mode 100644 examples/benign_samples/clean_essay.txt create mode 100644 examples/benign_samples/clean_homework.zip create mode 100644 examples/benign_samples/clean_photo.jpg create mode 100644 examples/benign_samples/clean_report.docx create mode 100644 examples/benign_samples/clean_worksheet.pdf create mode 100644 examples/benign_samples/coursework.7z create mode 100644 examples/benign_samples/empty_submission.docx create mode 100644 examples/benign_samples/essay.rtf create mode 100644 examples/benign_samples/image_is_really_a_program.jpg create mode 100644 examples/benign_samples/image_large_appended.jpg create mode 100644 examples/benign_samples/image_polyglot.png create mode 100644 "examples/benign_samples/invoice\342\200\256gpj.exe" create mode 100644 examples/benign_samples/links_suspicious.txt create mode 100644 examples/benign_samples/office_dde_field.docx create mode 100644 examples/benign_samples/office_embedded_object.docx create mode 100644 examples/benign_samples/office_remote_template.docx create mode 100644 examples/benign_samples/office_renamed_program.docx create mode 100644 examples/benign_samples/office_with_macro.docm create mode 100644 examples/benign_samples/pdf_appended_payload.pdf create mode 100644 examples/benign_samples/pdf_javascript.pdf create mode 100644 examples/benign_samples/pdf_launch_action.pdf delete mode 100644 examples/benign_samples/sample_text.txt delete mode 100644 mypy.ini create mode 100644 requirements-dev.txt delete mode 100644 requirements-optional.txt delete mode 100644 ruff.toml create mode 100644 scanner/detectors/archive.py create mode 100644 scanner/detectors/base.py create mode 100644 scanner/detectors/general.py create mode 100644 scanner/detectors/image.py delete mode 100644 scanner/detectors/image_rules.py create mode 100644 scanner/detectors/office.py delete mode 100644 scanner/detectors/office_rules.py create mode 100644 scanner/detectors/pdf.py delete mode 100644 scanner/detectors/pdf_rules.py delete mode 100644 scanner/detectors/zip_rules.py create mode 100644 scanner/findings.py create mode 100644 scanner/gui_model.py delete mode 100644 scanner/heuristics.py create mode 100644 scanner/limits.py delete mode 100644 scanner/reporting/html_theme.css create mode 100644 scanner/triage.py delete mode 100644 scanner/utils.py create mode 100644 scanner/verdict.py create mode 100755 scripts/build_binary.py delete mode 100644 scripts/build_pyinstaller.ps1 delete mode 100755 scripts/build_pyinstaller.sh create mode 100755 scripts/check_corpus.py delete mode 100644 setup.cfg create mode 100644 teacher-safe-scan.spec create mode 100644 tests/samples.py delete mode 100644 tests/test_detectors.py create mode 100644 tests/test_detectors_archive.py create mode 100644 tests/test_detectors_documents.py delete mode 100644 tests/test_heuristics.py delete mode 100644 tests/test_image_rules.py delete mode 100644 tests/test_integration.py delete mode 100644 tests/test_office_rules.py delete mode 100644 tests/test_pdf_rules.py create mode 100644 tests/test_quarantine.py create mode 100644 tests/test_reporting_and_cli.py create mode 100644 tests/test_scanner_core.py create mode 100644 tests/test_verdict.py delete mode 100644 tests/test_zip_rules.py delete mode 100644 tests/utils_make_samples.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3eb25bd..fd24570 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,28 +2,91 @@ name: CI on: push: - branches: ["main", "master"] + branches: [main] pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - build: - runs-on: ubuntu-latest + test: + name: test (${{ matrix.os }}, py${{ matrix.python }}) + runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: - python-version: ["3.10", "3.11"] + os: [ubuntu-latest, macos-latest, windows-latest] + python: ["3.10", "3.12"] steps: - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies + python-version: ${{ matrix.python }} + cache: pip + + - name: Install run: | python -m pip install --upgrade pip - pip install -r requirements.txt + pip install -e ".[dev]" + - name: Lint - run: ruff check . + run: ruff check scanner tests examples + - name: Type check - run: mypy . - - name: Run tests - run: pytest + run: mypy scanner + + - name: Test + run: pytest -q + + # The scanner core must work with nothing but the standard library. This + # job would have caught the original release, where `pip install -r + # requirements.txt` failed outright because PySimpleGUI had been pulled + # from PyPI. + - name: Verify the scanner runs with zero dependencies + shell: bash + run: | + python -m venv /tmp/bare + if [ -f /tmp/bare/bin/python ]; then PY=/tmp/bare/bin/python; else PY=/tmp/bare/Scripts/python; fi + $PY examples/generate_benign_samples.py --out /tmp/samples + set +e + $PY -m scanner --color never scan /tmp/samples --report-json /tmp/r.json + code=$? + set -e + test "$code" = "2" || { echo "expected exit code 2, got $code"; exit 1; } + $PY -c "import json;d=json.load(open('/tmp/r.json'));assert d['counts']['do_not_open']>10, d['counts']" + + detection-corpus: + name: detection corpus must not regress + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install -e ".[dev]" + - name: Every benign sample must land in its expected verdict + run: python scripts/check_corpus.py + + build: + name: standalone binary (${{ matrix.os }}) + needs: test + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install -e ".[dev,build]" + - run: python scripts/build_binary.py + - uses: actions/upload-artifact@v4 + with: + name: teacher-safe-scan-${{ matrix.os }} + path: | + dist/teacher-safe-scan* + if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 30d3ca0..03e3ca1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,61 +1,113 @@ -name: build-and-release +name: release + on: push: tags: ["v*.*.*"] + workflow_dispatch: + +permissions: + contents: write jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: { python-version: "3.12" } + - run: pip install -e ".[dev]" + - run: ruff check scanner tests examples + - run: mypy scanner + - run: pytest -q + - run: python scripts/check_corpus.py + build: + needs: verify strategy: + fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + include: + - os: ubuntu-latest + asset: teacher-safe-scan-linux-x86_64 + - os: macos-latest + asset: teacher-safe-scan-macos-arm64 + - os: windows-latest + asset: teacher-safe-scan-windows-x86_64.exe runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - name: Install deps - run: | - python -m pip install -U pip - pip install -r requirements.txt - pip install -r requirements-optional.txt || true - pip install pyinstaller PySimpleGUI - pip install ruff mypy pytest - - name: Lint & Typecheck - run: | - ruff check . - mypy . - - name: Test - run: pytest -q - - name: Build PyInstaller + with: { python-version: "3.12" } + - run: pip install -e ".[dev,build]" + + - name: Build + run: python scripts/build_binary.py + + - name: Name the artifact and checksum it shell: bash run: | - if [[ "$RUNNER_OS" == "Windows" ]]; then - python -m PyInstaller --noconfirm --clean --name TeacherSafeScanner --onefile --windowed --add-data "scanner/reporting/html_theme.css;scanner/reporting" scanner/gui.py - 7z a TeacherSafeScanner-windows.zip dist/TeacherSafeScanner.exe - elif [[ "$RUNNER_OS" == "macOS" ]]; then - python -m PyInstaller --noconfirm --clean --name TeacherSafeScanner --onefile --windowed --add-data "scanner/reporting/html_theme.css:scanner/reporting" scanner/gui.py - ditto -c -k --sequesterRsrc --keepParent dist/TeacherSafeScanner dist/TeacherSafeScanner-macos.zip || \ - (cd dist && zip -r ../TeacherSafeScanner-macos.zip TeacherSafeScanner) + mkdir -p out + src=dist/teacher-safe-scan + [ -f "$src.exe" ] && src="$src.exe" + cp "$src" "out/${{ matrix.asset }}" + cd out + if command -v sha256sum >/dev/null; then + sha256sum "${{ matrix.asset }}" > "${{ matrix.asset }}.sha256" else - python -m PyInstaller --noconfirm --clean --name TeacherSafeScanner --onefile --windowed --add-data "scanner/reporting/html_theme.css:scanner/reporting" scanner/gui.py - (cd dist && tar -czf ../TeacherSafeScanner-linux.tar.gz TeacherSafeScanner) + shasum -a 256 "${{ matrix.asset }}" > "${{ matrix.asset }}.sha256" fi - - name: Upload artifacts - uses: actions/upload-artifact@v4 + cat "${{ matrix.asset }}.sha256" + + - name: Smoke-test the built binary + shell: bash + run: | + bin="out/${{ matrix.asset }}" + chmod +x "$bin" || true + "$bin" --version + "$bin" make-samples --out ./smoke-samples + set +e; "$bin" --color never scan ./smoke-samples; code=$?; set -e + test "$code" = "2" || { echo "expected exit 2 from the corpus, got $code"; exit 1; } + + - uses: actions/upload-artifact@v4 with: - name: TeacherSafeScanner-${{ matrix.os }} - path: | - TeacherSafeScanner-*.zip - TeacherSafeScanner-*.tar.gz - release: + name: ${{ matrix.asset }} + path: out/* + + publish: needs: build runs-on: ubuntu-latest steps: + - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 - with: - path: ./artifacts - - name: Create Release + with: { path: artifacts, merge-multiple: true } + - run: ls -la artifacts + + - name: Publish release uses: softprops/action-gh-release@v2 with: - files: artifacts/**/* + files: artifacts/* + generate_release_notes: true + body: | + ## Verify your download + + ``` + sha256sum -c teacher-safe-scan-.sha256 + ``` + + ## These binaries are NOT code-signed + + They are built by GitHub Actions from the tagged commit, and the + build log is public. They are **not** signed with an Apple Developer + ID and **not** notarised, so: + + - **macOS** will refuse to run the binary until you clear it: + `xattr -d com.apple.quarantine ./teacher-safe-scan-macos-arm64` + - **Windows** SmartScreen will warn on first run. + + If that is not acceptable in your environment, install from source + instead — the scanner core needs nothing but the Python standard + library: + + ``` + pip install teacher-safe-local-file-scanner + ``` diff --git a/BEGINNERS_GUIDE.md b/BEGINNERS_GUIDE.md index f560dda..c4dc209 100644 --- a/BEGINNERS_GUIDE.md +++ b/BEGINNERS_GUIDE.md @@ -31,98 +31,96 @@ This guide is for people who are new to computers and want a safe way to check s If the terminal says "The system cannot find the path specified" or "No such file or directory", double-check the folder location and try again. -## 4. Create a safe Python environment +## 4. Install what you need — probably nothing -Copy and paste these commands into the terminal, one line at a time. Press Enter after each line. +The scanner needs **no extra software**. It uses only what comes with Python. -```bash -python -m venv .venv -``` +If you want the desktop window as well, you may need one extra package: -- On **Windows** run: - ```cmd - .venv\Scripts\activate - ``` -- On **macOS/Linux** run: - ```bash - source .venv/bin/activate - ``` +- **Windows and macOS:** nothing to do. It is already included. +- **Debian, Ubuntu, Mint:** `sudo apt install python3-tk` +- **Fedora:** `sudo dnf install python3-tkinter` -When the environment is active you will see `(.venv)` at the beginning of the terminal line. +## 5. Try it on the practice files first -## 5. Install the scanner +The project ships with a folder of harmless practice files. They are completely +safe — they are ordinary text and pictures built to *look* like risky files, so +you can see what the scanner does before you use it on real work. -```bash -pip install -r requirements.txt +``` +python examples/generate_benign_samples.py +python -m scanner scan examples/benign_samples ``` -Wait until the installation finishes. If you see an error, ensure your internet connection is working and run the command again. +You should see a list where most files are marked **DO NOT OPEN** and a few are +marked **LIKELY SAFE**. That is correct: most of the practice files are supposed +to be caught. -## 6. Create the example files +## 6. Check some real submissions -The repository avoids storing binary files, so you need to create the harmless samples locally. Run: +Put the files you want to check into one folder. Then: -```bash -python examples/generate_benign_samples.py +``` +python -m scanner scan "C:\Users\you\Downloads\period-3" --report-html report.html ``` -This command creates three safe files inside `examples/benign_samples/`: - -- `sample_text.txt` – a normal text file. -- `sample_image.png` – a tiny picture. -- `sample_docx.docx` – a Word document with no macros. - -## 7. Run your first scan +On macOS or Linux: -```bash -python -m scanner scan examples/benign_samples -python -m scanner.main scan examples/benign_samples +``` +python -m scanner scan ~/Downloads/period-3 --report-html report.html ``` -- If everything is safe, the program finishes with exit code `0` and prints a summary. -- If you ever see exit code `1`, `2`, or `3`, read the message shown on screen and follow the safety tips below. +Open `report.html` by double-clicking it. It opens in your web browser. It is a +single file — you can email it to your IT team. -## 8. What to do if something is flagged +## 7. Reading the result -1. **Do not open the file.** -2. Move it away from your main folders using: - ```bash - python -m scanner quarantine PATH_TO_FILE --dest quarantine - python -m scanner.main quarantine PATH_TO_FILE --dest quarantine - ``` -3. Share the JSON or HTML report with your school IT team. +| What you see | What it means | What to do | +| --- | --- | --- | +| ✓ **LIKELY SAFE TO REVIEW** | Nothing was found | Open it normally | +| ! **REVIEW WITH CAUTION** | Something is worth a look | Read the reason before opening | +| ✖ **DO NOT OPEN — CONTACT IT** | Something clearly risky was found | Do not open it. Send the report to IT | +| ? **COULD NOT FULLY INSPECT** | The scanner could not see inside | Treat it as unchecked, **not** as safe | -## 9. Keep things up to date +Under each file, the report explains in plain words what was found, why it +matters, and what to do. You can forward those sentences to a colleague or a +parent as they are. -- To update the scanner later, open the project folder, activate the virtual environment again, and run: - ```bash - git pull - pip install -r requirements.txt - ``` -- Run the generator script again if you need fresh example files. +## 8. If you want to open a window instead of typing commands + +``` +python -m scanner gui +``` -## 10. Extra help +Choose a folder, press **Scan**, and click any row to see the details. The +**Copy summary for IT** button puts a plain-text summary on your clipboard. -- Read [README.md](README.md) for advanced features. -- Read [SAFETY.md](SAFETY.md) for more safety advice. -- If you are stuck, ask a colleague or your IT support team for help. Share any error messages exactly as they appear. +## 9. Moving risky files out of the way -## Windows (PowerShell) +If you want the flagged files moved somewhere they cannot be opened by accident: -```powershell -py -3 -m venv .venv -. .venv\Scripts\Activate.ps1 -pip install -r requirements.txt -python -m scanner.gui +``` +python -m scanner scan ~/Downloads/period-3 --quarantine-dir ~/Documents/quarantine ``` -## macOS +**Nothing is ever deleted.** The files are moved, renamed so a double-click does +nothing, and written down in a list. To get one back: -```bash -python3 -m venv .venv -source .venv/bin/activate -pip install -r requirements.txt -python -m scanner.gui ``` +python -m scanner quarantine-list --dest ~/Documents/quarantine +python -m scanner restore --dest ~/Documents/quarantine +``` + +## 10. Important things to remember + +- This is **not** antivirus. Keep using whatever your school already installs. +- "Likely safe" means *nothing was found*, not *this is definitely fine*. +- The scanner never opens or runs the files it checks, and nothing is ever sent + over the internet. Student work stays on your computer. +- If a file is marked **DO NOT OPEN**, send the report to your IT team — not the + file itself. + +## Getting help -Stay safe and never execute files that you do not fully trust. +If something does not work, open an issue on the project page and paste what you +typed and what you saw. Do not attach the student file. diff --git a/CHANGELOG.md b/CHANGELOG.md index 18d3f70..a863884 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,29 +1,117 @@ # Changelog -All notable changes to this project will be documented here. +All notable changes to this project are documented here. +Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); +versions follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.3.0] — 2026-09-02 + +A revival release. The detection engine, severity model, reporting and +quarantine were rewritten; the dependency that made the project uninstallable +was removed. + +### Added +- **Folder-level triage.** The unit of work is now a folder of submissions, not + a single file. Output is a worst-first table with a plain-English headline + ("15 of 29 files should not be opened"), counts per verdict, duplicate + detection by hash, and a batch-wide finding roll-up. +- **Four-verdict model** — `LIKELY SAFE TO REVIEW`, `REVIEW WITH CAUTION`, + `DO NOT OPEN — CONTACT IT`, `COULD NOT FULLY INSPECT` — with the last of these + explicitly never treated as safe. +- **Explainable findings.** Every finding now carries plain-English text, why it + matters, a recommended action, concrete evidence, and separate severity and + confidence. See `docs/SCORING.md`. +- **New detections**: archive path traversal; decompression-bomb ratio and total; + encrypted archive entries; NUL, zero-width and right-to-left-override characters + in filenames and archive members; Office remote-template injection + (`attachedTemplate` external relationship); DDE/DDEAUTO fields; ActiveX + controls; legacy OLE macro storage; PDF `/Launch`; PDF header not at offset 0; + image polyglots (ZIP/PE/PDF after the terminator); impossible PNG chunk + lengths; oversized image metadata; extension-vs-content mismatch by magic bytes; + punycode, raw-IP and shortened links in text. +- **Quarantine restore.** `quarantine-list` and `restore` commands, an + append-only JSONL manifest recording original path and SHA-256, and hash + verification on the way back out. +- **`scanner/limits.py`** — every resource ceiling the scanner obeys, in one + reviewable struct. +- **Benign test corpus** (`examples/generate_benign_samples.py`): 28 harmless + files reproducing the structure of risky ones, plus + `scripts/check_corpus.py`, a detection-regression gate asserting both that + every hostile-shaped sample is caught and that every clean sample stays clean. +- **PyInstaller spec and `scripts/build_binary.py`**, producing a single + executable with a SHA-256 checksum, plus a release workflow that smoke-tests + the built binary before publishing. +- Cross-platform CI matrix (Linux/macOS/Windows × Python 3.10/3.12) including a + job that runs the scanner in a bare virtualenv with **zero** dependencies. +- `docs/SCORING.md` and `docs/ARCHITECTURE.md`. + +### Changed +- **The GUI is now Tkinter instead of PySimpleGUI.** PySimpleGUI moved to a paid + licence and its old versions were pulled from PyPI, which is why + `pip install -r requirements.txt` failed outright on the previous release. + Tkinter ships with CPython and freezes cleanly. Display logic moved to + `scanner/gui_model.py`, which has no Tk import and is unit tested. +- **The runtime now requires nothing outside the standard library.** All previous + requirements are optional extras. +- HTML report rewritten: self-contained, no JavaScript, no network requests, + light/dark aware, printable, grouped worst-first. +- Watch mode re-scans only files whose mtime or size changed, instead of + `rglob('*')` over the whole tree every ten seconds. +- Detector dispatch is by sniffed content type first and extension second, so a + program renamed `holiday_photo.png` is analysed as a program. -## [0.2.0] - 2026-05-29 ### Fixed -- `pyproject.toml` had a duplicate `[project]` table that made the file invalid TOML and prevented install/build. Consolidated into a single canonical block. -- `scanner/__init__.py` placed `__all__` before `from __future__ import annotations`, causing a `SyntaxError` at package import. -- `scanner/utils.py` and `scanner/scanner_core.py` each had a doubled `try: import .../except ImportError:` block left over from a botched merge, exercising dead code paths. -- `scanner/main.py` had stacked duplicate definitions of `parse_args`, `watch_loop`, `emit_results`, `handle_scan`, and `main`, plus a stray `)` that produced `SyntaxError: unmatched ')'`. Rewritten cleanly with the multi-target plural API (`targets`, `--report-json`, `--report-html`, `--pdf-rules`/`--office-rules`/`--zip-rules`/`--image-rules`). -- `scanner/reporters.py` had a duplicate `format_row` inner function, a duplicate `rows.append` block (causing `SyntaxError: '(' was never closed`), and a legacy `generate_html_report` literally nested inside the new function's return string. Tail rewritten with a single rich HTML renderer. -- `scanner/scanner_core.py` had a duplicate trailing `if config.use_yara` block where the second copy returned undeduplicated findings. -- `scanner/heuristics.py` had a doubled assignment in `calculate_score` that silently overwrote the new `code/rule` fallback with the older `code`-only lookup. -- Duplicate lines in `requirements.txt` and `requirements-optional.txt` deduplicated. +- **Oversized, unreadable and errored files were reported as `Safe`.** They are + now `COULD NOT FULLY INSPECT`. This was the most dangerous defect in the + previous release. +- **Every `.docx` was flagged.** `analyze_office` raised + `office_auto_actions_hint` on the presence of `word/settings.xml`, which exists + in every Word document ever saved. Removed. +- **PDF tokens spanning a chunk boundary were missed.** The scanner used a + 10-byte overlap while searching for tokens up to 13 bytes long. Overlap is now + derived from the longest token, with a regression test. +- **`/JS` and `/JavaScript` were counted as two separate findings**, doubling the + score for one fact. Markers are now grouped and fire once. +- **Appended image payloads larger than 8 KB were reported as clean.** The + trailing-data check only read the last 8 KB, so the terminator fell outside the + window precisely when the payload was large. It now searches backwards in + growing windows, and reports "terminator not found within the tail limit" as a + finding rather than silence. +- **Quarantining two files with the same name destroyed the first.** The + destination was `dest_dir / src.name` with no collision handling. Stored names + now include a content hash. +- **Quarantined files kept their original extension**, so a double-click still + handed them to the shell. They now gain a `.quarantined` suffix and lose their + execute bits. +- Symlinks are no longer followed: a link to `/dev/urandom` in an untrusted + folder would have hung the hasher indefinitely. +- Detector exceptions no longer abort a scan; the file becomes + `COULD NOT FULLY INSPECT` with the error recorded. +- Repeated identical findings can no longer inflate the risk score + (per-finding-code cap). +- Report output sanitises bidi and zero-width characters, so a filename using a + right-to-left override cannot spoof its own rendering *inside the report*. + +### Security +- All archive and document reads are bounded; decompression bombs, member + floods and unbounded recursion are detected and refused rather than absorbed. +- Archives are never extracted to disk, so a traversal path can never be written. +- OOXML relationship XML is parsed with `defusedxml` when available and by byte + scan when not — never by stdlib ElementTree on untrusted input. +- Quarantine directory is created `0700`, stored files `0600`. ### Removed -- Orphan `teacher-safe-scanner/` subdirectory at the repository root, a stale duplicate from the v0.1 "move project to root" refactor. -- Legacy `scanner/detectors.py` (213 lines) shadowed by the `scanner/detectors/` package; the package version is the canonical implementation. +- `PySimpleGUI` dependency (licensing; see above). +- `scanner/heuristics.py` additive weight table, superseded by + `scanner/verdict.py`. +- `scanner/detectors/{zip,pdf,office,image}_rules.py`, superseded by the + detectors in the same package. +- Duplicated `mypy.ini`, `ruff.toml` and `setup.cfg` configuration, now in + `pyproject.toml`. -### Changed -- Project URLs in `pyproject.toml` updated from `example.com` placeholders to the real `constripacity/Teacher-Safe-Local-File-Scanner` GitHub URLs. -- Console entry point added: `teacher-safe-scan = scanner.main:main`. -- `.gitignore` expanded with standard Python/.env/.venv/IDE/OS entries. +## [0.2.0] — 2026-05-29 +- CI fixes and release repair. -## [0.1.0] - 2024-01-01 -### Added -- Initial release of the Teacher-Safe Local File Scanner scaffold with CLI, detectors, heuristic scoring, reporters, and quarantine tooling. -- Example benign samples and example report for testing. -- GitHub Actions workflow for pytest and ruff. +## [0.1.0] +- Initial release: static detectors for ZIP/Office/PDF/image, heuristic scoring, + console/JSON/HTML reporting, PySimpleGUI window, quarantine helper. diff --git a/README.md b/README.md index fc2499d..4a46fd9 100644 --- a/README.md +++ b/README.md @@ -1,277 +1,283 @@ -# Teacher-Safe Local File Scanner +

Teacher-Safe Local File Scanner

-> **Defensive notice:** This project is provided for educational and defensive use by teachers and school IT staff. It is **not** a replacement for enterprise antivirus or endpoint protection. +

+ Point it at a folder of student submissions. It tells you which ones not to open, and why — in plain English. +

-![Demo GIF placeholder](https://img.shields.io/badge/Demo%20GIF-coming%20soon-blue) +

+ Runs entirely on your machine · never opens or runs the files it checks · no account, no upload, no API key +

-> To add your own walkthrough, drop a GIF at `docs/demo.gif` and update this link. +

+ Quickstart · + What it checks · + How verdicts work · + Limits · + Contributing +

-Teacher-Safe Local File Scanner is a Python-based, offline-friendly toolkit that helps educators quickly triage student-submitted files before opening them. It performs static checks only—no execution of untrusted code—and produces human-readable and machine-readable reports. +--- -## Table of contents +``` + 15 of 29 files should not be opened. 4 more need a closer look. + + ✖ 15 DO NOT OPEN ! 4 CAUTION ? 1 NOT CHECKED ✓ 9 LIKELY SAFE + + FILE VERDICT WHY + ------------------------------------------------------------------------------ + image_is_really_a_program.jpg ✖ DO NOT OPEN File contains a program, whatever it is named + invoicegpj.exe ✖ DO NOT OPEN Filename uses a right-to-left override character + office_remote_template.docx ✖ DO NOT OPEN Document links out to a remote template + office_with_macro.docm ✖ DO NOT OPEN Document contains a macro project + archive_path_traversal.zip ✖ DO NOT OPEN Archive entry escapes its own folder + pdf_launch_action.pdf ✖ DO NOT OPEN PDF tries to launch another program + links_suspicious.txt ! CAUTION Link uses a lookalike internationalised domain + broken_upload.zip ? NOT CHECKED Archive could not be opened +``` -1. [Features](#features) -2. [Quickstart](#quickstart) -3. [How scanning works](#how-scanning-works) -4. [Command reference](#command-reference) -5. [Optional defensive plugins](#optional-defensive-plugins) -6. [Workflow guidance for flagged files](#workflow-guidance-for-flagged-files) -7. [Safety, ethics, and limitations](#safety-ethics-and-limitations) -8. [Cross-platform notes](#cross-platform-notes) -9. [Reports and outputs](#reports-and-outputs) -10. [Troubleshooting & FAQ](#troubleshooting--faq) -11. [Development](#development) -12. [Contributing](#contributing) -13. [License](#license) +> **To add a screenshot:** run `teacher-safe-scan scan examples/benign_samples --report-html demo.html`, +> open `demo.html`, and drop the image at `docs/report.png`. The corpus is +> committed, so anyone can reproduce exactly the output above. -If you are new to command-line tools, start with the [Beginner Guide](BEGINNERS_GUIDE.md) for a slower, step-by-step walkthrough. +## Why this exists -## Features +A teacher gets thirty files through the LMS. One of them is a `.docm` with a +macro. Existing tools do not help with that specific problem: -- Static detectors for risky constructs in ZIP, Office, PDF, and image files -- Heuristic scoring with clear severity labels -- Console, JSON, and HTML reporting -- Optional directory watch mode using polling -- Quarantine helper that moves, never deletes, suspicious files -- Cross-platform (Windows, macOS, Linux) with standard library defaults -- Optional integrations with `python-magic` and `yara-python` +| Tool | What it gives you | +| --- | --- | +| **Antivirus** | A verdict on *known* malware. Silence on a novel macro document. | +| **[Dangerzone](https://github.com/freedomofpress/dangerzone)** | A sanitised copy of one file. Tells you nothing about what was in it, and can't take a folder. | +| **[oletools](https://github.com/decalage2/oletools)**, pdfid, YARA | Excellent analyst output. Assumes you already know what `VBA_PROJECT` means. | +| **VirusTotal-backed CLIs** | Great results — after uploading student work to a third party. | +| **This** | One page. One row per submission. Worst first. A sentence per finding you could forward to a parent. | -## Quickstart +The unit of work here is **the folder**, not the file. That is the whole product. -```bash -python -m venv .venv -source .venv/bin/activate # On Windows use: .venv\\Scripts\\activate -pip install -r requirements.txt -python examples/generate_benign_samples.py # Materialise demo files -``` +## 30-second start -### Scan files or folders +No install, no dependencies — the scanner core is pure standard library. ```bash -python -m scanner scan ./examples/benign_samples --max-file-size 5000000 --threads 4 -python -m scanner.main scan ./examples/benign_samples --max-file-size 5000000 --threads 4 -``` - -- Exit code `0`: no suspicious findings -- Exit code `1`: caution or suspicious findings -- Exit code `2`: high severity findings -- Exit code `3`: internal scanner error +git clone https://github.com/constripacity/Teacher-Safe-Local-File-Scanner +cd Teacher-Safe-Local-File-Scanner -### Watch a directory (polling, non-blocking) - -```bash -python -m scanner scan --watch ./incoming -python -m scanner.main scan --watch ./incoming +python examples/generate_benign_samples.py # 28 harmless files that trip every detector +python -m scanner scan examples/benign_samples # see the output above ``` -### Produce reports +Then point it at real work: ```bash -python -m scanner scan submissions --report-json scan_report.json --report-html scan_report.html -python -m scanner report scan_report.json --html --output scan_report.html -python -m scanner.main scan submissions --output scan_report.json -python -m scanner.main report scan_report.json --html --output scan_report.html +python -m scanner scan ~/Downloads/period-3-submissions --report-html report.html --open-report ``` -### Quarantine a file +Prefer a window? `python -m scanner gui` (needs `tkinter`; on Debian/Ubuntu: +`sudo apt install python3-tk`). -```bash -python -m scanner quarantine ./submissions/suspicious.docx --dest ./quarantine -python -m scanner.main quarantine ./submissions/suspicious.docx --dest ./quarantine -``` - -The quarantine command moves the file safely, sets read-only permissions, and leaves a `.meta.json` file with provenance details. - -### Refreshing the benign examples - -If you delete the generated examples or clone the repository fresh, run: +Install it properly if you want the `teacher-safe-scan` command on your PATH: ```bash -python examples/generate_benign_samples.py +pip install -e . +teacher-safe-scan scan ~/Downloads/submissions ``` -The script recreates a harmless text file, a minimal PNG image, and a macro-free `.docx` document without storing binary fixtures in the repository. - -## One-click binaries - -Grab the latest release assets for Windows, macOS, or Linux to run the scanner without Python. Each bundle ships offline-first and collects no telemetry. +## What it actually checks + +Every check is **static**. Files are read as bytes. No macro runs, no PDF is +rendered, no archive is extracted, no image is decoded. + +
+Archives (.zip, .jar, .apk) + +- entries that unpack **outside** the extraction folder (`../`, absolute paths, drive letters, UNC) +- executables and scripts inside the archive +- members disguised with a double extension (`essay.pdf.exe`) +- right-to-left overrides, zero-width and NUL characters in member names +- password-protected entries — reported as *not checked*, never as clean +- decompression-bomb shape: per-entry ratio and total unpacked size +- nested archives, opened to a bounded depth; anything deeper is reported as unchecked +
+ +
+Office documents (.docx/.xlsx/.pptx, .docm/.xlsm/.pptm, legacy .doc/.xls) + +- macro projects (`vbaProject.bin`), and whether they are signed +- **remote template injection** — the `attachedTemplate` external relationship +- DDE / DDEAUTO field codes +- ActiveX controls and embedded OLE objects +- legacy compound-file macro storage +- a file whose container does not match its extension + +Relationship XML is parsed with `defusedxml` when installed, and by a byte scan +when it is not — never by stdlib ElementTree, which is not safe on hostile input. +
+ +
+PDFs + +- `/Launch` actions +- JavaScript (`/JS`, `/JavaScript`) — counted once, not twice +- `/OpenAction` and `/AA` automatic actions +- embedded file attachments +- encryption (reported as *not fully checked*) +- content appended after the final `%%EOF` +- a PDF header that is not at byte zero +
+ +
+Images (.png, .jpg, .gif) + +- a program renamed to `.jpg` (content is checked, not the name) +- **polyglots** — a real ZIP, PE or PDF hidden after the image terminator +- large appended payloads, found however far from the end they are +- oversized metadata chunks +- impossible internal chunk lengths +
+ +
+Every file + +- extension vs. real content mismatch, by magic bytes +- double extensions and executable extensions +- right-to-left override and invisible characters in the filename +- shortened, raw-IP and punycode links inside text files +- SHA-256, and identical files submitted more than once +
+ +## How a verdict is decided + +Four verdicts, and **"could not check" is never rounded down to "safe"**: + +| | Meaning | +| --- | --- | +| ✓ **LIKELY SAFE TO REVIEW** | Nothing matched. Not a guarantee. | +| ! **REVIEW WITH CAUTION** | Something is worth a human look before opening. | +| ✖ **DO NOT OPEN — CONTACT IT** | A high-severity indicator matched with usable confidence. | +| ? **COULD NOT FULLY INSPECT** | Encrypted, too large, corrupt, or beyond a limit. **Unchecked ≠ clean.** | + +The verdict comes from four stated rules, not from summing opaque numbers. +Severity ("how bad if true") and confidence ("how sure are we") are tracked +separately, so *"this definitely has a macro"* and *"this might have an appended +payload"* are never treated alike. Full model, including the exact rules and the +per-finding-code cap that stops one hostile archive inflating a score: +**[docs/SCORING.md](docs/SCORING.md)**. + +Every finding carries five things, and the report shows all of them: -### Windows context menu - -1. Copy `TeacherSafeScanner.exe` to `C:\Program Files\TeacherSafe\`. -2. Double-click `scripts/windows_add_context_menu.reg` to register a **Scan with Teacher-Safe** right-click option. - -### GUI launcher - -- On Python: run `python -m scanner.gui` and use the picker to select files or folders, then press **Scan** and **Open Report**. -- On packaged builds: launch `TeacherSafeScanner` from the extracted bundle and follow the same steps to save and open the HTML report. - -## How scanning works - -The scanner combines lightweight type identification, static detectors, and heuristic scoring: - -| Phase | What happens | Key modules | -| --- | --- | --- | -| Discovery | Files are walked recursively (respecting `--max-file-size`) and hashed using streaming reads. | [`scanner.utils`](scanner/utils.py) | -| Type sniffing | If `python-magic` is enabled, MIME detection is delegated; otherwise magic bytes are inspected. | [`scanner.scanner_core`](scanner/scanner_core.py) | -| Detection | Format-specific rules look for risky markers (e.g., macros, embedded executables, appended payloads). | [`scanner.detectors`](scanner/detectors/__init__.py) | -| Detection | Format-specific rules look for risky markers (e.g., macros, embedded executables, appended payloads). | [`scanner.detectors`](scanner/detectors.py) | -| Scoring | Each finding contributes a weighted score mapped to Safe/Caution/Suspicious/High labels. | [`scanner.heuristics`](scanner/heuristics.py) | -| Reporting | Results are aggregated into JSON, console, or HTML outputs. | [`scanner.reporters`](scanner/reporters.py) | - -The entire pipeline avoids running untrusted content and is safe to execute on offline, air-gapped devices. - -## Command reference - -The CLI exposes three subcommands and several shared options: -The CLI exposes three subcommands and several shared options. - -### `scan` - -Scan one file or a directory tree. - -```bash -python -m scanner.main scan [--output report.json] [--threads 8] [--max-file-size 200000000] ``` +[HIGH · high confidence] Archive entry escapes its own folder -Useful flags: + One of the files inside this archive is set to unpack somewhere + outside the folder you unzip it into. -- `--watch `: poll for new files while continuing to monitor previously scanned ones. -- `--report-json` / `--report-html`: save structured and teacher-friendly reports in one run. -- `--pdf-rules`, `--office-rules`, `--zip-rules`, `--image-rules`: choose `off`, `normal`, or `strict` for per-format heuristics. -- `--use-magic` / `--use-yara`: opt into external libraries when installed. -- `--max-file-size`: skip overly large submissions to save time. -- `--threads`: increase if you have many CPU cores and fast storage. - -The scan command exits with a severity-driven code so it integrates well with CI or folder monitors. - -### `quarantine` + Why this matters This is how an archive overwrites a file elsewhere on + the computer the moment it is extracted. There is no + legitimate reason for a student submission to do this. + What to do Do not extract this archive. Send it to IT. + Evidence ../../autorun.txt + Detected by archive +``` -Move suspicious files to a safe holding area without deleting them. +## Reports ```bash -python -m scanner.main quarantine ./submissions/suspicious.docx --dest ./quarantine +teacher-safe-scan scan ./submissions \ + --report-html report.html \ # self-contained: no scripts, no network requests + --report-json report.json # for scripting, or re-render later with `report` ``` -The destination receives a read-only copy plus a `.meta.json` file recording the original location, hash, and timestamp. - -### `report` +The HTML report groups files worst-first, opens the flagged ones by default, +prints cleanly, and can be forwarded to IT as a single file. It contains **no +JavaScript and makes no network requests** — a report about untrusted files +should not itself phone anywhere. -Render previously generated JSON results into other formats. +## Quarantine — and getting files back ```bash -python -m scanner.main report scan_report.json --html --output scan_report.html +teacher-safe-scan scan ./submissions --quarantine-dir ./quarantine +teacher-safe-scan quarantine-list --dest ./quarantine +teacher-safe-scan restore 3f9a21c40b8e --dest ./quarantine ``` -Omit `--html` to stream a human-readable console summary instead. +- **Nothing is ever deleted.** Quarantine moves; restore moves back. +- Stored copies get a `.quarantined` suffix and lose their execute bits, so a + double-click hands them to nothing. +- Two students submitting `assignment.docx` produce two distinct entries. No + overwrite, no data loss. +- Every move is written to an append-only JSONL manifest with the SHA-256 before + and after, so a restore is verified and provable. -## Optional defensive plugins +## Command reference -Install optional packages only if your environment permits: +| Command | What it does | +| --- | --- | +| `scan ` | Scan files or folders and print the triage table | +| `scan … --watch` | Re-scan only files that appear or change | +| `scan … --quarantine-dir DIR` | Move blocked files aside (nothing deleted) | +| `report --html ` | Re-render a saved report | +| `quarantine --dest DIR` | Move one file into quarantine | +| `quarantine-list --dest DIR` | List what is in quarantine | +| `restore --dest DIR` | Move a file back out, hash-verified | +| `gui` | Open the desktop window | +| `make-samples` | Write the benign test corpus | + +Exit codes: `0` nothing found · `1` needs attention · `2` do not open · `3` +scanner error · `4` usage error. Useful in a script: ```bash -pip install -r requirements-optional.txt +teacher-safe-scan scan ./inbox || echo "something needs a look" ``` -- `python-magic`: richer MIME identification (`--use-magic`) -- `yara-python`: experimental pattern matching (`--use-yara`) - -The CLI flags are opt-in, and the scanner gracefully degrades when the libraries are unavailable. - -## Workflow guidance for flagged files +Optional extras, none required: `pip install -e ".[xml]"` (hardened XML), +`".[yara]"` (`--yara-rules your.yar`). -1. **Do not open the file.** Treat warnings as serious until reviewed by IT. -1. Do not open the file. Treat warnings as serious until reviewed by IT. -2. Move the file to the quarantine folder for record keeping. -3. Escalate to your IT or security team with the JSON/HTML report. -4. Review in an isolated virtual machine if your institution allows it. -5. When in doubt, collect additional context (e.g., student name, assignment) in a secure ticketing system. +## What this is not -## Safety, ethics, and limitations +- **Not antivirus.** No signature database, no known-malware detection. Run it + *alongside* your school's endpoint protection, never instead of it. +- **Not a guarantee.** "Likely safe" means nothing matched the checks in this + tool. A novel technique this tool does not model will come back clean. +- **Not a sanitiser.** It does not produce a safe copy. For that, use + [Dangerzone](https://github.com/freedomofpress/dangerzone) — the two compose + well: triage here, sanitise there. +- **Not for offensive use.** It detects; it never builds, packs, or executes. -- Static analysis only; no attempt is made to remove malware. -- Large or encrypted archives may hide malicious content the scanner cannot inspect. -- The heuristics prioritise minimizing false negatives but may produce false positives—always confirm with professional tools. -- The tool never executes or modifies untrusted binaries beyond safe hashing and metadata reads. +## Privacy -Read more in [SAFETY.md](SAFETY.md). +Nothing leaves your machine. There is no telemetry, no update check, no network +code of any kind in the scanner. Student work is student data, and it stays on +the laptop it arrived on. -## Cross-platform notes - -- Paths are managed with `pathlib`. When running on Windows, prefer PowerShell or CMD with UTF-8 enabled (`chcp 65001`). -- Quarantine sets read-only attributes; if you need to restore a quarantined file, manually adjust permissions via `attrib -r` on Windows or `chmod +w` on Unix. -- Polling-based watch mode relies on filesystem timestamps; on slow or networked drives expect a 10-second delay before changes are detected. -- For macOS Gatekeeper prompts, run `xattr -dr com.apple.quarantine ` only on files you trust and after verifying reports. - -## Reports and outputs - -Reports follow a stable JSON schema so they can be ingested by help-desk systems: +## Development -```json -{ - "path": "submissions/homework1.zip", - "sha256": "abc123...", - "size": 34567, - "magic_type": "zip", - "issues": [ - {"code": "exe_in_zip", "description": "Found executable file payload.exe inside archive", "evidence": "payload.exe"}, - {"code": "double_extension", "description": "Filename uses double extension 'report.pdf.exe'", "evidence": "report.pdf.exe"} - ], - "score": 75, - "severity": "Suspicious" -} +```bash +pip install -e ".[dev]" +pytest -q # 114 tests +ruff check scanner tests examples +mypy scanner +python scripts/check_corpus.py # detection regression: every sample must land where it should ``` -When exporting HTML the report includes: - -- A safety banner reminding readers not to open flagged files. -- A severity-coloured table summarising each item. -- Collapsible detail sections for detector evidence. -- Footer tips on next steps for educators. - -Console output defaults to a clean table suitable for terminal screenshots. Use `--verbose` during scans for additional logging. +`scripts/check_corpus.py` is the test that matters most. It asserts both +directions: every hostile-shaped sample is caught, **and** every clean sample +stays clean. A triage tool that cries wolf gets uninstalled. -## Troubleshooting & FAQ - -**The scanner skips files larger than expected.** - -- Confirm the `--max-file-size` flag; the default is 100 MB. Some learning management systems export multi-gigabyte ZIPs that may need a higher limit. - -**`python-magic` or `yara-python` import errors appear.** - -- Ensure you installed `requirements-optional.txt`. On Windows you may need the Visual C++ Build Tools; on macOS install Homebrew `libmagic` first. - -**Watching a network share misses changes.** - -- Keep the watch directory local when possible. The default 10-second polling interval may drift on congested networks—re-run the command if scans appear delayed. - -**How do I update the benign sample files?** - -- Run `python examples/generate_benign_samples.py --force` to regenerate all fixtures. The script never overwrites files unless the hash changes, so it is safe to run repeatedly. - -**Can I integrate results into another system?** - -- Yes. The JSON report is linearly structured. Use `jq`, Python, or your preferred language to parse the `issues` array per file. The exit code makes automation straightforward. - -## Development +Building a standalone binary (unsigned — see the script's own warning): ```bash -pip install -r requirements.txt -pytest -ruff check . +pip install -e ".[build]" +python scripts/build_binary.py ``` -Recommended editor settings: - -- Enable `black`-style formatting at 88 columns. -- Turn on type checking (MyPy or Pyright) for early detection of annotation issues. -- Configure your IDE to respect `.editorconfig` if present. - ## Contributing -We welcome defensive-minded contributions. See [CONTRIBUTING.md](CONTRIBUTING.md) for coding standards and submission guidelines. +New detectors are very welcome, especially with a benign sample in +`examples/generate_benign_samples.py` and a row in `scripts/check_corpus.py`. +Start with [CONTRIBUTING.md](CONTRIBUTING.md) and +[docs/ARCHITECTURE.md](docs/ARCHITECTURE.md). Please read +[SAFETY.md](SAFETY.md) before opening a PR — this project stays defensive. ## License -MIT License © Teacher Safe Maintainers -``` +MIT — see [LICENSE](LICENSE). diff --git a/SAFETY.md b/SAFETY.md index 9d13e46..0b77c13 100644 --- a/SAFETY.md +++ b/SAFETY.md @@ -1,15 +1,74 @@ -# Safety & Ethics Guidance +# Safety and scope -Teacher-Safe Local File Scanner exists to reduce risk for teachers receiving student files. It must be used responsibly: +## This project is defensive only -- **Defensive only:** The project must never be repurposed to build or distribute malicious tooling. Contributions that violate this principle will be rejected. -- **No execution:** The scanner reads metadata and file structures only. It never runs embedded macros, executables, or scripts. -- **Verification:** Treat scanner output as advisory. Confirm suspicious findings with professional antivirus or your institution’s security operations before taking disciplinary action. -- **Handling flagged files:** - 1. Quarantine the file using the provided command or another safe storage mechanism. - 2. Notify your IT or security team and share the generated reports. - 3. Review the file only inside an isolated sandbox with no network access. - 4. Document all actions for accountability and compliance. -- **Data privacy:** Reports may contain file paths or filenames. Store them securely and follow your school’s privacy requirements. +Teacher-Safe Local File Scanner exists to help a teacher or school IT assistant +decide **which files not to open**. It detects; it never builds, packs, obfuscates +or executes anything. -The maintainers welcome responsible disclosures and feedback via issues or pull requests. +Contributions that add offensive capability will be declined. That includes: +payload generation, exploitation, evasion testing against other products, +credential harvesting, or anything whose primary use is attacking rather than +triaging. + +## The absolute constraint + +**Untrusted content is never executed.** In this codebase that means, with no +exceptions: + +- no macro is run, and no Office document is opened by an Office application +- no PDF is rendered, and no PDF library that executes embedded content is used +- no archive is ever extracted to a filesystem path +- no image is decoded by an image library +- no file is passed to a shell, an interpreter, or `subprocess` + +Detectors read bytes and report structure. `tests/test_detectors_archive.py::test_nothing_is_written_to_disk` +guards the archive case explicitly. + +## Resource limits are part of the threat model + +Hostile input tries to exhaust the machine that inspects it. Every bound the +scanner obeys lives in one struct, `scanner/limits.py`: + +| Limit | Default | Guards against | +| --- | --- | --- | +| `max_file_size` | 100 MB | reading an enormous submission into memory | +| `max_read_bytes` | 8 MB | a detector holding a whole file | +| `max_archive_members` | 5,000 | central-directory floods | +| `max_archive_depth` | 3 | archive-in-archive recursion | +| `max_total_uncompressed` | 512 MB | decompression bombs | +| `max_compression_ratio` | 200:1 | per-entry bomb ratio | +| `max_nested_extract_bytes` | 16 MB | bounded nested reads | +| `max_findings_per_file` | 200 | report and memory blowup | + +Anything cut off by a limit is reported as **COULD NOT FULLY INSPECT** — never +as safe. + +## No malware in this repository + +The test corpus in `examples/generate_benign_samples.py` reproduces the +*structure* of risky files using inert payloads (`echo "harmless test payload"`). +An archive member that escapes its folder contains a text file. A "polyglot" PNG +has a real, harmless ZIP appended. + +Never commit a real sample, even a defanged one, even in an encrypted archive. +If a detector needs a real-world sample to develop against, obtain it from a +malware repository under your own account, keep it out of git, and contribute the +detector plus a synthetic fixture. + +## Reporting a vulnerability + +See [SECURITY.md](SECURITY.md). In short: for a flaw in this scanner, please open +a private security advisory rather than a public issue — a bypass in a triage +tool is a real risk to the people using it. + +## Honest limits, stated in the product + +The tool says all of the following in its own output, and this project treats +weakening any of them as a bug: + +- it is **not antivirus** and has no signature database +- "likely safe" means "nothing matched the checks this tool performs" +- a novel technique the tool does not model will come back clean +- encrypted, oversized and corrupt files are reported as *unchecked*, and + unchecked is not clean diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..c130548 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,91 @@ +# Architecture + +``` +scanner/ +├─ findings.py Finding, Severity, Confidence, Verdict — the vocabulary +├─ limits.py ScanLimits — every ceiling the scanner obeys, in one struct +├─ verdict.py The four rules that turn findings into a verdict +├─ scanner_core.py Walk targets, open each file once, dispatch, decide +├─ triage.py Folder-level summary: counts, duplicates, headline +├─ reporters.py Console / JSON / self-contained HTML +├─ quarantine.py Move-only store with an append-only manifest and restore +├─ gui_model.py Everything the window shows — no Tk import, fully tested +├─ gui.py Tkinter widgets and wiring only +├─ main.py CLI +└─ detectors/ + ├─ base.py Bounded reads, magic sniffing, filename analysis primitives + ├─ archive.py ZIP family: traversal, bombs, encryption, bounded recursion + ├─ office.py OOXML + legacy OLE + ├─ pdf.py Bounded streaming structural scan + ├─ image.py PNG/JPEG/GIF container structure and polyglots + └─ general.py Filename and content checks that apply to any file +``` + +## Two invariants + +Every detector must hold both. They are the reason this tool is safe to run on +hostile input. + +**1. Nothing is executed.** No macro runs, no PDF is rendered, no archive is +extracted to a path the operating system could act on, no image is decoded. +Detectors read bytes and report structure. + +**2. Nothing is unbounded.** Reads, recursion depth, archive member counts and +finding counts are all capped by `ScanLimits`. Detectors receive an open handle +and stream; none calls `f.read()` without a bound. + +`tests/test_detectors_archive.py::test_nothing_is_written_to_disk` is a +regression guard for the first. `ScanLimits` is a single frozen dataclass so a +reviewer can read one struct and know exactly what the scanner refuses to do. + +## Data flow + +``` +path + └─ scan_file() + ├─ analyze_name() filename-only checks + ├─ stat / symlink / size gates → COULD_NOT_INSPECT on any failure + ├─ open once → sha256, magic sniff + ├─ _dispatch() by sniffed type first, extension second + │ ├─ office / archive / pdf / image / text detectors + │ └─ optional YARA + ├─ _dedupe() (code, evidence) pairs + └─ decide() the four rules → Verdict + risk_score +``` + +Dispatch is by **content, then name**. A Windows executable renamed +`holiday_photo.png` is analysed as an executable, because that is what the +operating system will do with it. + +## Adding a detector + +1. Write a module in `scanner/detectors/` exposing + `analyze_x(handle, *, limits, ...) -> list[Finding]`. +2. Every `Finding` needs all five human fields: `plain`, `why`, `action`, + `evidence`, plus `severity` and `confidence`. If you cannot write the `why` in + one sentence a non-technical reader understands, the detector is not ready. +3. Set `inspection_incomplete=True` for anything meaning "I could not finish + looking". That is what keeps unchecked files out of the green column. +4. Wire it into `scanner_core._dispatch`. +5. Add a **benign** sample to `examples/generate_benign_samples.py` and a row to + `scripts/check_corpus.py`. Never commit real malware; reproduce the + *structure* with an inert payload. +6. Add tests in `tests/test_detectors_*.py`, including at least one asserting a + normal file does **not** trigger it. False positives are the failure mode + that gets this tool uninstalled. + +## Why the GUI is split in two + +`gui_model.py` has no `import tkinter`. Every decision about what the window +shows — row ordering, colours, detail text, the paste-into-email summary — lives +there and is unit tested on a machine with no display. `gui.py` is widgets and +wiring, and is exercised by hand. This split exists because the development and +CI environments for this project frequently have no `tkinter` at all. + +## Threading + +`scan()` uses a `ThreadPoolExecutor`; the work is I/O-bound so the GIL is not the +constraint. Each `scan_file` call opens its own handle and shares nothing +mutable. The GUI runs the scan on a worker thread and marshals progress back +through a `queue.Queue` drained on the Tk main loop — Tk widgets are only ever +touched from the main thread. diff --git a/docs/NEXT_20_COMMITS.md b/docs/NEXT_20_COMMITS.md new file mode 100644 index 0000000..ddb781d --- /dev/null +++ b/docs/NEXT_20_COMMITS.md @@ -0,0 +1,1149 @@ +# The next 18 commits + +## Where this leaves off + +v0.3.0 is a working folder-triage tool. `teacher-safe-scan scan ` walks a +directory, opens each file once, dispatches by sniffed content type, and prints a +worst-first table under a plain-English headline; the four-verdict model in +`scanner/verdict.py` is tested rule by rule, `COULD NOT FULLY INSPECT` is never +rounded down to safe, and `scripts/check_corpus.py` asserts both directions over +28 generated samples. What is **not** verified is the part a teacher touches +first: the Tkinter window in `scanner/gui.py` has never been executed by anyone, +because `tkinter` is absent from the environment this revival was built in +(`gui_model.py` is unit tested; `gui.py` is not, and CI has no job that opens a +window). The PyInstaller binary has likewise never been built or run — the spec +and `scripts/build_binary.py` are written but unexercised locally, and the whole +revision is still an uncommitted working tree, so no CI run has ever seen it. +Three defects were found by reading and are not yet fixed: `.rar`, `.7z`, +`.tar.gz`, `.tar` and `.rtf` submissions receive **no** content analysis at all +and come back `LIKELY SAFE TO REVIEW` (verified — this is the same class of bug +the revival release fixed for oversized files); every structurally clean PDF over +8 MB is reported `COULD NOT FULLY INSPECT` because `iter_windows` bounds total +bytes streamed by `max_read_bytes` (verified with a 12.1 MB file); and +`teacher-safe-scan make-samples` will fail on any pip-installed copy, because +`handle_make_samples` imports `examples.generate_benign_samples` while +`pyproject.toml` packages only `scanner*`. `SAFETY.md` links a `SECURITY.md` that +does not exist, `CONTRIBUTING.md` still tells contributors to install a +`requirements-optional.txt` that was deleted, and `--pdf-rules strict` is accepted +by the CLI and does nothing. + +The ordering below is: verify what could not be verified, then make the thing +installable, then build the LMS-ingestion path that is the actual wedge, then +detector depth, then the rest. Commits 03 and 17 are defect fixes sitting inside +the verification and tail blocks respectively; if anything slips, pull those two +forward rather than shipping a signed binary that carries them. + +--- + +## 01 — test(gui): open the window in CI and assert what it renders + +**Goal** Execute `scanner/gui.py` for the first time, on all three platforms, and +leave behind a test suite that fails when the window breaks. + +**Why it matters** `docs/ARCHITECTURE.md` says `gui.py` "is exercised by hand". +It has not been exercised by anyone. Every claim in the README about the desktop +window is currently unverified, and the window is the entry point for the user +this project is named after. + +**Files** +- `tests/test_gui_window.py` — new +- `.github/workflows/ci.yml` — a `gui` job, and `python3-tk` + `xvfb` on the Linux runner +- `pyproject.toml` — register a `gui` marker under `[tool.pytest.ini_options]` +- `scanner/gui.py` — only what the first run forces (keep fixes in 02) + +**Implementation** Gate the module with `tk = pytest.importorskip("tkinter")` and +a `try: root = tk.Tk() / except tk.TclError: pytest.skip("no display")`. Never +call `mainloop()`; construct `ScannerWindow(root)`, then drive it with +`root.update()` and manual `_drain_queue()` polls until `self._summary` is set. +Reuse the existing `corpus` fixture from `tests/conftest.py` so the expected six +rows and their verdicts are already pinned by `test_scanner_core.py`. +Monkeypatch `filedialog.asksaveasfilename` and `messagebox.*` — a test that +blocks on a modal dialog hangs CI forever. On Ubuntu, `setup-python` builds do +ship `tkinter`, but the runner has no X server, so wrap the job in `xvfb-run -a`; +macOS and Windows runners need neither. Note that `addopts` already carries +`--strict-markers`, so the `gui` marker must be registered or every gui test +errors on collection. + +**Tests** Window constructs without raising. `_start_scan` against the corpus +populates the Treeview with six rows, worst-first, each tagged with its verdict +slug. Selecting each row leaves `self.detail` non-empty and containing the +finding's `plain` text. `_save_html` and `_save_json` write real files. +`_copy_summary` puts `summarise_for_email` output where `root.clipboard_get()` +can read it. `_quarantine` with a monkeypatched `askdirectory` moves exactly the +`DO NOT OPEN` files and no others. + +**Depends on:** nothing + +**Risk** Low for the codebase, moderate for CI stability: Tk tests are flaky when +they race the event loop. Keep every assertion behind an explicit `root.update()` +and never sleep. If the macOS runner proves unreliable, run gui tests on Linux and +Windows only and say so in the workflow comment rather than marking them +`xfail`. + +**Acceptance criteria** +- [ ] `pytest -m gui` passes locally under `xvfb-run` on Linux +- [ ] The `gui` job is green on ubuntu-latest, windows-latest and macos-latest +- [ ] Every public method of `ScannerWindow` is touched by at least one test +- [ ] No test can block on a dialog; the suite finishes in under 60 s per platform +- [ ] `docs/ARCHITECTURE.md` no longer says the GUI is exercised only by hand + +**Scope** M + +--- + +## 02 — fix(gui): the defects the first real run exposes + +**Goal** Fix what commit 01 turns up, starting with the four failures that +reading the code predicts. + +**Why it matters** A window that renders wrong, or silently loses the summary a +teacher just copied, is worse than no window: it is the part of the product that +gets demonstrated. + +**Files** +- `scanner/gui.py` — `_copy_summary`, `_pick_files`, `_quarantine`, `_build`, `_start_scan` +- `scanner/gui_model.py` — `VERDICT_COLORS`, a dark-mode variant +- `tests/test_gui_window.py` — a regression test per fix + +**Implementation** Four predicted defects, in order of user impact. (1) +`_copy_summary` calls `clipboard_clear()`/`clipboard_append()`; on X11 and on +Windows the clipboard is owned by the process, so the text vanishes when the +teacher closes the window to go and paste it. Follow the append with +`self.root.update()` and add a "Save summary as .txt…" fallback next to it. (2) +`_pick_files` joins the `askopenfilenames` tuple with `";"` and hands it to +`parse_dropped_paths`, which splits on `";"` — any path containing a semicolon is +silently split into two nonexistent paths. Keep the real list on the instance and +use the joined string for display only. (3) After `_quarantine` moves files, the +Treeview still lists them at paths that no longer exist and selecting one shows +stale detail; mark moved rows or re-scan. (4) `self.detail` is a bare `tk.Text` +with default colours while the ttk widgets follow the system theme, so it is a +white block in a dark window on macOS; set explicit foreground/background, and +give `VERDICT_COLORS` a dark variant since its four background colours are chosen +for a light ground. Also confirm on macOS whether `Treeview.tag_configure` +backgrounds actually apply under the `clam` theme — if they do not, colour is +still not the only channel, because `VERDICT_SYMBOL` is already in the row text. + +**Tests** One regression test per fix. The clipboard case asserts the text +survives a `root.update()` cycle; the path case asserts a filename containing +`;` round-trips; the quarantine case asserts no row points at a missing file +after the move. + +**Depends on:** 01 + +**Risk** Low. Confined to widgets and the presentation model, both of which now +have tests. The dark palette is a judgement call — pick contrast ratios that +still pass 4.5:1 against the dark ground and keep the symbols. + +**Acceptance criteria** +- [ ] Copying the summary, closing the window, and pasting yields the summary +- [ ] A path containing `;` scans correctly from the file picker +- [ ] After quarantining, no visible row refers to a moved file +- [ ] The window is legible under both a light and a dark system theme on all three platforms +- [ ] Each fix has a test that fails on the parent commit + +**Scope** M + +--- + +## 03 — fix(detectors): unreadable container formats are unchecked, not safe + +**Goal** Stop reporting `.rar`, `.7z`, `.tar`, `.tar.gz`, `.bz2` and `.xz` +submissions as `LIKELY SAFE TO REVIEW` when nothing has looked inside them. + +**Why it matters** Verified: a file with RAR magic and a `.rar` name produces zero +findings and lands in the green column. `analyze_archive` is ZIP-only, +`_dispatch` runs it only when `detected == "zip"`, and `EXTENSION_EXPECTATIONS` +happily confirms that a `.rar` contains RAR — so the extension/content check +passes too and the file falls through every branch in silence. This is exactly +the defect class the revival release fixed for oversized files, and the corpus +never caught it because the corpus contains no RAR or 7z sample. It also +contradicts the project's loudest claim: "unchecked ≠ clean". + +**Files** +- `scanner/detectors/base.py` — `UNINSPECTABLE_CONTAINER_TYPES`, a `ustar`-at-257 case in `sniff_magic` +- `scanner/scanner_core.py` — a branch in `_dispatch` +- `examples/generate_benign_samples.py` — three samples in `SAMPLES` +- `scripts/check_corpus.py` — three rows in `EXPECTED` +- `README.md`, `SAFETY.md` — state the limitation where the limits are stated + +**Implementation** Add a frozen set of container types this scanner cannot read +(`rar`, `7z`, `gzip`, `bzip2`, `xz`, `tar`) to `base.py`, next to +`MAGIC_SIGNATURES` where a reviewer will find it. `sniff_magic` currently takes +only the head bytes and matches prefixes; add a `ustar` check at offset 257 so a +plain `.tar` stops sniffing as `unknown`. In `_dispatch`, after the existing +format branches, emit one finding when `detected` is in that set: severity `LOW`, +confidence `HIGH`, `inspection_incomplete=True`, so rule 1 in `decide()` carries +it to `COULD NOT FULLY INSPECT` without inventing a threat that has not been +observed. The `plain` text should say what it is: "This is a RAR archive. This +scanner can only look inside ZIP archives, so nothing inside this file has been +checked." The `action` should be the useful one: ask for a `.zip`, which the +scanner can read. Do not add RAR/7z parsing — that means a third-party +dependency reading hostile input, which is the opposite of this project's +posture. + +**Tests** A unit test per format asserting `COULD_NOT_INSPECT` and the finding +code. A test that a ZIP is unaffected. Corpus rows expecting +`(UNKNOWN, "container_not_inspected")` for `archive_rar_shaped.rar`, +`archive_7z_shaped.7z` and `archive_tar_gz.tar.gz` — the samples only need +correct magic bytes and inert padding, which is consistent with how +`_encrypted_flag_zip` already fakes structure it cannot legitimately produce. + +**Depends on:** nothing + +**Risk** Low, but it moves files out of the green column, so a school that +routinely receives `.rar` coursework will see a new blue block. That is the +correct answer, and the finding text says so plainly. Keep it `LOW`/`HIGH` and +not `MEDIUM`, or two such files corroborate into `REVIEW WITH CAUTION` under +rule 3 and the tool starts crying wolf. + +**Acceptance criteria** +- [ ] `.rar`, `.7z`, `.tar`, `.tar.gz`, `.bz2`, `.xz` all land in `COULD NOT FULLY INSPECT` +- [ ] A plain `.tar` sniffs as `tar`, not `unknown` +- [ ] Two such files in one folder do not escalate each other past caution +- [ ] `scripts/check_corpus.py` covers all three new samples +- [ ] README's "What this is not" states which archive formats can be opened + +**Scope** M + +--- + +## 04 — test(build): verify the frozen binary past `--version` + +**Goal** Prove the PyInstaller build actually works, for every subcommand, before +anyone signs it. + +**Why it matters** `release.yml` smoke-tests `--version`, `make-samples` and +`scan`. It does not test `report`, `quarantine`, `quarantine-list`, `restore`, or +`gui`, and nothing checks that `tkinter` was bundled at all — the spec's +`try: import tkinter` runs on the *build* machine, so a runner without Tk +silently produces a binary whose `gui` command can never work. Nobody has run +this binary. + +**Files** +- `scripts/smoke_binary.py` — new +- `scripts/build_binary.py` — call it from `main()` instead of the inline `--version` check +- `.github/workflows/ci.yml` — the `build` job runs the smoke script +- `.github/workflows/release.yml` — the build matrix runs the same script + +**Implementation** One script, taking the binary path, exercising the whole +`HANDLERS` table in `scanner/main.py`: generate the corpus, scan it and assert +exit code 2, write both reports, re-render the JSON through `report`, quarantine +a file, list it, restore it, and assert the restored bytes match. Then compare the +frozen binary's JSON report against one produced by `python -m scanner` over the +same corpus, ignoring `generated_at`, `duration_ms`, `roots` and absolute paths — +if a bundled resource diverges, the verdict counts diverge and this catches it. +For the GUI, launch ` gui` with a deadline, assert the process is alive +after two seconds and then terminate it; on Linux run it under `xvfb-run`. Assert +`sys.platform == "darwin"`'s `argv_emulation=True` does not swallow arguments — +that flag has historically eaten the first argument on macOS. Record the binary +size and fail above a stated ceiling; a teacher downloading over a school network +is the constraint the spec cites for its `excludes` list. + +**Tests** The script is the test. Add a `pytest` wrapper that runs it against +`dist/` when the binary exists and skips otherwise, so a local build is checked +the same way CI checks it. + +**Depends on:** 01, 03 + +**Risk** Moderate CI time — three platforms building a onefile binary plus a full +smoke run. Cache PyInstaller's bootloader. The GUI-launch check is the flakiest +part; keep it to "process survives two seconds" rather than screenshotting. + +**Acceptance criteria** +- [ ] Every subcommand in `HANDLERS` is exercised against the frozen binary +- [ ] The frozen binary's verdicts match the source tree's, file for file +- [ ] ` gui` opens a window on all three platforms +- [ ] The build fails if `tkinter` was not bundled +- [ ] Binary size is printed in CI and fails above the agreed ceiling + +**Scope** M + +--- + +## 05 — fix(pkg): ship the corpus inside the package + +**Goal** Make `teacher-safe-scan make-samples` work on a pip-installed copy. + +**Why it matters** `handle_make_samples` does +`sys.path.insert(0, )` and then +`from examples.generate_benign_samples import write_samples`. In a wheel that +parent is `site-packages`, and `pyproject.toml` packages only `scanner*`, so the +import raises `ModuleNotFoundError` and the command dies with a traceback rather +than one of the CLI's own exit codes. It works from a git checkout and it works +in the frozen binary (the spec bundles the file as data and PyInstaller puts +`_MEIPASS` on `sys.path`), which is exactly why nobody has noticed. This must be +fixed before there is a PyPI release for anyone to hit it with. + +**Files** +- `scanner/corpus.py` — new home for `SAMPLES`, `write_samples`, the `_png`/`_zip`/`_docx`/`_pdf` builders +- `examples/generate_benign_samples.py` — becomes a thin shim re-exporting from `scanner.corpus` +- `scanner/main.py` — `handle_make_samples` imports from the package, and the `sys.path` hack goes +- `scripts/check_corpus.py` — import from `scanner.corpus` +- `teacher-safe-scan.spec` — drop the now-unneeded `datas` entry + +**Implementation** Move the module wholesale; keep the public names identical so +`check_corpus.py`'s `EXPECTED` and `tests/samples.py` need no edits. The shim at +`examples/generate_benign_samples.py` keeps the documented +`python examples/generate_benign_samples.py` command working and keeps +`ruff check scanner tests examples` meaningful. `examples/benign_samples/` stays +committed and unchanged — it is what makes the README's output reproducible. + +**Tests** Add a test that imports `scanner.corpus` with the repository root +absent from `sys.path`, which is the condition a wheel install creates. The real +proof is in 06's installed-wheel smoke test. + +**Depends on:** nothing + +**Risk** Low. The only subtlety is `write_samples` writing a `README.md` into the +output directory; that behaviour must not change, because `check_corpus.py`'s +`EXPECTED` has a row for it. + +**Acceptance criteria** +- [ ] `scanner/` has no import of `examples` +- [ ] `python examples/generate_benign_samples.py` still writes the same 28 files plus README +- [ ] `scripts/check_corpus.py` passes unchanged +- [ ] The spec no longer bundles a Python source file as data +- [ ] `pip install .` into a clean venv, then `teacher-safe-scan make-samples`, succeeds + +**Scope** S + +--- + +## 06 — feat(pkg): publish to PyPI with Trusted Publishing + +**Goal** `pip install teacher-safe-local-file-scanner` works, and the release +workflow does it without a long-lived token. + +**Why it matters** `release.yml`'s own release notes already tell users to +`pip install teacher-safe-local-file-scanner` as the alternative to an unsigned +binary. That instruction is currently false. For a teacher on a machine that has +Python but a blocked binary download, pip is the only route in, and the scanner +core needs nothing outside the standard library so the install is trivial — +once it exists. + +**Files** +- `.github/workflows/release.yml` — a `pypi` job with `permissions: id-token: write` +- `pyproject.toml` — SPDX licence metadata, 3.13 classifier, `Intended Audience :: Education`, package data for `py.typed` +- `scanner/py.typed` — new, empty +- `scripts/smoke_wheel.py` — new + +**Implementation** Build with `python -m build`, `twine check dist/*`, then +publish with `pypa/gh-action-pypi-publish` using OIDC Trusted Publishing so no API +token is stored in repository secrets — configure the publisher on PyPI against +this repository and the `release.yml` workflow before the first tag. Gate the job +on the existing `verify` job so nothing publishes without ruff, mypy, pytest and +`check_corpus.py` passing. Before publishing anything, install the built wheel +into a clean venv and run `scripts/smoke_wheel.py`: `teacher-safe-scan --version`, +`make-samples`, `scan` with both report formats, and the quarantine round-trip. +That script is what would have caught commit 05's bug. Switch +`license = { file = "LICENSE" }` to the SPDX `license = "MIT"` plus +`license-files`, which recent setuptools warns about. Ship `scanner/py.typed`, +since the package is fully annotated and `mypy scanner` is already in CI. Confirm +the distribution name is free on PyPI before tagging; the console script stays +`teacher-safe-scan` regardless. + +**Tests** `scripts/smoke_wheel.py` runs in CI on every push against a locally +built wheel, not only at release time — the install path is the one that breaks +silently. + +**Depends on:** 05 + +**Risk** Publishing is irreversible: a released version number can never be +reused. Publish `0.3.1` to TestPyPI first and install from it, then do the real +one. The `dependencies = []` line is load-bearing — if anything ever moves out of +`[project.optional-dependencies]`, the "no dependencies" claim in the README +becomes false and the bare-venv CI job is the thing that catches it. + +**Acceptance criteria** +- [ ] A tag publishes to PyPI with no API token in repository secrets +- [ ] `pip install teacher-safe-local-file-scanner` into a clean venv gives a working `teacher-safe-scan` +- [ ] The installed package pulls in zero dependencies +- [ ] `scripts/smoke_wheel.py` runs on every CI push, not only on tags +- [ ] The release notes' pip instruction is now true + +**Scope** M + +--- + +## 07 — build(release): sign and notarise, or say plainly that we have not + +**Goal** Produce macOS and Windows binaries that run without a security warning — +and make it impossible for the release notes to claim that before it is true. + +**Why it matters** This is the largest single adoption blocker. A teacher on a +managed laptop who downloads the current binary gets Gatekeeper refusing to run +it on macOS and a SmartScreen warning on Windows. Telling a non-technical user to +run `xattr -d com.apple.quarantine`, as the release notes currently do, is asking +them to disable a security control in order to use a security tool. + +**Costs, stated up front.** macOS requires Apple Developer Program membership at +USD 99/year; a Developer ID Application certificate cannot be obtained any other +way, and an organisation enrolment needs a D-U-N-S number and takes weeks. +Notarisation additionally needs an App Store Connect API key. Windows requires an +OV or EV code-signing certificate — roughly USD 200–600/year, with EV requiring a +hardware token or a cloud HSM; Azure Trusted Signing is currently the cheapest +route at around USD 10/month but needs an Azure subscription and a verified +organisation identity. Even after signing, SmartScreen reputation accrues with +download volume, so early users may still see a warning. Linux has no signing +authority at all. **If the school or maintainer will not fund these, do the +Sigstore half of this commit and leave the honest warning exactly as blunt as it +is now.** + +**Files** +- `.github/workflows/release.yml` — signing steps in the macOS and Windows matrix legs, a Sigstore step for all three +- `scripts/build_binary.py` — read the signing state instead of hard-coding the "UNSIGNED" warning +- `teacher-safe-scan.spec` — `codesign_identity` and `entitlements_file` from the environment +- `docs/RELEASING.md` — new: key handling, rotation, what to do when a certificate expires +- `entitlements.plist` — new, minimal + +**Implementation** macOS: import the `.p12` into a temporary keychain, +`codesign --timestamp --options runtime --sign "Developer ID Application: …"`, +then `xcrun notarytool submit --wait` with the API key and `xcrun stapler staple`. +Note that a PyInstaller `--onefile` binary notarises awkwardly because the +bootloader extracts to a temp directory at run time; if notarisation fights it, +ship a `--onedir` build inside a signed `.app` in a signed `.dmg` and keep the +onefile CLI binary as the unsigned developer artifact. Windows: +`signtool sign /fd sha256 /tr /td sha256`. All three platforms: +`cosign sign-blob` with keyless OIDC, which costs nothing and gives a public +transparency-log entry for every artifact — do this even if nothing else here +gets funded. The important structural piece is that the release-notes body must +be generated from whether signing actually ran, not hand-written: a repository +variable gates both the signing steps and the wording, so an unsigned build +physically cannot ship notes claiming otherwise. `scripts/build_binary.py` reads +the same variable for its own printed warning. Do not claim reproducible builds; +PyInstaller output is not byte-reproducible and the SHA-256 in the release is a +download-integrity check, not a build attestation. + +**Tests** A workflow-level check that the published macOS artifact passes +`spctl -a -vvv -t install` and that the Windows artifact's signature verifies with +`signtool verify /pa`. A test that the release-notes template renders the +unsigned warning when the signing variable is unset. + +**Depends on:** 04 + +**Risk** High operational risk, low code risk. Secrets in CI: the `.p12` and its +password, the App Store Connect key, and the Windows certificate all become +repository secrets that can sign anything with this project's identity. Restrict +the signing job to a protected `release` environment with required reviewers. +Certificate expiry silently breaks releases a year later — put the expiry date in +`docs/RELEASING.md` and in a calendar. + +**Acceptance criteria** +- [ ] A signed macOS binary runs on a clean machine with no `xattr` incantation +- [ ] A signed Windows binary shows no SmartScreen block from a verified publisher +- [ ] Every artifact carries a Sigstore signature, signed or not +- [ ] With signing disabled, the release notes still carry the current blunt warning +- [ ] Signing secrets live in a protected environment with required reviewers +- [ ] `docs/RELEASING.md` records the yearly costs, the renewal dates and the rotation steps + +**Scope** L + +--- + +## 08 — docs(deploy): a route onto a managed laptop, and a working disclosure path + +**Goal** Give a school IT administrator the one page they need, and give a +security researcher somewhere to send a bypass. + +**Why it matters** `SAFETY.md` says "See SECURITY.md" and there is no +`SECURITY.md`. A triage tool with no disclosure channel invites a public issue +describing a bypass, which is the worst outcome for its users. Separately, the +adoption blocker is not "can this be downloaded" but "will the SOE let it run", +and the answer — allowlist it by the SHA-256 the release already publishes — is +not written down anywhere. + +**Files** +- `SECURITY.md` — new +- `docs/DEPLOYING.md` — new +- `scripts/windows_add_context_menu.reg` — fix +- `CONTRIBUTING.md` — remove the stale instructions +- `requirements-dev.txt` — reconcile with the `[dev]` extra + +**Implementation** `SECURITY.md`: private GitHub Security Advisories as the +channel, a stated response window, and an explicit scope — a detection **bypass** +(a file that should be flagged and is not, a way to make the report misrepresent +a file, a way to make a detector write to disk or execute anything) is in scope; +"it did not detect this novel malware sample" is not, because the README already +says a novel technique will come back clean. `docs/DEPLOYING.md`: the three +routes in order of how locked-down the machine is — pip for a machine with +Python, the signed binary, and hash-allowlisting for AppLocker or WDAC where the +published `.sha256` is the exact input the policy needs. The `.reg` file is +currently broken twice over: it invokes `TeacherSafeScanner.exe`, a name the build +has never produced, and it passes `"%1"` with no subcommand, so argparse's +required subparser rejects it with exit code 4 and the console window closes +before anyone reads why. Rewrite it to call `teacher-safe-scan.exe scan "%1"` +under `cmd /k` so the output stays visible, add a `Directory\shell` entry since +the unit of work is a folder, and use `REG_EXPAND_SZ` if `%ProgramFiles%` is +meant to expand. `CONTRIBUTING.md` still points at `requirements-optional.txt`, +deleted in the revival, and at `pip install -r requirements.txt`, which now +installs nothing because that file is a comment block; replace with +`pip install -e ".[dev]"` and the actual check commands from CI. +`requirements-dev.txt` lists `pyinstaller` while the `[dev]` extra does not — pick +one source of truth and make the other a pointer. + +**Tests** A docs link check in CI that fails on a relative Markdown link to a +file that does not exist. That single check would have caught the missing +`SECURITY.md`. + +**Depends on:** nothing + +**Risk** None to the code. The `.reg` change should be verified on a real Windows +machine; a registry file that half-works is worse than none. + +**Acceptance criteria** +- [ ] `SECURITY.md` exists, is linked from `SAFETY.md` and `README.md`, and states scope +- [ ] CI fails on any broken relative link in the Markdown +- [ ] The context-menu entry actually scans and leaves its output on screen +- [ ] `CONTRIBUTING.md`'s commands all work on a fresh clone +- [ ] `docs/DEPLOYING.md` covers pip, signed binary and hash-allowlisting + +**Scope** S + +--- + +## 09 — refactor(core): scan from an open handle, not only from a path + +**Goal** Split `scan_file` into a filesystem front half and a +`scan_stream(handle, …)` back half, so anything that can produce bytes can be +triaged. + +**Why it matters** Everything downstream needs it. An LMS export is one ZIP +containing thirty submissions; the product answer is thirty rows, and the +project's invariant is that no archive is ever extracted to a filesystem path. +Those two facts can only both hold if a submission can be scanned from an +in-memory handle. Rule packs need it too, because `_run_yara` currently matches +on a file path that a member inside an archive does not have. + +**Files** +- `scanner/scanner_core.py` — `scan_file`, new `scan_stream`, `_dispatch`, `ScanResult` +- `scanner/triage.py` — `build_summary` handles results with a container +- `tests/test_scanner_core.py` — stream-path coverage + +**Implementation** `scan_file` keeps what needs a filesystem: `analyze_name`, the +symlink branch, `stat`, the empty-file and size gates. Everything after the +`path.open("rb")` — `sha256_of`, `read_head`/`sniff_magic`, `analyze_content`, +`_dispatch`, `_dedupe`, `decide` — moves into +`scan_stream(handle, *, name, size, config, container=None)`. `_dispatch` takes +`path.suffix` and `path.name` only, so widen its parameter to `PurePath` and let +callers pass a `PurePosixPath` built from a member name. Add +`ScanResult.container: Optional[str]` recording which archive a result came from, +carried through `to_dict`/`from_dict`; leave submitter attribution to 11 so the +JSON schema changes once per concept rather than twice. `_run_yara` still takes a +path: for a stream result with no path, make it emit the existing +`inspection_incomplete` non-result rather than crash — commit 13 removes the +special case entirely. + +**Tests** `scan_stream` over an `io.BytesIO` of each corpus sample produces the +same findings and verdict as `scan_file` over the same bytes on disk — a +parametrised equivalence test across the whole `SAMPLES` table is the strongest +form of this and is cheap. Existing `test_scanner_core.py` tests must pass +untouched, which is the point of the split. + +**Depends on:** nothing + +**Risk** Moderate: this is the hot path and every detector runs through it. The +equivalence test over the whole corpus is what makes it safe. Watch for handle +position assumptions — several detectors `seek(0)` themselves +(`analyze_office`, `analyze_archive`, `read_head`) but `_appended_data` and +`_trailing_bytes` rely on `size` being passed correctly, which is now the +caller's job rather than a `stat` result. + +**Acceptance criteria** +- [ ] `scan_stream` and `scan_file` produce identical results for every corpus sample +- [ ] `scan_file` is a thin wrapper: filesystem gates, then delegate +- [ ] `ScanResult.container` round-trips through JSON +- [ ] No detector signature changes +- [ ] `scripts/check_corpus.py` passes unchanged + +**Scope** M + +--- + +## 10 — feat(ingest): read LMS bulk-export archives without extracting them + +**Goal** `teacher-safe-scan scan submissions.zip` produces one row per student +submission, not one row for the ZIP. + +**Why it matters** This is the wedge. The workflow that actually happens is: open +the LMS, click "Download all submissions", get a ZIP, and then have no idea what +is in it. Today that ZIP gets a single row saying it contains other archives. +Every existing competitor is worse at this, not better — Dangerzone cannot take a +folder at all, and oletools has no notion of a batch. Duplicate detection by +SHA-256 in `build_summary` also becomes genuinely useful here: two byte-identical +submissions from different students is a signal a teacher wants. + +**Files** +- `scanner/ingest.py` — new +- `scanner/scanner_core.py` — dispatch an export to the ingest walker +- `scanner/limits.py` — `max_member_scan_bytes` +- `SAFETY.md` — a row in the limits table +- `tests/test_ingest.py` — new + +**Implementation** `detect_layout(names) -> ExportLayout | None` votes over member +names and returns a layout with a confidence. The four shapes worth supporting: +**Canvas** — flat, `lastnamefirstname___.ext`, +with `_late_` inserted for late work, group submissions using the group name, and +resubmissions suffixed `-1`, `-2` before the extension. **Moodle** — one directory +per submission, `Firstname Lastname__assignsubmission_file_/`, with +an `_assignsubmission_onlinetext_` variant. **Blackboard** — +`__attempt__` plus a per-submission +`.txt` receipt that should be recognised and not reported as a submission. +**Google Classroom via Drive** — flat, `Student Name - Original Name.ext`. When no +layout wins, fall back to treating the archive as a plain archive, which is +today's behaviour. Members are read through `zipfile.ZipFile.open` into a bounded +`io.BytesIO` and handed to `scan_stream`; nothing is written to disk, and +`tests/test_detectors_archive.py::test_nothing_is_written_to_disk` gains a sibling +for this path. The existing `analyze_archive` checks still run on the container +first — an LMS export is still an archive and traversal or bomb findings on the +container matter more than any member. Add `max_member_scan_bytes` (32 MB) +rather than reusing `max_nested_extract_bytes` (16 MB), because a legitimate +video submission is larger than a legitimate nested archive; anything over it is +reported incomplete. + +**Never guess an attribution.** A layout match below the confidence threshold +yields an unattributed row, not a guessed student name. A wrong name on a report +that may be forwarded to a parent is far worse than a missing one. + +**Tests** A fixture builder per layout producing a synthetic export from the +existing corpus builders — a Canvas ZIP containing `office_with_macro.docm` under +a mangled Canvas name, and so on. Assert the member is flagged, the original +filename is recovered, and the row count equals the submission count. Assert a +plain `.zip` of coursework is still handled as a plain archive. Assert a +Blackboard receipt `.txt` is not counted as a submission. Assert a member over +`max_member_scan_bytes` becomes `COULD NOT FULLY INSPECT`. + +**Depends on:** 09 + +**Risk** The filename grammars are the fragile part — they vary by LMS version and +by institution configuration, and there is no specification for any of them. +Mitigate by keeping detection conservative, printing which layout was detected, +and making `--lms none` an escape hatch (commit 11). Second risk: a hostile export +whose member names are crafted to look like a different layout — attribution is +display metadata only and must never influence a verdict, which the design +already guarantees because attribution happens after `decide()`. + +**Acceptance criteria** +- [ ] Canvas, Moodle, Blackboard and Classroom-via-Drive exports each yield one row per submission +- [ ] Original filenames are recovered and shown alongside the mangled member name +- [ ] Nothing is written to disk during ingest, with a test asserting it +- [ ] A plain coursework `.zip` is unaffected +- [ ] Low-confidence layout matches produce unattributed rows, never guessed names +- [ ] `SAFETY.md`'s limits table lists `max_member_scan_bytes` + +**Scope** L + +--- + +## 11 — feat(cli): submitter attribution through triage and exit codes + +**Goal** Carry the student name from ingest to the console table, the summary and +the JSON report, and decide what quarantine means for a file that lives inside an +archive. + +**Why it matters** "Three of thirty submissions" is the product's sentence. +Making it "three students" is the difference between a list of filenames and an +answer a teacher can act on in the ten minutes before a lesson. + +**Files** +- `scanner/scanner_core.py` — `ScanResult.submitter` +- `scanner/triage.py` — `TriageSummary.by_submitter`, `headline()` +- `scanner/main.py` — `--lms`, `_quarantine_results`, console output +- `scanner/reporters.py` — `print_console_report` gains a submitter column when populated + +**Implementation** Add `submitter: Optional[str]` to `ScanResult` and round-trip +it. `build_summary` gains a `by_submitter` mapping and an `unattributed` count. +`headline()` grows an export-shaped variant — "4 of 29 submissions should not be +opened, from 3 students" — while leaving the existing folder wording untouched; +both are tested. The CLI gets +`--lms {auto,canvas,moodle,blackboard,classroom,none}`, defaulting to `auto`, and +prints one line naming the detected layout and submission count before the table. +Auto-detection that happens silently is wrong in a security tool: the operator +must be able to see that the tool decided something. + +Quarantine needs an explicit decision. A member inside an export has no +filesystem path, so `_quarantine_results` calling `store.quarantine(result.path)` +would fail per member. Quarantine the **container** once, with a reason listing +the member codes that triggered it, and print exactly that. Silently skipping +would be the worst option: the teacher would believe flagged files were moved +aside when they were not. + +**Tests** Console output for an export includes a submitter column and omits it +for a plain folder. Exit codes are unchanged by attribution. JSON round-trips +`submitter` and `container`. Quarantining an export moves the export once, not +zero times and not once per member, and says so. + +**Depends on:** 10 + +**Risk** Low, mostly presentation. The quarantine semantics are the judgement +call and should be stated in the README next to the existing "nothing is ever +deleted" guarantees. + +**Acceptance criteria** +- [ ] The console table shows a submitter column when attribution exists, and no empty column when it does not +- [ ] The headline counts students as well as files for an export +- [ ] `--lms none` reverts to plain-archive behaviour +- [ ] The detected layout is printed, never applied silently +- [ ] Quarantining an export moves the container once and reports which members caused it +- [ ] `submitter` and `container` survive a JSON round-trip + +**Scope** M + +--- + +## 12 — feat(report): group by student, and write the message to that student + +**Goal** Make the HTML report and the window answer "which students do I need to +talk to", and produce the paragraph a teacher sends to one of them. + +**Why it matters** `summarise_for_email` already produces a block for IT. The +other message a teacher has to write is to the student: "the file you submitted +could not be opened safely, please re-send it as X". Writing that thirty times is +the work the tool should remove. + +**Files** +- `scanner/reporters.py` — `generate_html_report`, `_result_html` +- `scanner/gui_model.py` — `message_for_submitter`, a `submitter` field on `RowView` +- `scanner/gui.py` — a Student column and a "Copy message for this student" button +- `tests/test_reporting_and_cli.py` — coverage for both + +**Implementation** The HTML report gains a "By student" section, worst student +first, above the existing per-verdict groups; a batch with no attribution renders +exactly as it does today. `_result_html`'s summary line shows the recovered +original filename as the label and the mangled member name as secondary metadata +— the Canvas member name is unreadable and should not be what a teacher scans +down a column. `message_for_submitter(summary, submitter)` produces a short plain +block covering only that student's files, with the finding's `action` text as the +instruction, and it must not name any other student. + +**A student name from an LMS filename is attacker-controlled input.** Every +submitter string goes through `sanitize_display` before it reaches HTML, the +console or a Tk widget, exactly as filenames already do — otherwise a student who +renames their submission with a bidi override reorders the report that is about +them. There is already a test for this shape +(`test_bidi_names_cannot_spoof_themselves_in_the_report`); extend it to +submitters. + +**Tests** Report with attribution contains a By-student section and every +submitter appears sanitised. Report without attribution is byte-identical to the +current output for the same summary. `message_for_submitter` mentions only that +student's files. The GUI's Student column is hidden for a plain folder scan. + +**Depends on:** 11, 02 + +**Risk** Low. Keep the no-JavaScript, no-network guarantee — the existing +`test_html_report_is_self_contained` test enforces it and must not be relaxed for +a collapsible By-student section; `
` is already how the report does +disclosure without script. + +**Acceptance criteria** +- [ ] The report groups by student, worst first, when attribution exists +- [ ] A folder scan's report is unchanged +- [ ] Submitter names are sanitised everywhere they are rendered +- [ ] `message_for_submitter` names one student's files and no others +- [ ] The report still contains no script and makes no network request + +**Scope** M + +--- + +## 13 — feat(rules): rule packs a school can add without touching the code + +**Goal** Let a school ship its own detections — as YARA rules where +`yara-python` is available, and as a stdlib-only literal-match pack where it is +not. + +**Why it matters** Every school has a local pattern: the phishing template that +circulates each September, a filename their SIS produces, a macro their own +finance office uses legitimately. Today the only extension point is +`--yara-rules `, which requires a compiler-backed dependency that will +not install on a locked-down machine, and which produces findings with no +teacher-facing text unless the rule author happened to write `meta:` fields. + +**Files** +- `scanner/rules.py` — new: `RulePack.load`, `RulePack.match` +- `scanner/scanner_core.py` — replace `_run_yara`, compile once per scan +- `scanner/main.py` — `--rules-dir`, `--yara-rules` kept as an alias +- `examples/rule_packs/example/` — a demonstrable pack +- `scripts/check_corpus.py` — `--rules-dir` mode +- `SAFETY.md` — what a rule pack can and cannot do + +**Implementation** A pack is a directory: `*.yar`/`*.yara` files plus a +`pack.json` manifest. JSON, not TOML, because `requires-python = ">=3.10"` and +`tomllib` arrives in 3.11. The manifest supplies the five human fields +(`plain`, `why`, `action`, `severity`, `confidence`) per rule name, so a rule +author can use stock YARA rules without editing them; YARA `meta:` values still +work and take precedence, as `_run_yara` already implements. **A rule missing +`plain`, `why` or `action` fails to load, loudly.** That is the same bar +`docs/ARCHITECTURE.md` sets for built-in detectors, and a finding a teacher +cannot read is worse than no finding. + +Two real defects get fixed here. `_run_yara` calls `yara.compile` **inside the +per-file path**, so scanning thirty files compiles the rules thirty times. Compile +once when `ScanConfig` is built. And it calls `rules.match(str(path))`, which +cannot work for a member scanned from a stream; switch to `rules.match(data=…)` +over bounded bytes, which is why this depends on 09. Also, a missing rules file +currently produces a `yara_no_rules` finding on *every* file — thirty identical +rows in the report; make a load failure one message on stderr and one batch-level +note. + +The stdlib fallback keeps `pack.json` entries with `literal` (ASCII or hex byte +strings, minimum four bytes) and `filename_glob`, matched against the same +bounded head/tail windows the detectors already use. Deliberately weak: no regex, +so no catastrophic backtracking on attacker-controlled input, and no code +execution of any kind. Say that in `SAFETY.md` — a rule pack is data, and a +plugin system that executed code would undo the guarantee the whole project rests +on. + +**Tests** Loading a pack with a rule missing `why` fails with a message naming the +rule. Rules compile once per scan, asserted by counting calls. A literal-match +pack fires on the corpus's known harmless payload string. A pack directory that +does not exist is one error, not one finding per file. Matching works for a +member scanned from a stream. `scripts/check_corpus.py --rules-dir +examples/rule_packs/example` passes. + +**Depends on:** 09 + +**Risk** Moderate. Rule severity is under the school's control, and a pack that +declares everything `high`/`high` will flood the DO NOT OPEN column under rule 2. +Cap what a pack may declare — `high`/`high` is allowed but the loader warns, and +`docs/SCORING.md` gains a paragraph explaining that a pack cannot change the four +rules, only add findings that feed them. + +**Acceptance criteria** +- [ ] `--rules-dir` loads a directory of YARA rules plus a `pack.json` +- [ ] Rules compile once per scan, not once per file +- [ ] Matching works on bytes, so members inside an LMS export are covered +- [ ] A rule without teacher-facing text refuses to load +- [ ] A stdlib-only pack works with `yara-python` absent +- [ ] `scripts/check_corpus.py --rules-dir` gates a school's own pack the way CI gates ours +- [ ] `SAFETY.md` states that packs are data and cannot execute anything + +**Scope** L + +--- + +## 14 — feat(detectors): RTF + +**Goal** Give `.rtf` submissions the same treatment `.docx` gets. + +**Why it matters** Verified: an RTF containing `\objdata` produces zero findings +and lands in `LIKELY SAFE TO REVIEW`. `sniff_magic` already recognises `{\rtf` +and `EXTENSION_EXPECTATIONS` maps `.rtf → {rtf}`, so the extension check passes +and `_dispatch` has no RTF branch — the file falls through everything. RTF is a +long-running carrier for embedded OLE and Equation Editor exploits precisely +because it is treated as a plain-text-ish format by tools that inspect `.docx` +carefully. + +**Files** +- `scanner/detectors/rtf.py` — new +- `scanner/detectors/__init__.py` — export `analyze_rtf` +- `scanner/scanner_core.py` — an RTF branch in `_dispatch` +- `scanner/corpus.py` — `rtf_embedded_object.rtf`, `clean_notes.rtf` +- `scripts/check_corpus.py` — two rows +- `README.md` — RTF in the Office section + +**Implementation** `analyze_rtf(handle, *, limits)` streams with the existing +`iter_windows`, with the overlap derived from the longest token exactly as +`pdf.py` does — that bug is documented in `pdf.py`'s docstring and must not be +reintroduced. Check for `\objdata` and `\objupdate` (an embedded object that +refreshes on open, `HIGH`/`MEDIUM`), `\objclass` naming an executable-ish class, +the `Equation.3` / `Equation Native` strings in the hex payload (reuse the +reasoning already written for `office_equation_object`, `MEDIUM`/`LOW`, since +maths coursework legitimately contains equations), `\dde` and `\ddeauto` +(`HIGH`/`MEDIUM`, matching `office_dde_field`), and an RTF whose `{\rtf` header is +not at byte 0 (`MEDIUM`, matching `pdf_header_offset`). Decode nothing and parse +nothing: scan bytes and report structure, like every other detector here. Note +that whitespace and comment groups can be interleaved inside a control word to +evade a naive literal search — normalise only by stripping whitespace within a +bounded window, and where a match is only reached after normalisation, report it +at lower confidence and say so in the evidence. + +**Tests** A clean RTF produces no findings — that assertion matters more than the +positive ones. `\objdata` fires. An obfuscated `\ob jdata` fires at lower +confidence. A `.rtf` whose contents are actually a PE is caught by the existing +`content_is_executable` path and not double-reported. + +**Depends on:** nothing + +**Risk** RTF's grammar is permissive and false positives are the failure mode +that gets the tool uninstalled. Keep confidence honest and make sure the clean +sample stays clean in `check_corpus.py`, which asserts both directions. + +**Acceptance criteria** +- [ ] An RTF with `\objdata` is `DO NOT OPEN` or `REVIEW WITH CAUTION`, never safe +- [ ] A clean RTF produces zero findings +- [ ] Detection streams, with overlap derived from the longest token +- [ ] Two corpus samples, one clean, one flagged +- [ ] README lists RTF among the document formats checked + +**Scope** M + +--- + +## 15 — feat(detectors): HTML, SVG and script-bearing markup + +**Goal** Check the submissions that are web pages, including the ones with a +`.svg` extension. + +**Why it matters** Verified: an `.svg` containing ` -""" - template = f""" - - + common = ( + '

What was found across the batch

' + '' + f"{rows}
SeverityFindingFiles
" + ) + + return f""" + -Teacher-Safe Scanner Report - - - + +File triage report — {_esc(summary.generated_at)} + +
-

Teacher-Safe Local File Scanner

-

This report is generated for defensive and educational purposes. If any file is flagged, do not open it on a production machine.

-

Consult your IT department or open it in an isolated, school-approved sandbox.

+
Teacher-Safe Local File Scanner + v{_esc(summary.scanner_version)}
+

{_esc(summary.headline())}

+

{summary.total_files} file(s), {_esc(_human(summary.total_bytes))} scanned in + {summary.duration_ms / 1000:.1f}s · {_esc(summary.generated_at)}

+

{_esc(", ".join(summary.roots))}

- - - - {summary_html} - -
PathSeverityScore
-{sections_html} -{script} - - -""" - return template +
{tiles}
+
+{"".join(sections)} +{common} +{dupes} +
+
+

What this report is. Every check here is static: files were + read, never opened or run. Findings are indicators, not antivirus verdicts. + “Likely safe” means nothing matched the checks this tool performs — + it is not a guarantee, and it is not a substitute for endpoint protection.

+

What to do with a red result. Do not open the file. Send this + report to your IT team along with the filename. Do not forward the file itself + by email.

+

This report contains no scripts and makes no network + requests. It is safe to store and to forward.

+
+""" + + +def _human(num: int) -> str: + value = float(num) + for unit in ("B", "KB", "MB", "GB"): + if value < 1024 or unit == "GB": + return f"{value:,.0f} {unit}" if unit == "B" else f"{value:,.1f} {unit}" + value /= 1024 + return f"{num} B" -def write_html_report(results: List[dict], destination: Path) -> None: - """Write a standalone HTML report to *destination*.""" - destination.write_text(generate_html_report(results), encoding="utf-8") +_CSS = """ +:root{ + --bg:#f6f7f9; --card:#fff; --ink:#16191d; --muted:#5c6672; --line:#e3e7ec; + --block:#b3261e; --caution:#9a6700; --unknown:#0b5cad; --safe:#1a7f37; + --block-bg:#fdeceb; --caution-bg:#fff8e6; --unknown-bg:#eaf3fc; --safe-bg:#eaf6ec; +} +@media (prefers-color-scheme: dark){ + :root{ --bg:#111418; --card:#191d23; --ink:#e8ecf1; --muted:#98a2ae; --line:#2a3138; + --block-bg:#33191a; --caution-bg:#332a12; --unknown-bg:#132638; --safe-bg:#12291a; + --block:#ff8b82; --caution:#e5b23c; --unknown:#71b7f7; --safe:#63c77f; } +} +*{box-sizing:border-box} +body{margin:0;background:var(--bg);color:var(--ink); + font:15px/1.55 -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif} +header{padding:32px 24px 20px;max-width:1080px;margin:0 auto} +.brand{font-size:12px;letter-spacing:.08em;text-transform:uppercase;color:var(--muted); + font-weight:600} +.brand span{opacity:.7} +h1{font-size:26px;line-height:1.3;margin:10px 0 6px;max-width:44ch} +.sub{color:var(--muted);font-size:13px;margin:2px 0} +.roots{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:12px;word-break:break-all} +.tiles{display:grid;grid-template-columns:repeat(auto-fit,minmax(140px,1fr));gap:12px; + max-width:1080px;margin:0 auto 24px;padding:0 24px} +.tile{background:var(--card);border:1px solid var(--line);border-radius:10px;padding:14px 16px; + border-left-width:4px} +.tile .n{font-size:28px;font-weight:700;line-height:1} +.tile .l{font-size:11px;letter-spacing:.06em;text-transform:uppercase;color:var(--muted); + margin-top:4px;font-weight:600} +.tile.v-block{border-left-color:var(--block)} .tile.v-block .n{color:var(--block)} +.tile.v-caution{border-left-color:var(--caution)} .tile.v-caution .n{color:var(--caution)} +.tile.v-unknown{border-left-color:var(--unknown)} .tile.v-unknown .n{color:var(--unknown)} +.tile.v-safe{border-left-color:var(--safe)} .tile.v-safe .n{color:var(--safe)} +.wrap{max-width:1080px;margin:0 auto;padding:0 24px 40px} +.group{margin-bottom:28px} +.group h2{font-size:14px;letter-spacing:.05em;text-transform:uppercase;color:var(--muted); + margin:0 0 10px;display:flex;align-items:center;gap:8px} +.group h2 .count{background:var(--line);border-radius:99px;padding:1px 9px;font-size:12px; + color:var(--ink)} +.group.v-block h2{color:var(--block)} .group.v-caution h2{color:var(--caution)} +.group.v-unknown h2{color:var(--unknown)} .group.v-safe h2{color:var(--safe)} +details.file{background:var(--card);border:1px solid var(--line);border-radius:10px; + margin-bottom:8px;overflow:hidden;border-left-width:4px} +details.file.v-block{border-left-color:var(--block)} +details.file.v-caution{border-left-color:var(--caution)} +details.file.v-unknown{border-left-color:var(--unknown)} +details.file.v-safe{border-left-color:var(--safe)} +summary{cursor:pointer;padding:12px 16px;display:flex;align-items:center;gap:12px;flex-wrap:wrap} +summary::-webkit-details-marker{display:none} +.verdict{font-size:11px;font-weight:700;letter-spacing:.04em;padding:3px 9px;border-radius:99px; + white-space:nowrap} +.verdict.v-block{background:var(--block-bg);color:var(--block)} +.verdict.v-caution{background:var(--caution-bg);color:var(--caution)} +.verdict.v-unknown{background:var(--unknown-bg);color:var(--unknown)} +.verdict.v-safe{background:var(--safe-bg);color:var(--safe)} +.fname{font-weight:600;word-break:break-all;flex:1;min-width:200px} +.fmeta{font-size:12px;color:var(--muted);font-variant-numeric:tabular-nums} +.file-body{padding:0 16px 16px;border-top:1px solid var(--line)} +.path{margin:12px 0} +.path code,.hash code{font-size:11.5px;color:var(--muted);word-break:break-all; + font-family:ui-monospace,SFMono-Regular,Menlo,monospace} +.rationale{background:var(--bg);border-radius:8px;padding:10px 14px;margin:10px 0} +.rationale span{font-size:11px;text-transform:uppercase;letter-spacing:.06em;color:var(--muted); + font-weight:600} +.rationale ul{margin:6px 0 0;padding-left:18px;font-size:13.5px} +ul.findings{list-style:none;margin:12px 0 0;padding:0} +li.finding{border:1px solid var(--line);border-radius:8px;padding:12px 14px;margin-bottom:8px; + border-left-width:3px} +li.finding.sev-high{border-left-color:var(--block)} +li.finding.sev-medium{border-left-color:var(--caution)} +li.finding.sev-low{border-left-color:var(--unknown)} +li.finding.sev-info{border-left-color:var(--line)} +.finding-head{display:flex;align-items:center;gap:8px;flex-wrap:wrap;margin-bottom:6px} +.sev-pill{font-size:10px;font-weight:700;text-transform:uppercase;letter-spacing:.05em; + padding:2px 7px;border-radius:4px} +.sev-pill.sev-high{background:var(--block-bg);color:var(--block)} +.sev-pill.sev-medium{background:var(--caution-bg);color:var(--caution)} +.sev-pill.sev-low{background:var(--unknown-bg);color:var(--unknown)} +.sev-pill.sev-info{background:var(--line);color:var(--muted)} +.conf{font-size:11px;color:var(--muted)} +.plain{margin:0 0 8px;font-size:14.5px} +.why,.action{margin:4px 0;font-size:13px;color:var(--muted)} +.why span,.action span{display:inline-block;font-size:10px;font-weight:700;text-transform:uppercase; + letter-spacing:.05em;color:var(--ink);opacity:.7;margin-right:6px} +.evidence{margin-top:8px;background:var(--bg);border-radius:6px;padding:7px 10px} +.evidence span{font-size:10px;text-transform:uppercase;letter-spacing:.06em;color:var(--muted); + font-weight:700;display:block;margin-bottom:2px} +.evidence code{font-size:11.5px;word-break:break-all; + font-family:ui-monospace,SFMono-Regular,Menlo,monospace} +.detector{font-size:10.5px;color:var(--muted);margin-top:8px;opacity:.75} +.error{color:var(--block);font-size:13px;font-weight:600} +.panel{background:var(--card);border:1px solid var(--line);border-radius:10px;padding:16px 18px; + margin-bottom:20px} +.panel h2{font-size:14px;margin:0 0 12px;letter-spacing:.04em;text-transform:uppercase; + color:var(--muted)} +.tbl{width:100%;border-collapse:collapse;font-size:13px} +.tbl th{text-align:left;font-size:10.5px;text-transform:uppercase;letter-spacing:.06em; + color:var(--muted);padding:0 10px 6px 0;border-bottom:1px solid var(--line)} +.tbl td{padding:7px 10px 7px 0;border-bottom:1px solid var(--line);vertical-align:top} +footer{max-width:1080px;margin:0 auto;padding:24px;border-top:1px solid var(--line); + color:var(--muted);font-size:13px} +footer p{margin:0 0 10px;max-width:78ch} +footer strong{color:var(--ink)} +.offline{font-size:12px;opacity:.8} +@media print{ + body{background:#fff} details.file{break-inside:avoid} details{open:true} + summary{cursor:default} .tiles{page-break-after:avoid} +} +""" + +__all__ = [ + "write_json_report", + "write_html_report", + "generate_html_report", + "print_console_report", + "sanitize_display", +] diff --git a/scanner/reporting/html_theme.css b/scanner/reporting/html_theme.css deleted file mode 100644 index 3067bc9..0000000 --- a/scanner/reporting/html_theme.css +++ /dev/null @@ -1,6 +0,0 @@ -.badge-low{background:#e0f2f1;color:#00695c;padding:2px 6px;border-radius:6px} -.badge-medium{background:#fff3e0;color:#e65100;padding:2px 6px;border-radius:6px} -.badge-high{background:#ffebee;color:#b71c1c;padding:2px 6px;border-radius:6px} -.details{margin:.25rem 0} -.summary-row{cursor:pointer} -.next-steps{margin-top:1rem;padding:.75rem;border:1px dashed #aaa;border-radius:8px} diff --git a/scanner/scanner_core.py b/scanner/scanner_core.py index e356d76..89005ab 100644 --- a/scanner/scanner_core.py +++ b/scanner/scanner_core.py @@ -1,238 +1,529 @@ -"""Core orchestration for the Teacher-Safe Local File Scanner.""" +"""Core orchestration: walk a target, run detectors, produce verdicts. + +Design notes worth knowing before changing anything here: + +* **Nothing is ever reported as safe by default.** A file that is too large, is + a symlink, cannot be opened, or raised an exception becomes + ``COULD_NOT_INSPECT``. The previous implementation returned ``severity="Safe"`` + for oversized and errored files, which is the most dangerous possible failure + mode for a triage tool. +* **Every file is opened exactly once** and the handle is passed to the + detectors, instead of each detector re-opening and re-reading the file. +* **Symlinks are not followed.** A submission containing a link to + ``/dev/urandom`` would otherwise hang the hasher forever. +* Detectors are dispatched on *sniffed content type first*, extension second, + so a renamed file is still analysed as what it really is. +""" from __future__ import annotations +import hashlib import logging +import os from concurrent.futures import ThreadPoolExecutor, as_completed -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, List, Optional +from typing import Any, BinaryIO, Callable, Dict, Iterator, List, Optional, Sequence -from . import detectors, heuristics, utils +from . import __version__ +from .detectors import archive as archive_detector +from .detectors import general as general_detector +from .detectors import image as image_detector +from .detectors import office as office_detector +from .detectors import pdf as pdf_detector +from .detectors.base import read_head, sniff_magic +from .findings import Confidence, Finding, Severity, Verdict +from .limits import DEFAULT_LIMITS, ScanLimits +from .verdict import VerdictResult, decide LOGGER = logging.getLogger(__name__) -try: # pragma: no cover - optional dependency - import yara -except ImportError: # pragma: no cover - yara = None +CHUNK_SIZE = 1024 * 1024 + +OFFICE_SUFFIXES = frozenset( + {".docx", ".xlsx", ".pptx", ".docm", ".xlsm", ".pptm", ".doc", ".xls", ".ppt"} +) +IMAGE_SUFFIXES = frozenset({".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp"}) +TEXTISH_SUFFIXES = frozenset( + {".txt", ".md", ".csv", ".html", ".htm", ".xml", ".json", ".log", ".srt", ""} +) + +#: Container formats this scanner has no detector for. +#: +#: Reporting these as LIKELY SAFE would break the project's central promise — +#: "unchecked is never clean" — in the most dangerous way possible, because a +#: .rar or .7z is exactly where someone puts something they do not want looked +#: at. Each of these needs a third-party library to open, which is a dependency +#: this project deliberately does not have. +UNINSPECTABLE_CONTAINERS: Dict[str, str] = { + "rar": "RAR archive", + "7z": "7-Zip archive", + "gzip": "gzip archive", + "bzip2": "bzip2 archive", + "xz": "xz archive", + "rtf": "RTF document", +} +UNINSPECTABLE_SUFFIXES: Dict[str, str] = { + ".rar": "RAR archive", + ".7z": "7-Zip archive", + ".gz": "gzip archive", + ".tgz": "gzip archive", + ".bz2": "bzip2 archive", + ".xz": "xz archive", + ".tar": "TAR archive", + ".rtf": "RTF document", + ".iso": "disc image", + ".dmg": "macOS disk image", + ".cab": "Windows cabinet archive", + ".arj": "ARJ archive", + ".ace": "ACE archive", + ".lzh": "LZH archive", + ".msg": "Outlook message", + ".eml": "email message", + ".one": "OneNote notebook", +} @dataclass class ScanConfig: - """Configuration for a scan operation.""" + """Everything that changes scanner behaviour, in one place.""" - max_file_size: int = 100 * 1024 * 1024 - use_magic: bool = False - use_yara: bool = False + limits: ScanLimits = field(default_factory=lambda: DEFAULT_LIMITS) threads: int = 4 + follow_symlinks: bool = False + use_yara: bool = False + yara_rules_path: Optional[Path] = None + + # Per-family switches kept for CLI compatibility: "off" | "normal" | "strict". pdf_rules: str = "normal" office_rules: str = "normal" zip_rules: str = "normal" image_rules: str = "normal" + @property + def max_file_size(self) -> int: + return self.limits.max_file_size + @dataclass class ScanResult: - """Result produced for each scanned file.""" + """One scanned file, its findings, and the verdict derived from them.""" path: Path - sha256: str size: int - magic_type: str - issues: List[Dict[str, str]] - score: int - severity: str - reasons: List[str] + sha256: str + detected_type: str + findings: List[Finding] + verdict: Verdict + risk_score: int + rationale: List[str] error: Optional[str] = None + duration_ms: int = 0 + + # -- convenience -------------------------------------------------- + @property + def needs_attention(self) -> bool: + return self.verdict in (Verdict.DO_NOT_OPEN, Verdict.REVIEW_WITH_CAUTION) - def to_dict(self) -> Dict[str, object]: - return { + @property + def top_finding(self) -> Optional[Finding]: + if not self.findings: + return None + return max( + self.findings, + key=lambda f: (f.severity.rank, f.confidence.rank), + ) + + def to_dict(self) -> Dict[str, Any]: + payload: Dict[str, Any] = { "path": str(self.path), - "sha256": self.sha256, + "name": self.path.name, "size": self.size, - "magic_type": self.magic_type, - "issues": self.issues, - "score": self.score, - "severity": self.severity, - "reasons": self.reasons, - **({"error": self.error} if self.error else {}), + "sha256": self.sha256, + "detected_type": self.detected_type, + "verdict": self.verdict.value, + "verdict_slug": self.verdict.slug, + "risk_score": self.risk_score, + "rationale": list(self.rationale), + "findings": [f.to_dict() for f in self.findings], + "duration_ms": self.duration_ms, } + if self.error: + payload["error"] = self.error + return payload + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "ScanResult": + verdict = next( + (v for v in Verdict if v.value == data.get("verdict")), + Verdict.COULD_NOT_INSPECT, + ) + return cls( + path=Path(str(data.get("path", ""))), + size=int(data.get("size", 0)), + sha256=str(data.get("sha256", "")), + detected_type=str(data.get("detected_type", "unknown")), + findings=[Finding.from_dict(f) for f in data.get("findings", [])], + verdict=verdict, + risk_score=int(data.get("risk_score", 0)), + rationale=list(data.get("rationale", [])), + error=data.get("error"), + duration_ms=int(data.get("duration_ms", 0)), + ) -YARA_RULES = """ -rule TeacherSafeSuspiciousStrings { - strings: - $mz = "This program cannot be run in DOS mode" - $powershell = "powershell" - condition: - any of them -} -""" +def _incomplete(code: str, plain: str, why: str, action: str, evidence: str) -> Finding: + return Finding( + code=code, + title=plain, + plain=plain, + why=why, + action=action, + severity=Severity.LOW, + confidence=Confidence.HIGH, + evidence=evidence, + detector="core", + inspection_incomplete=True, + ) -def _run_yara(path: Path) -> List[Dict[str, str]]: - if yara is None: - return [] - try: - rules = yara.compile(source=YARA_RULES) - matches = rules.match(str(path)) - findings: List[Dict[str, str]] = [] - for match in matches: - if match.strings: - evidence = ",".join(s[2] for s in match.strings) - else: - evidence = match.rule - findings.append( - { - "code": f"yara_{match.rule}", - "description": "YARA rule matched potential suspicious content", - "evidence": evidence, - } - ) - return findings - except Exception as exc: # pragma: no cover - optional path - LOGGER.warning("YARA scanning failed for %s: %s", path, exc) - return [] +def iter_targets(root: Path, *, follow_symlinks: bool = False) -> Iterator[Path]: + """Yield files beneath *root*, without following directory symlinks.""" + if root.is_file() or root.is_symlink(): + yield root + return + for dirpath, dirnames, filenames in os.walk(root, followlinks=follow_symlinks): + base = Path(dirpath) + if not follow_symlinks: + dirnames[:] = [d for d in dirnames if not (base / d).is_symlink()] + for filename in sorted(filenames): + yield base / filename -def _normalize_rule_findings(rule_findings: List[Dict]) -> List[Dict[str, str]]: - issues: List[Dict[str, str]] = [] - for finding in rule_findings: - rule = str(finding.get("rule", "rule")) - detail = str(finding.get("detail", "")) - issue: Dict[str, str] = { - "code": rule, - "description": detail, - } - severity = finding.get("severity") - if severity: - issue["severity"] = str(severity) - issues.append(issue) - return issues + +def sha256_of(handle: BinaryIO) -> str: + handle.seek(0) + digest = hashlib.sha256() + for chunk in iter(lambda: handle.read(CHUNK_SIZE), b""): + digest.update(chunk) + return digest.hexdigest() -def _deduplicate(findings: List[Dict[str, str]]) -> List[Dict[str, str]]: - seen: set[tuple[str | None, str | None, str | None]] = set() - unique: List[Dict[str, str]] = [] +def _dedupe(findings: Sequence[Finding], limit: int) -> List[Finding]: + seen: set = set() + out: List[Finding] = [] for finding in findings: - key = ( - finding.get("code"), - finding.get("evidence"), - finding.get("description"), - ) + key = finding.dedupe_key if key in seen: continue seen.add(key) - unique.append(finding) - return unique + out.append(finding) + if len(out) >= limit: + break + return out -def _collect_findings(path: Path, magic_type: str, config: ScanConfig) -> List[Dict[str, str]]: - findings: List[Dict[str, str]] = [] - maybe = detectors.detect_double_extension(path) - if maybe: - findings.append(maybe) - maybe = detectors.detect_pe_headers(path) - if maybe: - findings.append(maybe) +def scan_file(path: Path, config: ScanConfig) -> ScanResult: + """Inspect one file. Never raises; failures become COULD_NOT_INSPECT.""" + import time - suffix = path.suffix.lower() - if magic_type == "zip" or suffix in {".zip", ".docx", ".pptx", ".xlsx"}: - findings.extend(detectors.detect_zip_contents(path)) - if magic_type == "pdf" or suffix == ".pdf": - findings.extend(detectors.detect_pdf_risks(path)) - if suffix in {".png", ".jpg", ".jpeg"}: - maybe = detectors.detect_image_appended_data(path) - if maybe: - findings.append(maybe) - maybe = detectors.detect_office_macro(path) - if maybe: - findings.append(maybe) - findings.extend(detectors.extract_urls_and_flag(path)) - - if config.pdf_rules != "off" and (magic_type == "pdf" or suffix == ".pdf"): - try: - with path.open("rb") as handle: - findings.extend( - _normalize_rule_findings( - detectors.analyze_pdf(handle, strict=config.pdf_rules == "strict") - ) - ) - except OSError as exc: - LOGGER.warning("Unable to run PDF rules for %s: %s", path, exc) - if config.office_rules != "off" and suffix in {".docx", ".pptx", ".xlsx", ".docm", ".xlsm", ".pptm"}: + started = time.perf_counter() + limits = config.limits + + def finish( + findings: List[Finding], + *, + size: int = 0, + sha: str = "", + detected: str = "unknown", + error: Optional[str] = None, + ) -> ScanResult: + deduped = _dedupe(findings, limits.max_findings_per_file) + outcome: VerdictResult = decide(deduped, scan_error=error) + return ScanResult( + path=path, + size=size, + sha256=sha, + detected_type=detected, + findings=deduped, + verdict=outcome.verdict, + risk_score=outcome.risk_score, + rationale=outcome.rationale, + error=error, + duration_ms=int((time.perf_counter() - started) * 1000), + ) + + name_findings = general_detector.analyze_name(path) + + if path.is_symlink() and not config.follow_symlinks: try: - with path.open("rb") as handle: - findings.extend( - _normalize_rule_findings( - detectors.analyze_office(handle, strict=config.office_rules == "strict") - ) + target = os.readlink(path) + except OSError: + target = "?" + return finish( + name_findings + + [ + _incomplete( + "symlink_not_followed", + "This entry is a shortcut to another location, not a real file.", + "Following links from an untrusted folder can lead the scanner " + "to a device file or somewhere outside the folder you meant to " + "check, so links are listed but not followed.", + "Check what the link points at before doing anything with it.", + f"-> {target}", ) - except OSError as exc: - LOGGER.warning("Unable to run Office rules for %s: %s", path, exc) - if config.zip_rules != "off" and (magic_type == "zip" or suffix == ".zip"): - try: - with path.open("rb") as handle: - findings.extend( - _normalize_rule_findings( - detectors.analyze_zip(handle, strict=config.zip_rules == "strict") - ) + ], + detected="symlink", + ) + + try: + stat = path.stat() + except OSError as exc: + return finish(name_findings, error=f"cannot stat file: {exc}") + + if not path.is_file(): + return finish(name_findings, error="not a regular file", detected="special") + + size = stat.st_size + + if size == 0: + return finish( + name_findings + + [ + Finding( + code="file_empty", + title="File is empty", + plain="This file contains no data at all.", + why="An empty submission is a failed upload, not a threat.", + action="Ask the student to submit it again.", + severity=Severity.INFO, + confidence=Confidence.HIGH, + detector="core", ) - except OSError as exc: - LOGGER.warning("Unable to run ZIP rules for %s: %s", path, exc) - if config.image_rules != "off" and suffix in {".png", ".jpg", ".jpeg"}: - try: - with path.open("rb") as handle: - findings.extend( - _normalize_rule_findings( - detectors.analyze_image(handle, strict=config.image_rules == "strict") - ) + ], + size=0, + sha=hashlib.sha256(b"").hexdigest(), + detected="empty", + ) + + if size > limits.max_file_size: + return finish( + name_findings + + [ + _incomplete( + "file_too_large", + f"This file is {size / (1024 * 1024):,.0f} MB, larger than the " + "scanner's limit, so it was not examined.", + "A file that was not examined has not been cleared. Raising " + "--max-file-size will scan it, at the cost of more memory.", + "Either raise the size limit and re-scan, or treat this file as " + "unchecked.", + f"{size} bytes > {limits.max_file_size} limit", ) - except OSError as exc: - LOGGER.warning("Unable to run image rules for %s: %s", path, exc) + ], + size=size, + detected="unknown", + ) + + try: + with path.open("rb") as handle: + sha = sha256_of(handle) + head = read_head(handle, min(limits.head_bytes, 4096)) + detected = sniff_magic(head) + findings = list(name_findings) + findings.extend( + general_detector.analyze_content(handle, path, limits=limits, magic=detected) + ) + findings.extend( + _dispatch(handle, path, detected=detected, size=size, config=config) + ) + except (OSError, MemoryError) as exc: + return finish(name_findings, size=size, error=f"could not read file: {exc}") + except Exception as exc: # pragma: no cover - detector bug guard + LOGGER.exception("Detector raised on %s", path) + return finish(name_findings, size=size, error=f"scanner error: {exc!r}") + + return finish(findings, size=size, sha=sha, detected=detected) + + +def _dispatch( + handle: BinaryIO, path: Path, *, detected: str, size: int, config: ScanConfig +) -> List[Finding]: + """Route to format detectors by sniffed type first, extension second.""" + limits = config.limits + suffix = path.suffix.lower() + findings: List[Finding] = [] + + is_ooxml_name = suffix in OFFICE_SUFFIXES + is_zip_like = detected == "zip" + + if config.office_rules != "off" and (is_ooxml_name or detected == "ole"): + findings.extend(office_detector.analyze_office(handle, limits=limits, suffix=suffix)) + + # A .docx is a ZIP, but running the archive detector on it would flag every + # normal document. Only run archive analysis when this is a real archive. + if config.zip_rules != "off" and is_zip_like and not is_ooxml_name: + findings.extend( + archive_detector.analyze_archive(handle, limits=limits, display_name=path.name) + ) + elif config.zip_rules != "off" and is_ooxml_name and is_zip_like: + # Still worth checking an OOXML container for traversal and bombs, but + # not for "contains other archives" style findings. + findings.extend( + f + for f in archive_detector.analyze_archive( + handle, limits=limits, display_name=path.name, depth=1 + ) + if f.code + in { + "archive_path_traversal", + "archive_bomb_ratio", + "archive_bomb_total", + "archive_nul_in_name", + "archive_bidi_filename", + "archive_member_flood", + "archive_encrypted", + } + ) + + if config.pdf_rules != "off" and (detected == "pdf" or suffix == ".pdf"): + findings.extend(pdf_detector.analyze_pdf(handle, limits=limits, size=size)) + + if config.image_rules != "off" and ( + detected in {"png", "jpeg", "gif"} or suffix in IMAGE_SUFFIXES + ): + findings.extend( + image_detector.analyze_image(handle, limits=limits, size=size, suffix=suffix) + ) + + # A format with no detector must say so. See UNINSPECTABLE_CONTAINERS. + label = UNINSPECTABLE_CONTAINERS.get(detected) or UNINSPECTABLE_SUFFIXES.get(suffix) + if label and not findings: + findings.append( + _incomplete( + "container_not_inspectable", + f"This is a {label}, which this scanner cannot look inside.", + "Opening this format needs software this scanner deliberately does " + "not bundle, so nothing inside it has been checked. An archive " + "nobody can inspect is a common way to move a file past a scanner " + "— though it is also just a normal way to send a folder.", + "Ask the student to re-send the work as a ZIP, or as loose files. " + "If you must open it, do so on a machine you can afford to lose.", + f"{label} ({suffix or detected})", + ) + ) + + if detected in {"unknown", "xml", "script"} and suffix in TEXTISH_SUFFIXES: + findings.extend(general_detector.analyze_text_urls(handle, limits=limits, size=size)) if config.use_yara: - findings.extend(_run_yara(path)) - return _deduplicate(findings) + findings.extend(_run_yara(path, config)) + + return findings -def _scan_file(path: Path, config: ScanConfig) -> ScanResult: +def _run_yara(path: Path, config: ScanConfig) -> List[Finding]: + """Optional YARA pass. Absent or broken rules degrade to a stated non-result.""" try: - size = path.stat().st_size - except OSError as exc: - return ScanResult(path, "", 0, "unknown", [], 0, "Safe", [], error=str(exc)) + import yara + except ImportError: + return [ + _incomplete( + "yara_unavailable", + "YARA scanning was requested but the yara-python package is not " + "installed, so no rules were run.", + "The scan completed without the extra rule checks you asked for.", + "Install the optional extra with:\n" + " pip install 'teacher-safe-local-file-scanner[yara]'", + "yara-python missing", + ) + ] - if size > config.max_file_size: - return ScanResult( - path, - "", - size, - "unknown", - [{"code": "skipped_large", "description": "File skipped due to size"}], - 0, - "Safe", - ["skipped_large"], - ) + rules_path = config.yara_rules_path + try: + if rules_path and Path(rules_path).exists(): + rules = yara.compile(filepath=str(rules_path)) + else: + return [ + _incomplete( + "yara_no_rules", + "YARA scanning was requested but no rules file was supplied.", + "Without rules there is nothing for YARA to match.", + "Pass --yara-rules /path/to/rules.yar", + str(rules_path or ""), + ) + ] + matches = rules.match(str(path), timeout=30) + except Exception as exc: + return [ + _incomplete( + "yara_error", + "The YARA rules could not be run against this file.", + "The extra rule checks did not complete.", + "Check the rules file compiles with yarac.", + str(exc)[:200], + ) + ] - sha256 = utils.sha256_stream(path) - magic_type = utils.detect_magic_type(path, use_magic=config.use_magic) - findings = _collect_findings(path, magic_type, config) - score, severity, reasons = heuristics.calculate_score(findings) - return ScanResult(path, sha256, size, magic_type, findings, score, severity, reasons) + findings: List[Finding] = [] + for match in matches: + meta = getattr(match, "meta", {}) or {} + findings.append( + Finding( + code=f"yara_{match.rule}", + title=f"YARA rule matched: {match.rule}", + plain=str( + meta.get("description") + or f"A custom detection rule named '{match.rule}' matched this file." + ), + why=str( + meta.get("why") + or "This rule was supplied by your school or IT team; what it " + "means depends on the rule." + ), + action=str(meta.get("action") or "Follow your school's guidance for this rule."), + severity=Severity.parse(meta.get("severity"), Severity.MEDIUM), + confidence=Confidence.parse(meta.get("confidence"), Confidence.MEDIUM), + evidence=", ".join(sorted({str(t) for t in getattr(match, "tags", [])})) or None, + detector="yara", + ) + ) + return findings -def scan(root: Path, config: ScanConfig) -> List[ScanResult]: - """Scan *root* recursively and return a list of :class:`ScanResult`.""" - targets = list(utils.iter_directory_files(root)) - results: List[ScanResult] = [] +def scan( + root: Path, + config: ScanConfig, + *, + progress: Optional[Callable[[int, int, Path], None]] = None, +) -> List[ScanResult]: + """Scan *root* recursively. Results are sorted worst-first.""" + targets = list(iter_targets(root, follow_symlinks=config.follow_symlinks)) if not targets: - LOGGER.info("No files found for scanning in %s", root) - return results + LOGGER.info("No files found under %s", root) + return [] - with ThreadPoolExecutor(max_workers=config.threads) as executor: - future_map = {executor.submit(_scan_file, path, config): path for path in targets} - for future in as_completed(future_map): + results: List[ScanResult] = [] + total = len(targets) + with ThreadPoolExecutor(max_workers=max(1, config.threads)) as pool: + futures = {pool.submit(scan_file, path, config): path for path in targets} + for index, future in enumerate(as_completed(futures), start=1): result = future.result() results.append(result) - results.sort(key=lambda res: res.path) + if progress is not None: + progress(index, total, result.path) + + results.sort(key=lambda r: (-r.verdict.rank, -r.risk_score, str(r.path))) return results + + +def scanner_version() -> str: + return __version__ + + +__all__ = [ + "ScanConfig", + "ScanResult", + "scan", + "scan_file", + "iter_targets", + "scanner_version", +] diff --git a/scanner/triage.py b/scanner/triage.py new file mode 100644 index 0000000..1b0215b --- /dev/null +++ b/scanner/triage.py @@ -0,0 +1,203 @@ +"""Folder-level triage: the answer a teacher actually needs. + +The unit of work for a teacher is not one file, it is *a folder of thirty +submissions*, or the ZIP their LMS exported. The question is never "what is in +this file" — it is "which of these should I not open, and what do I tell the +student". + +:class:`TriageSummary` is that answer: counts by verdict, the worst files first, +and a one-line headline that can be read at a glance or pasted into an email. +""" +from __future__ import annotations + +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Sequence + +from .findings import Verdict +from .scanner_core import ScanResult + +#: Verdicts in the order a report should present them. +VERDICT_ORDER = [ + Verdict.DO_NOT_OPEN, + Verdict.REVIEW_WITH_CAUTION, + Verdict.COULD_NOT_INSPECT, + Verdict.LIKELY_SAFE, +] + + +@dataclass +class TriageSummary: + generated_at: str + scanner_version: str + roots: List[str] + total_files: int + total_bytes: int + counts: Dict[str, int] + duplicate_groups: List[Dict[str, Any]] + top_findings: List[Dict[str, Any]] + detectors_used: List[str] + duration_ms: int = 0 + results: List[ScanResult] = field(default_factory=list) + + # -- headline ------------------------------------------------------ + @property + def blocked(self) -> int: + return self.counts.get(Verdict.DO_NOT_OPEN.slug, 0) + + @property + def caution(self) -> int: + return self.counts.get(Verdict.REVIEW_WITH_CAUTION.slug, 0) + + @property + def unchecked(self) -> int: + return self.counts.get(Verdict.COULD_NOT_INSPECT.slug, 0) + + @property + def clear(self) -> int: + return self.counts.get(Verdict.LIKELY_SAFE.slug, 0) + + @property + def needs_attention(self) -> int: + return self.blocked + self.caution + self.unchecked + + def headline(self) -> str: + """One sentence, written for a human, safe to paste into an email.""" + if self.total_files == 0: + return "No files were found to check." + if self.blocked: + return ( + f"{self.blocked} of {self.total_files} files should not be opened. " + f"{self.caution} more need a closer look." + if self.caution + else f"{self.blocked} of {self.total_files} files should not be opened." + ) + if self.caution: + return ( + f"Nothing here is clearly dangerous, but {self.caution} of " + f"{self.total_files} files are worth a closer look before you open them." + ) + if self.unchecked: + return ( + f"{self.clear} of {self.total_files} files look fine. " + f"{self.unchecked} could not be fully checked — that is not the same " + "as safe." + ) + return ( + f"All {self.total_files} files passed the checks this scanner performs. " + "That is reassuring, not a guarantee." + ) + + def exit_code(self) -> int: + """0 clean · 1 caution/unchecked · 2 do-not-open · 3 scanner error.""" + if any(r.error for r in self.results): + return 3 + if self.blocked: + return 2 + if self.caution or self.unchecked: + return 1 + return 0 + + def to_dict(self) -> Dict[str, Any]: + return { + "generated_at": self.generated_at, + "scanner_version": self.scanner_version, + "roots": list(self.roots), + "headline": self.headline(), + "total_files": self.total_files, + "total_bytes": self.total_bytes, + "counts": dict(self.counts), + "duplicate_groups": list(self.duplicate_groups), + "top_findings": list(self.top_findings), + "detectors_used": list(self.detectors_used), + "duration_ms": self.duration_ms, + "files": [r.to_dict() for r in self.results], + } + + +def build_summary( + results: Sequence[ScanResult], + *, + roots: Sequence[Path], + scanner_version: str, + duration_ms: int = 0, +) -> TriageSummary: + counts: Counter[str] = Counter() + for result in results: + counts[result.verdict.slug] += 1 + + # Duplicate detection: identical content submitted under different names is + # both a plagiarism signal and a "you only need to look at this once" signal. + by_hash: Dict[str, List[ScanResult]] = defaultdict(list) + for result in results: + if result.sha256: + by_hash[result.sha256].append(result) + duplicates = [ + { + "sha256": digest, + "count": len(group), + "verdict": group[0].verdict.value, + "paths": [str(r.path) for r in group[:12]], + } + for digest, group in sorted(by_hash.items(), key=lambda kv: -len(kv[1])) + if len(group) > 1 + ][:20] + + finding_counter: Counter[str] = Counter() + finding_meta: Dict[str, Dict[str, Any]] = {} + for result in results: + for finding in result.findings: + finding_counter[finding.code] += 1 + finding_meta.setdefault( + finding.code, + { + "code": finding.code, + "title": finding.title, + "plain": finding.plain, + "severity": finding.severity.value, + "confidence": finding.confidence.value, + }, + ) + top_findings = [ + {**finding_meta[code], "files": count} + for code, count in finding_counter.most_common(12) + ] + + detectors = sorted({f.detector for r in results for f in r.findings}) + + return TriageSummary( + generated_at=datetime.now(timezone.utc).isoformat(timespec="seconds"), + scanner_version=scanner_version, + roots=[str(r) for r in roots], + total_files=len(results), + total_bytes=sum(r.size for r in results), + counts=dict(counts), + duplicate_groups=duplicates, + top_findings=top_findings, + detectors_used=detectors, + duration_ms=duration_ms, + results=list(results), + ) + + +def summary_from_dict(data: Dict[str, Any]) -> TriageSummary: + """Rebuild a summary from a JSON report so ``report`` can re-render it.""" + results = [ScanResult.from_dict(item) for item in data.get("files", [])] + return TriageSummary( + generated_at=str(data.get("generated_at", "")), + scanner_version=str(data.get("scanner_version", "unknown")), + roots=list(data.get("roots", [])), + total_files=int(data.get("total_files", len(results))), + total_bytes=int(data.get("total_bytes", 0)), + counts=dict(data.get("counts", {})), + duplicate_groups=list(data.get("duplicate_groups", [])), + top_findings=list(data.get("top_findings", [])), + detectors_used=list(data.get("detectors_used", [])), + duration_ms=int(data.get("duration_ms", 0)), + results=results, + ) + + +__all__ = ["TriageSummary", "build_summary", "summary_from_dict", "VERDICT_ORDER"] diff --git a/scanner/utils.py b/scanner/utils.py deleted file mode 100644 index cfdcf9f..0000000 --- a/scanner/utils.py +++ /dev/null @@ -1,141 +0,0 @@ -"""Utility helpers for the Teacher-Safe Local File Scanner. - -This module provides pure helper utilities shared across the project. The -functions defined here do **not** execute untrusted content and are limited to -reading metadata, hashes, and small sections of files. -""" -from __future__ import annotations - -import hashlib -import logging -import os -import shutil -from pathlib import Path -from typing import Iterator - -LOGGER = logging.getLogger(__name__) - - -CHUNK_SIZE = 1024 * 1024 - - -def sha256_stream(path: Path) -> str: - """Return the SHA256 digest of *path* using a streaming read.""" - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(CHUNK_SIZE), b""): - digest.update(chunk) - return digest.hexdigest() - - -def safe_read_head(path: Path, nbytes: int) -> bytes: - """Read up to *nbytes* bytes from the beginning of *path* safely.""" - try: - with path.open("rb") as handle: - return handle.read(nbytes) - except (OSError, IOError) as exc: # pragma: no cover - exercised indirectly - LOGGER.warning("Unable to read head of %s: %s", path, exc) - return b"" - - -def safe_read_tail(path: Path, nbytes: int) -> bytes: - """Read the last *nbytes* bytes of *path* without loading the file.""" - try: - size = path.stat().st_size - except OSError as exc: # pragma: no cover - exercised indirectly - LOGGER.warning("Unable to stat %s: %s", path, exc) - return b"" - if size == 0: - return b"" - offset = max(size - nbytes, 0) - try: - with path.open("rb") as handle: - handle.seek(offset) - return handle.read(nbytes) - except (OSError, IOError) as exc: # pragma: no cover - LOGGER.warning("Unable to read tail of %s: %s", path, exc) - return b"" - - -MAGIC_SIGNATURES = { - b"%PDF": "pdf", - b"PK\x03\x04": "zip", - b"PK\x05\x06": "zip", - b"PK\x07\x08": "zip", - b"\xFF\xD8\xFF": "jpeg", - b"\x89PNG\r\n\x1A\n": "png", - b"MZ": "pe", -} - - -try: # pragma: no cover - optional dependency - import magic -except ImportError: # pragma: no cover - no optional dep in tests - magic = None - - -def detect_magic_type(path: Path, *, use_magic: bool = False) -> str: - """Detect the file type using python-magic if enabled, otherwise magic bytes.""" - if use_magic and magic is not None: - try: - mime = magic.from_file(str(path), mime=True) - return mime or "unknown" - except Exception as exc: # pragma: no cover - optional path - LOGGER.debug("python-magic failed for %s: %s", path, exc) - head = safe_read_head(path, 16) - for signature, label in MAGIC_SIGNATURES.items(): - if head.startswith(signature): - return label - return "unknown" - - -def is_text_file(path: Path, *, max_bytes: int = 4096) -> bool: - """Heuristic to decide whether *path* appears to be text.""" - head = safe_read_head(path, max_bytes) - if not head: - return False - if b"\x00" in head: - return False - try: - head.decode("utf-8") - return True - except UnicodeDecodeError: - return False - - -def safe_list_zip_members(path: Path) -> list[str]: - """Return the member list of a zip archive, handling corruption gracefully.""" - import zipfile - - members: list[str] = [] - try: - with zipfile.ZipFile(path) as archive: - for info in archive.infolist(): - members.append(info.filename) - except zipfile.BadZipFile as exc: - LOGGER.warning("Corrupt ZIP %s: %s", path, exc) - except OSError as exc: - LOGGER.warning("Unable to open ZIP %s: %s", path, exc) - return members - - -def iter_directory_files(root: Path) -> Iterator[Path]: - """Yield files within *root* recursively.""" - if root.is_file(): - yield root - return - for dirpath, _, filenames in os.walk(root): - base = Path(dirpath) - for filename in filenames: - yield base / filename - - -def copy_file(src: Path, dest: Path) -> None: - """Copy *src* to *dest* using buffered IO.""" - with src.open("rb") as source, dest.open("wb") as target: - shutil.copyfileobj(source, target, length=CHUNK_SIZE) - - -def ensure_directory(path: Path) -> None: - """Ensure *path* exists as a directory.""" - path.mkdir(parents=True, exist_ok=True) diff --git a/scanner/verdict.py b/scanner/verdict.py new file mode 100644 index 0000000..2ab9b91 --- /dev/null +++ b/scanner/verdict.py @@ -0,0 +1,159 @@ +"""Turn a list of findings into a verdict a teacher can act on. + +The rules below are the entire severity model. They are stated in code and in +``docs/SCORING.md`` so that a school IT reviewer can disagree with them +specifically, rather than with an unexplained integer. + +Rules, applied in order: + +1. **Inspection completeness first.** If any finding is flagged + ``inspection_incomplete`` the file can never be reported as safe. It becomes + ``COULD_NOT_INSPECT`` unless something worse was also found. +2. **A single high-severity, high-or-medium-confidence finding is decisive.** + A ``vbaProject.bin`` inside a ``.docx`` is not a matter of degree. +3. **Corroboration promotes.** Two independent medium-severity findings, or a + high-severity finding that only reached low confidence, indicate caution. +4. **Weak signals never escalate on their own.** Any number of ``LOW``/``INFO`` + findings stays at caution at most, and a single low finding stays safe. + +The numeric ``risk_score`` exists purely to sort a folder worst-first. It is +derived from the same severity/confidence matrix and is capped **per finding +code**, so an archive with 900 ``.exe`` members cannot inflate past one with +one ``.exe``. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Dict, Iterable, List, Sequence + +from .findings import Confidence, Finding, Severity, Verdict + +#: Points contributed by one finding, before the per-code cap. +#: Severity picks the row, confidence scales it. Documented, not magic. +_BASE_POINTS: Dict[Severity, int] = { + Severity.INFO: 0, + Severity.LOW: 4, + Severity.MEDIUM: 18, + Severity.HIGH: 45, +} + +_CONFIDENCE_MULTIPLIER: Dict[Confidence, float] = { + Confidence.LOW: 0.4, + Confidence.MEDIUM: 0.8, + Confidence.HIGH: 1.0, +} + +#: A given finding code may contribute at most this multiple of its single-hit +#: value, no matter how many times it fires. +_PER_CODE_CAP_FACTOR = 2.0 + + +@dataclass(frozen=True) +class VerdictResult: + verdict: Verdict + risk_score: int + rationale: List[str] + + def to_dict(self) -> dict: + return { + "verdict": self.verdict.value, + "verdict_slug": self.verdict.slug, + "risk_score": self.risk_score, + "rationale": list(self.rationale), + } + + +def finding_points(finding: Finding) -> float: + return _BASE_POINTS[finding.severity] * _CONFIDENCE_MULTIPLIER[finding.confidence] + + +def risk_score(findings: Sequence[Finding]) -> int: + """Sorting weight in 0..100, capped per finding code.""" + per_code: Dict[str, float] = {} + caps: Dict[str, float] = {} + for finding in findings: + points = finding_points(finding) + per_code[finding.code] = per_code.get(finding.code, 0.0) + points + caps[finding.code] = max(caps.get(finding.code, 0.0), points * _PER_CODE_CAP_FACTOR) + total = sum(min(value, caps[code]) for code, value in per_code.items()) + return int(min(round(total), 100)) + + +def decide(findings: Iterable[Finding], *, scan_error: str | None = None) -> VerdictResult: + """Apply the four rules above and explain which one fired.""" + items: List[Finding] = list(findings) + rationale: List[str] = [] + + incomplete = [f for f in items if f.inspection_incomplete] + decisive = [ + f + for f in items + if f.severity is Severity.HIGH and f.confidence in (Confidence.HIGH, Confidence.MEDIUM) + ] + mediums = [f for f in items if f.severity is Severity.MEDIUM] + weak_high = [f for f in items if f.severity is Severity.HIGH and f.confidence is Confidence.LOW] + lows = [f for f in items if f.severity is Severity.LOW] + + score = risk_score(items) + + if decisive: + rationale.append( + "Rule 2: {n} high-severity finding(s) with usable confidence ({codes}).".format( + n=len(decisive), codes=", ".join(sorted({f.code for f in decisive})) + ) + ) + if incomplete: + rationale.append( + "Note: inspection was also incomplete, so there may be more than is listed." + ) + return VerdictResult(Verdict.DO_NOT_OPEN, score, rationale) + + caution = len(mediums) >= 2 or bool(weak_high) or len(lows) >= 3 or len(mediums) == 1 + if caution: + if len(mediums) >= 2: + rationale.append( + "Rule 3: {n} independent medium-severity findings corroborate each other.".format( + n=len(mediums) + ) + ) + elif weak_high: + rationale.append( + "Rule 3: a high-severity pattern matched but only at low confidence " + "({codes}) — worth a human look, not a firm verdict.".format( + codes=", ".join(sorted({f.code for f in weak_high})) + ) + ) + elif len(mediums) == 1: + rationale.append( + "Rule 3: one medium-severity finding ({code}).".format(code=mediums[0].code) + ) + else: + rationale.append( + "Rule 4: {n} low-severity signals accumulated; none is conclusive.".format( + n=len(lows) + ) + ) + if incomplete: + rationale.append("Inspection was incomplete; treat the result as a lower bound.") + return VerdictResult(Verdict.REVIEW_WITH_CAUTION, score, rationale) + + if incomplete or scan_error: + reason = scan_error or ", ".join(sorted({f.code for f in incomplete})) + rationale.append( + "Rule 1: the file could not be fully inspected ({reason}). " + "Absence of findings here is not evidence of safety.".format(reason=reason) + ) + return VerdictResult(Verdict.COULD_NOT_INSPECT, score, rationale) + + if lows: + rationale.append( + "Rule 4: only {n} weak signal(s) found; not enough to warrant caution.".format( + n=len(lows) + ) + ) + else: + rationale.append("No risk indicators matched in the checks this scanner performs.") + return VerdictResult(Verdict.LIKELY_SAFE, score, rationale) + + +__all__ = ["VerdictResult", "decide", "risk_score", "finding_points"] diff --git a/scripts/build_binary.py b/scripts/build_binary.py new file mode 100755 index 0000000..a43891c --- /dev/null +++ b/scripts/build_binary.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Build a standalone executable and write a checksum next to it. + +Usage: + pip install -e ".[build]" + python scripts/build_binary.py + +The checksum is the point: a security tool distributed as a binary that nobody +can verify is a security problem, not a security product. +""" +from __future__ import annotations + +import hashlib +import platform +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + + +def main() -> int: + if shutil.which("pyinstaller") is None: + print("pyinstaller not found. Install it with: pip install -e '.[build]'", file=sys.stderr) + return 1 + + print("Building…") + result = subprocess.run( + ["pyinstaller", "--noconfirm", "--clean", "teacher-safe-scan.spec"], + cwd=ROOT, + ) + if result.returncode != 0: + return result.returncode + + suffix = ".exe" if platform.system() == "Windows" else "" + binary = ROOT / "dist" / f"teacher-safe-scan{suffix}" + if not binary.exists(): + print(f"expected {binary} but it was not produced", file=sys.stderr) + return 1 + + digest = hashlib.sha256(binary.read_bytes()).hexdigest() + label = f"{platform.system().lower()}-{platform.machine().lower()}" + checksum_file = binary.with_name(f"{binary.name}.sha256") + checksum_file.write_text(f"{digest} {binary.name}\n", encoding="utf-8") + + size_mb = binary.stat().st_size / (1024 * 1024) + print(f"\n {binary} ({size_mb:.1f} MB, {label})") + print(f" sha256 {digest}") + print(f" written to {checksum_file}") + print( + "\n This binary is UNSIGNED. macOS Gatekeeper will block it until it is\n" + " signed and notarised with an Apple Developer ID; Windows SmartScreen\n" + " will warn. Do not describe it as signed in release notes." + ) + + print("\n Smoke test:") + smoke = subprocess.run([str(binary), "--version"], capture_output=True, text=True) + print(" ", smoke.stdout.strip() or smoke.stderr.strip()) + return smoke.returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/build_pyinstaller.ps1 b/scripts/build_pyinstaller.ps1 deleted file mode 100644 index 45eb4bf..0000000 --- a/scripts/build_pyinstaller.ps1 +++ /dev/null @@ -1,8 +0,0 @@ -param([string]$Entry="scanner/gui.py", [string]$Name="TeacherSafeScanner") - -pyinstaller --noconfirm --clean ` - --name $Name ` - --onefile ` - --windowed ` - --add-data "scanner/reporting/html_theme.css;scanner/reporting" ` - $Entry diff --git a/scripts/build_pyinstaller.sh b/scripts/build_pyinstaller.sh deleted file mode 100755 index 0fd1c21..0000000 --- a/scripts/build_pyinstaller.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -ENTRY="${1:-scanner/gui.py}" -NAME="${2:-TeacherSafeScanner}" -pyinstaller --noconfirm --clean \ - --name "$NAME" \ - --onefile \ - --windowed \ - --add-data "scanner/reporting/html_theme.css:scanner/reporting" \ - "$ENTRY" diff --git a/scripts/check_corpus.py b/scripts/check_corpus.py new file mode 100755 index 0000000..e469819 --- /dev/null +++ b/scripts/check_corpus.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Assert that every benign sample lands in the verdict it is meant to. + +This is the project's detection regression test. If a detector is weakened, a +sample silently drops to LIKELY SAFE and this fails loudly. If a detector becomes +noisy, a clean sample stops being clean and this fails too — which is the failure +that actually matters, because a triage tool that cries wolf gets uninstalled. +""" +from __future__ import annotations + +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) + +from examples.generate_benign_samples import write_samples # noqa: E402 +from scanner.findings import Verdict # noqa: E402 +from scanner.limits import DEFAULT_LIMITS # noqa: E402 +from scanner.scanner_core import ScanConfig, scan # noqa: E402 + +BLOCK = Verdict.DO_NOT_OPEN +CAUTION = Verdict.REVIEW_WITH_CAUTION +UNKNOWN = Verdict.COULD_NOT_INSPECT +SAFE = Verdict.LIKELY_SAFE + +#: filename -> (expected verdict, a finding code that must be present) +EXPECTED: dict[str, tuple[Verdict, str | None]] = { + "README.md": (SAFE, None), + "clean_essay.txt": (SAFE, None), + "clean_report.docx": (SAFE, None), + "clean_diagram.png": (SAFE, None), + "clean_photo.jpg": (SAFE, None), + "clean_worksheet.pdf": (SAFE, None), + "clean_homework.zip": (SAFE, None), + "empty_submission.docx": (SAFE, "file_empty"), + "archive_nested_deep.zip": (SAFE, "archive_nested"), + "archive_path_traversal.zip": (BLOCK, "archive_path_traversal"), + "archive_with_program.zip": (BLOCK, "archive_executable_member"), + "archive_double_extension.zip": (BLOCK, "archive_double_extension"), + "archive_zip_bomb_shape.zip": (BLOCK, "archive_bomb_ratio"), + "office_with_macro.docm": (BLOCK, "office_macro_present"), + "office_remote_template.docx": (BLOCK, "office_external_attachedtemplate"), + "office_dde_field.docx": (BLOCK, "office_dde_field"), + "office_embedded_object.docx": (CAUTION, "office_embedded_object"), + "office_renamed_program.docx": (BLOCK, "content_is_executable"), + "pdf_javascript.pdf": (BLOCK, "pdf_javascript"), + "pdf_launch_action.pdf": (BLOCK, "pdf_launch_action"), + "pdf_appended_payload.pdf": (CAUTION, "pdf_appended_data"), + "image_polyglot.png": (BLOCK, "image_polyglot"), + "image_large_appended.jpg": (BLOCK, "image_large_appended_data"), + "image_is_really_a_program.jpg": (BLOCK, "content_is_executable"), + "Assignment.pdf.exe": (BLOCK, "name_double_extension"), + "invoice‮gpj.exe": (BLOCK, "name_bidi_override"), + "links_suspicious.txt": (CAUTION, "url_punycode_host"), + "archive_password_protected.zip": (CAUTION, "archive_encrypted"), + "broken_upload.zip": (UNKNOWN, "archive_corrupt"), + "coursework.7z": (UNKNOWN, "container_not_inspectable"), + "essay.rtf": (UNKNOWN, "container_not_inspectable"), +} + + +def main() -> int: + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "samples" + write_samples(out) + results = scan(out, ScanConfig(limits=DEFAULT_LIMITS, threads=4)) + + by_name = {r.path.name: r for r in results} + failures: list[str] = [] + + unexpected = set(by_name) - set(EXPECTED) + if unexpected: + failures.append(f"corpus has files with no expectation recorded: {sorted(unexpected)}") + missing = set(EXPECTED) - set(by_name) + if missing: + failures.append(f"expected samples were not produced: {sorted(missing)}") + + for name, (verdict, code) in EXPECTED.items(): + result = by_name.get(name) + if result is None: + continue + if result.verdict is not verdict: + failures.append( + f"{name}: expected {verdict.name}, got {result.verdict.name} " + f"(findings: {sorted(f.code for f in result.findings)})" + ) + if code and code not in {f.code for f in result.findings}: + failures.append( + f"{name}: expected finding {code!r}, got " + f"{sorted(f.code for f in result.findings)}" + ) + + if failures: + print("DETECTION REGRESSION\n") + for line in failures: + print(f" ✗ {line}") + return 1 + + blocked = sum(1 for v, _ in EXPECTED.values() if v is BLOCK) + clean = sum(1 for v, _ in EXPECTED.values() if v is SAFE) + print( + f"corpus OK: {len(EXPECTED)} samples — {blocked} correctly blocked, " + f"{clean} correctly clean, no false positives." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 060fb67..0000000 --- a/setup.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[metadata] -license_file = LICENSE - -[mypy] -python_version = 3.10 -ignore_missing_imports = True -warn_unused_ignores = True -warn_return_any = True diff --git a/teacher-safe-scan.spec b/teacher-safe-scan.spec new file mode 100644 index 0000000..d902471 --- /dev/null +++ b/teacher-safe-scan.spec @@ -0,0 +1,78 @@ +# -*- mode: python ; coding: utf-8 -*- +"""PyInstaller spec for a standalone Teacher-Safe Scanner. + +The target user is a teacher on a managed school laptop where Python may not be +installed and `pip` may be blocked by policy. A single downloadable executable is +therefore not a nicety — it is the difference between adoption and zero. + +Build: + pip install -e ".[build]" + pyinstaller teacher-safe-scan.spec + +Produces `dist/teacher-safe-scan` (or `.exe` on Windows). One file, no installer. + +NOTE ON SIGNING: the binaries this spec produces are UNSIGNED. On macOS, +Gatekeeper will refuse to run them until they are signed and notarised with an +Apple Developer ID; on Windows, SmartScreen will warn. Do not claim otherwise in +release notes. +""" +import sys + +block_cipher = None + +# tkinter is optional at runtime: the CLI works without it and prints installation +# guidance if the window is requested. It is bundled when available so the frozen +# build has a GUI. +hidden = ["scanner.gui", "scanner.gui_model"] +try: + import tkinter # noqa: F401 +except ImportError: + excluded_gui = ["tkinter", "tkinter.ttk", "tkinter.filedialog", "tkinter.messagebox"] + hidden = ["scanner.gui_model"] +else: + excluded_gui = [] + +a = Analysis( + ["scanner/__main__.py"], + pathex=["."], + binaries=[], + datas=[("examples/generate_benign_samples.py", "examples")], + hiddenimports=hidden, + hookspath=[], + runtime_hooks=[], + # Nothing here needs numpy/scipy/PIL; excluding them keeps the binary small + # enough to download over a school network. + excludes=[ + "numpy", "scipy", "PIL", "matplotlib", "pandas", "setuptools", "pip", + "pytest", "IPython", "test", "unittest", + ] + excluded_gui, + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False, +) + +pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) + +exe = EXE( + pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + [], + name="teacher-safe-scan", + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=False, # UPX-packed binaries trip antivirus heuristics, which is + # a bad look for a security tool. + upx_exclude=[], + runtime_tmpdir=None, + console=True, # the CLI is the primary interface; `gui` opens a window + disable_windowed_traceback=False, + argv_emulation=sys.platform == "darwin", + target_arch=None, + codesign_identity=None, + entitlements_file=None, +) diff --git a/tests/conftest.py b/tests/conftest.py index 6938c11..c80b15f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,16 +1,36 @@ +"""Pytest fixtures. + +Sample builders live in :mod:`tests.samples`. Every sample is harmless: the +corpus reproduces the *structure* of a risky file using inert payloads, so the +detectors can be exercised without malware ever entering this repository. +""" +from __future__ import annotations + import sys from pathlib import Path import pytest -ROOT = Path(__file__).resolve().parent.parent -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from scanner.limits import DEFAULT_LIMITS, ScanLimits # noqa: E402 +from tests.samples import HARMLESS, make_docx, make_png, make_zip # noqa: E402 + +@pytest.fixture() +def limits() -> ScanLimits: + return DEFAULT_LIMITS -@pytest.fixture(scope="session", autouse=True) -def generate_examples() -> None: - """Ensure benign sample files are materialised before tests run.""" - from examples.generate_benign_samples import main as generate - generate() +@pytest.fixture() +def corpus(tmp_path: Path) -> Path: + """A small folder of submissions: three clean, three flagged.""" + root = tmp_path / "submissions" + root.mkdir() + (root / "clean_essay.txt").write_bytes(b"An essay about rivers.\n") + (root / "clean_report.docx").write_bytes(make_docx()) + (root / "clean_photo.png").write_bytes(make_png()) + (root / "macro.docm").write_bytes(make_docx({"word/vbaProject.bin": b"\x00" * 64})) + (root / "traversal.zip").write_bytes(make_zip([("../../escape.txt", HARMLESS)])) + (root / "Assignment.pdf.exe").write_bytes(b"MZ" + HARMLESS) + return root diff --git a/tests/samples.py b/tests/samples.py new file mode 100644 index 0000000..442d6c7 --- /dev/null +++ b/tests/samples.py @@ -0,0 +1,108 @@ +"""Harmless sample builders shared by the test suite. + +Every sample built here is inert. The corpus reproduces the *structure* of a +risky file — an archive member that escapes its folder, a PNG with a ZIP glued +to the end, a document with a remote-template relationship — using payloads like +``echo "harmless"``. No malware ever enters this repository. +""" +from __future__ import annotations + +import io +import struct +import zipfile +import zlib +from typing import Dict, Iterable, List, Tuple + +HARMLESS = b'echo "harmless test payload"\n' + +CONTENT_TYPES = ( + '' +) +ROOT_RELS = ( + '' + '' +) +DOC_XML = ( + '' + "hello" +) +EXTERNAL_TEMPLATE_RELS = ( + '' + '' +) + + +def make_zip( + entries: Iterable[Tuple[str, bytes]], compress: int = zipfile.ZIP_DEFLATED +) -> bytes: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", compress) as archive: + for name, data in entries: + archive.writestr(name, data) + return buf.getvalue() + + +def make_docx(extra: Dict[str, bytes] | None = None, body: str = DOC_XML) -> bytes: + entries: List[Tuple[str, bytes]] = [ + ("[Content_Types].xml", CONTENT_TYPES.encode()), + ("_rels/.rels", ROOT_RELS.encode()), + ("word/document.xml", body.encode()), + ("word/settings.xml", b""), + ] + entries.extend((k, v) for k, v in (extra or {}).items()) + return make_zip(entries, zipfile.ZIP_STORED) + + +def make_png(trailer: bytes = b"", chunks: bytes = b"") -> bytes: + out = b"\x89PNG\r\n\x1a\n" + ihdr = struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0) + out += ( + struct.pack(">I", len(ihdr)) + + b"IHDR" + + ihdr + + struct.pack(">I", zlib.crc32(b"IHDR" + ihdr)) + ) + out += chunks + out += struct.pack(">I", 0) + b"IEND" + struct.pack(">I", zlib.crc32(b"IEND")) + return out + trailer + + +def make_jpeg(trailer: bytes = b"") -> bytes: + return b"\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01" + b"\x00" * 64 + b"\xff\xd9" + trailer + + +def make_pdf(extra_catalog: str = "", trailer_extra: bytes = b"") -> bytes: + return ( + "%PDF-1.5\n" + f"1 0 obj\n<< /Type /Catalog {extra_catalog}>>\nendobj\n" + "trailer\n<< /Root 1 0 R >>\n%%EOF\n" + ).encode() + trailer_extra + + +def encrypted_flag_zip() -> bytes: + """A ZIP whose entry is *marked* encrypted. + + The standard library cannot write encrypted entries, so general-purpose bit + 0 is set in both the local header and the central directory. The detector + reads the flag, not the payload, so this exercises the real code path. + """ + raw = bytearray(make_zip([("secret.txt", HARMLESS)], zipfile.ZIP_STORED)) + local = raw.find(b"PK\x03\x04") + central = raw.find(b"PK\x01\x02") + if local >= 0: + raw[local + 6] |= 0x01 + if central >= 0: + raw[central + 8] |= 0x01 + return bytes(raw) + + +def codes(findings) -> set: + """Set of finding codes — the usual assertion target.""" + return {f.code for f in findings} diff --git a/tests/test_detectors.py b/tests/test_detectors.py deleted file mode 100644 index 51769a6..0000000 --- a/tests/test_detectors.py +++ /dev/null @@ -1,36 +0,0 @@ -from scanner import detectors - - -def test_detect_double_extension(tmp_path): - path = tmp_path / "essay.pdf.exe" - path.write_bytes(b"MZ") - issue = detectors.detect_double_extension(path) - assert issue and issue["code"] == "double_extension" - - -def test_detect_pdf_risks(tmp_path): - path = tmp_path / "test.pdf" - path.write_bytes(b"%PDF-1.4 /JavaScript") - findings = detectors.detect_pdf_risks(path) - assert any(f["code"] == "pdf_token" for f in findings) - - -def test_detect_zip_contents_flags_macro(tmp_path): - import zipfile - - path = tmp_path / "doc.zip" - with zipfile.ZipFile(path, "w") as zf: - zf.writestr("word/vbaProject.bin", b"dummy") - zf.writestr("payload.exe", b"MZ") - findings = detectors.detect_zip_contents(path) - codes = {finding["code"] for finding in findings} - assert "exe_in_zip" in codes - assert "zip_vba_project" in codes - - -def test_extract_urls_and_flag(tmp_path): - path = tmp_path / "notes.txt" - path.write_text("Visit http://xn--example.com for details", encoding="utf-8") - findings = detectors.extract_urls_and_flag(path) - assert findings - assert findings[0]["code"] in {"url", "url_suspicious"} diff --git a/tests/test_detectors_archive.py b/tests/test_detectors_archive.py new file mode 100644 index 0000000..0c6484c --- /dev/null +++ b/tests/test_detectors_archive.py @@ -0,0 +1,119 @@ +"""Archive detector: traversal, bombs, encryption, deception, bounded recursion.""" +from __future__ import annotations + +import io + +import pytest + +from scanner.detectors.archive import analyze_archive +from scanner.limits import ScanLimits +from tests.samples import HARMLESS, codes, encrypted_flag_zip, make_zip + + +def run(data: bytes, limits: ScanLimits, **kwargs): + return analyze_archive(io.BytesIO(data), limits=limits, **kwargs) + + +def test_ordinary_archive_is_clean(limits): + data = make_zip([("essay.txt", b"words"), ("notes/refs.txt", b"more")]) + assert codes(run(data, limits)) == set() + + +@pytest.mark.parametrize( + "member", + ["../../evil.txt", "/etc/passwd", "..\\..\\evil.txt", "a/../../b.txt"], +) +def test_path_traversal_is_detected(limits, member): + assert "archive_path_traversal" in codes(run(make_zip([(member, HARMLESS)]), limits)) + + +def test_relative_paths_that_stay_inside_are_not_flagged(limits): + data = make_zip([("a/../b.txt", b"x"), ("./c.txt", b"y")]) + assert "archive_path_traversal" not in codes(run(data, limits)) + + +def test_executable_member_is_high_severity(limits): + findings = run(make_zip([("setup.exe", b"MZ")]), limits) + match = next(f for f in findings if f.code == "archive_executable_member") + assert match.severity.value == "high" + assert match.action # a teacher is told what to do + assert match.why + + +def test_double_extension_member(limits): + assert "archive_double_extension" in codes(run(make_zip([("essay.pdf.exe", b"MZ")]), limits)) + + +def test_bidi_override_in_member_name(limits): + assert "archive_bidi_filename" in codes(run(make_zip([("cv‮gpj.exe", b"x")]), limits)) + + +def test_encrypted_entry_marks_inspection_incomplete(limits): + findings = run(encrypted_flag_zip(), limits) + match = next(f for f in findings if f.code == "archive_encrypted") + assert match.inspection_incomplete is True + + +def test_compression_bomb_ratio(limits): + data = make_zip([("zeros.bin", b"\x00" * (40 * 1024 * 1024))]) + assert "archive_bomb_ratio" in codes(run(data, limits)) + assert len(data) < 100 * 1024 # the point: tiny on disk, huge unpacked + + +def test_corrupt_archive_is_incomplete_not_clean(limits): + findings = run(b"PK\x03\x04not-really-a-zip", limits) + assert any(f.inspection_incomplete for f in findings) + + +def test_nested_archives_are_inspected(limits): + inner = make_zip([("payload.exe", b"MZ")]) + outer = make_zip([("bundle.zip", inner)]) + found = codes(run(outer, limits)) + assert "archive_executable_member" in found + assert "archive_nested" in found + + +def test_recursion_stops_at_the_depth_limit(limits): + tight = ScanLimits(**{**limits.__dict__, "max_archive_depth": 1}) + deepest = make_zip([("secret.exe", b"MZ")]) + level2 = make_zip([("l2.zip", deepest)]) + level1 = make_zip([("l1.zip", level2)]) + found = codes(run(level1, tight, depth=1)) + assert "archive_depth_limit" in found + + +def test_office_documents_are_not_reported_as_nested_archives(limits): + """A .docx inside a .zip is a normal submission, not 'an archive in an archive'.""" + from tests.samples import make_docx + + data = make_zip([("report.docx", make_docx())]) + assert "archive_nested" not in codes(run(data, limits)) + + +def test_a_broken_inner_archive_does_not_condemn_the_outer_one(limits): + data = make_zip([("inner.zip", b"not a zip at all")]) + found = codes(run(data, limits)) + assert "archive_corrupt" not in found + assert "archive_nested_unreadable" in found + + +def test_member_flood_is_capped(limits): + tight = ScanLimits(**{**limits.__dict__, "max_archive_members": 10}) + data = make_zip([(f"f{i}.txt", b"x") for i in range(50)]) + findings = run(data, tight) + match = next(f for f in findings if f.code == "archive_member_flood") + assert match.inspection_incomplete is True + + +def test_findings_are_capped_per_file(limits): + tight = ScanLimits(**{**limits.__dict__, "max_findings_per_file": 5}) + data = make_zip([(f"prog{i}.exe", b"MZ") for i in range(100)]) + assert len(run(data, tight)) <= 5 + + +def test_nothing_is_written_to_disk(limits, tmp_path, monkeypatch): + """Regression guard: the detector must never extract.""" + monkeypatch.chdir(tmp_path) + before = set(tmp_path.iterdir()) + run(make_zip([("../../escape.txt", HARMLESS), ("inner.zip", make_zip([("a", b"b")]))]), limits) + assert set(tmp_path.iterdir()) == before diff --git a/tests/test_detectors_documents.py b/tests/test_detectors_documents.py new file mode 100644 index 0000000..6f84601 --- /dev/null +++ b/tests/test_detectors_documents.py @@ -0,0 +1,186 @@ +"""Office, PDF and image detectors.""" +from __future__ import annotations + +import io + +import pytest + +from scanner.detectors.image import analyze_image +from scanner.detectors.office import analyze_office +from scanner.detectors.pdf import analyze_pdf +from tests.samples import ( + DOC_XML, + EXTERNAL_TEMPLATE_RELS, + codes, + make_docx, + make_jpeg, + make_pdf, + make_png, + make_zip, +) + + +# --------------------------------------------------------------------- office +def office(data: bytes, limits, suffix=".docx"): + return analyze_office(io.BytesIO(data), limits=limits, suffix=suffix) + + +def test_ordinary_word_document_is_clean(limits): + """Regression: the old detector flagged every .docx because every .docx + contains word/settings.xml.""" + assert codes(office(make_docx(), limits)) == set() + + +def test_macro_project_is_detected(limits): + findings = office(make_docx({"word/vbaProject.bin": b"\x00" * 64}), limits, ".docm") + match = next(f for f in findings if f.code == "office_macro_present") + assert match.severity.value == "high" + assert match.confidence.value == "high" + + +def test_macro_enabled_extension_without_macros_is_only_informational(limits): + findings = office(make_docx(), limits, ".docm") + match = next(f for f in findings if f.code == "office_macro_extension_only") + assert match.severity.value == "info" + + +def test_remote_template_relationship_is_detected(limits): + data = make_docx({"word/_rels/settings.xml.rels": EXTERNAL_TEMPLATE_RELS.encode()}) + assert "office_external_attachedtemplate" in codes(office(data, limits)) + + +def test_ordinary_hyperlink_is_not_flagged(limits): + rels = ( + '' + '' + "" + ) + data = make_docx({"word/_rels/document.xml.rels": rels.encode()}) + assert codes(office(data, limits)) == set() + + +def test_dde_field_is_detected(limits): + body = DOC_XML.replace("hello", "DDEAUTO calc.exe") + assert "office_dde_field" in codes(office(make_docx(body=body), limits)) + + +def test_embedded_object_and_activex(limits): + assert "office_embedded_object" in codes( + office(make_docx({"word/embeddings/oleObject1.bin": b"\x00"}), limits) + ) + assert "office_activex" in codes( + office(make_docx({"word/activeX/activeX1.bin": b"\x00"}), limits) + ) + + +def test_program_renamed_to_docx(limits): + assert "office_container_mismatch" in codes(office(b"MZ\x90\x00" + b"\x00" * 64, limits)) + + +def test_legacy_ole_macro_storage(limits): + data = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 64 + b"V\x00B\x00A\x00" + b"\x00" * 64 + assert "office_legacy_macro" in codes(office(data, limits, ".doc")) + + +# ------------------------------------------------------------------------ pdf +def pdf(data: bytes, limits): + return analyze_pdf(io.BytesIO(data), limits=limits, size=len(data)) + + +def test_ordinary_pdf_is_clean(limits): + assert codes(pdf(make_pdf(), limits)) == set() + + +def test_javascript_and_open_action(limits): + found = codes(pdf(make_pdf("/OpenAction << /S /JavaScript /JS (x) >> "), limits)) + assert {"pdf_javascript", "pdf_open_action"} <= found + + +def test_javascript_fires_once_not_twice(limits): + """/JS and /JavaScript are the same fact; the old scanner counted both.""" + findings = pdf(make_pdf("/OpenAction << /S /JavaScript /JS (x) >> "), limits) + assert len([f for f in findings if f.code == "pdf_javascript"]) == 1 + + +def test_launch_action_is_high_severity(limits): + findings = pdf(make_pdf("/Launch (calc.exe) "), limits) + match = next(f for f in findings if f.code == "pdf_launch_action") + assert match.severity.value == "high" + + +def test_token_spanning_a_window_boundary_is_still_found(limits): + """The original used a 10-byte overlap while searching 13-byte tokens.""" + filler = b"A" * (512 * 1024 - len(b"%PDF-1.4\n") - 6) + data = b"%PDF-1.4\n" + filler + b"/Launc" + b"h (x)\n%%EOF\n" + assert "pdf_launch_action" in codes(pdf(data, limits)) + + +def test_appended_payload_after_eof(limits): + data = make_pdf(trailer_extra=b"PK\x03\x04" + b"A" * 4096) + assert "pdf_appended_data" in codes(pdf(data, limits)) + + +def test_prefix_before_pdf_header(limits): + assert "pdf_header_offset" in codes(pdf(b"GIF89a" + b"\x00" * 40 + make_pdf(), limits)) + + +def test_encrypted_pdf_marks_inspection_incomplete(limits): + findings = pdf(make_pdf().replace(b"trailer", b"trailer /Encrypt 5 0 R "), limits) + assert next(f for f in findings if f.code == "pdf_encrypted").inspection_incomplete + + +# ---------------------------------------------------------------------- image +def image(data: bytes, limits, suffix=".png"): + return analyze_image(io.BytesIO(data), limits=limits, size=len(data), suffix=suffix) + + +def test_ordinary_image_is_clean(limits): + assert codes(image(make_png(), limits)) == set() + assert codes(image(make_jpeg(), limits, ".jpg")) == set() + + +def test_polyglot_zip_inside_png(limits): + data = make_png(trailer=make_zip([("hidden.txt", b"payload")])) + match = next(f for f in image(data, limits) if f.code == "image_polyglot") + assert match.severity.value == "high" + + +def test_large_appended_payload_is_detected(limits): + """The original only read the last 8 KB, so a 2 MB payload passed as clean.""" + data = make_png(trailer=b"Q" * (2 * 1024 * 1024)) + assert "image_large_appended_data" in codes(image(data, limits)) + + +def test_moderate_appended_payload_is_detected(limits): + assert "image_appended_data" in codes(image(make_png(trailer=b"Q" * 40_000), limits)) + + +def test_tiny_trailer_is_only_informational(limits): + findings = image(make_png(trailer=b"q" * 40), limits) + match = next(f for f in findings if f.code == "image_small_trailer") + assert match.severity.value == "info" + + +def test_program_renamed_to_jpg(limits): + match = next( + f for f in image(b"MZ\x90\x00" + b"\x00" * 512, limits, ".jpg") + if f.code == "image_not_an_image" + ) + assert match.severity.value == "high" + + +def test_impossible_chunk_length(limits): + import struct + + bad = struct.pack(">I", 2**31) + b"tEXt" + b"x" * 4 + assert "image_bad_chunk_length" in codes(image(make_png(chunks=bad), limits)) + + +@pytest.mark.parametrize("suffix", [".png", ".jpg"]) +def test_every_finding_has_teacher_facing_text(limits, suffix): + data = make_png(trailer=b"PK\x03\x04" + b"Z" * 4000) + for finding in image(data, limits, suffix): + assert finding.plain and finding.why and finding.action + assert finding.detector diff --git a/tests/test_heuristics.py b/tests/test_heuristics.py deleted file mode 100644 index 1cc5f9e..0000000 --- a/tests/test_heuristics.py +++ /dev/null @@ -1,20 +0,0 @@ -from scanner import heuristics - - -def test_calculate_score_with_multiple_findings(): - findings = [ - {"code": "exe_in_zip"}, - {"code": "pdf_token"}, - {"code": "url"}, - ] - score, label, reasons = heuristics.calculate_score(findings) - assert score >= 60 - assert label in {"Suspicious", "High"} - assert any("exe_in_zip" in reason for reason in reasons) - - -def test_calculate_score_safe_when_empty(): - score, label, reasons = heuristics.calculate_score([]) - assert score == 0 - assert label == "Safe" - assert reasons == [] diff --git a/tests/test_image_rules.py b/tests/test_image_rules.py deleted file mode 100644 index 5ed9858..0000000 --- a/tests/test_image_rules.py +++ /dev/null @@ -1,13 +0,0 @@ -from pathlib import Path - -from scanner.detectors.image_rules import analyze_image -from tests.utils_make_samples import make_png_with_appended - - -def test_png_appended_data(tmp_path: Path) -> None: - sample = tmp_path / "sample.png" - make_png_with_appended(sample) - with sample.open("rb") as handle: - findings = analyze_image(handle, strict=False) - rules = {finding["rule"] for finding in findings} - assert "png_appended_data" in rules diff --git a/tests/test_integration.py b/tests/test_integration.py deleted file mode 100644 index 1bd90ce..0000000 --- a/tests/test_integration.py +++ /dev/null @@ -1,12 +0,0 @@ -from pathlib import Path - -from scanner.scanner_core import ScanConfig, scan - - -def test_scan_examples_benign_samples(): - sample_dir = Path(__file__).resolve().parent.parent / "examples" / "benign_samples" - results = scan(sample_dir, ScanConfig(max_file_size=5_000_000, threads=2)) - assert results - for result in results: - assert result.severity in {"Safe", "Caution"} - assert result.score < 50 diff --git a/tests/test_office_rules.py b/tests/test_office_rules.py deleted file mode 100644 index 2811b0b..0000000 --- a/tests/test_office_rules.py +++ /dev/null @@ -1,13 +0,0 @@ -from pathlib import Path - -from scanner.detectors.office_rules import analyze_office -from tests.utils_make_samples import make_office_with_vba - - -def test_office_vba_detection(tmp_path: Path) -> None: - sample = tmp_path / "sample.docx" - make_office_with_vba(sample) - with sample.open("rb") as handle: - findings = analyze_office(handle, strict=False) - rules = {finding["rule"] for finding in findings} - assert "office_vba_project" in rules diff --git a/tests/test_pdf_rules.py b/tests/test_pdf_rules.py deleted file mode 100644 index ef77c81..0000000 --- a/tests/test_pdf_rules.py +++ /dev/null @@ -1,14 +0,0 @@ -from pathlib import Path - -from scanner.detectors.pdf_rules import analyze_pdf -from tests.utils_make_samples import make_pdf_with_js - - -def test_pdf_js(tmp_path: Path) -> None: - sample = tmp_path / "sample.pdf" - make_pdf_with_js(sample) - with sample.open("rb") as handle: - findings = analyze_pdf(handle, strict=True) - rules = {finding["rule"] for finding in findings} - assert "pdf_javascript" in rules - assert "pdf_auto_actions" in rules diff --git a/tests/test_quarantine.py b/tests/test_quarantine.py new file mode 100644 index 0000000..dd67166 --- /dev/null +++ b/tests/test_quarantine.py @@ -0,0 +1,160 @@ +"""Quarantine must never delete, never overwrite, and always be reversible.""" +from __future__ import annotations + +import json +import os +import stat + +import pytest + +from scanner.quarantine import MANIFEST_NAME, QuarantineError, QuarantineStore, sha256_file + + +@pytest.fixture() +def store(tmp_path) -> QuarantineStore: + return QuarantineStore(tmp_path / "quarantine") + + +def test_two_files_with_the_same_name_do_not_overwrite_each_other(tmp_path, store): + """Regression: the original used dest/src.name, so the second file + destroyed the first.""" + a = tmp_path / "a" / "assignment.docx" + b = tmp_path / "b" / "assignment.docx" + a.parent.mkdir(parents=True) + b.parent.mkdir(parents=True) + a.write_bytes(b"student one") + b.write_bytes(b"student two") + + entry_a = store.quarantine(a) + entry_b = store.quarantine(b) + + assert entry_a.stored_name != entry_b.stored_name + assert (store.root / entry_a.stored_name).read_bytes() == b"student one" + assert (store.root / entry_b.stored_name).read_bytes() == b"student two" + + +def test_stored_name_is_not_executable_by_the_shell(tmp_path, store): + source = tmp_path / "malware.exe" + source.write_bytes(b"MZ") + entry = store.quarantine(source) + assert entry.stored_name.endswith(".quarantined") + stored = store.root / entry.stored_name + mode = stored.stat().st_mode + assert not mode & stat.S_IXUSR + assert not mode & stat.S_IXGRP + assert not mode & stat.S_IXOTH + + +def test_original_is_moved_not_copied(tmp_path, store): + source = tmp_path / "x.txt" + source.write_bytes(b"data") + store.quarantine(source) + assert not source.exists() + + +def test_nothing_is_ever_deleted(tmp_path, store): + source = tmp_path / "x.txt" + source.write_bytes(b"important") + entry = store.quarantine(source) + assert (store.root / entry.stored_name).read_bytes() == b"important" + + +def test_symlinks_are_refused(tmp_path, store): + real = tmp_path / "real.txt" + real.write_bytes(b"x") + link = tmp_path / "link.txt" + link.symlink_to(real) + with pytest.raises(QuarantineError, match="symbolic link"): + store.quarantine(link) + assert real.exists() + + +def test_restore_returns_the_file_to_its_original_path(tmp_path, store): + source = tmp_path / "sub" / "essay.docx" + source.parent.mkdir() + source.write_bytes(b"my essay") + entry = store.quarantine(source) + restored = store.restore(entry.entry_id) + assert restored == source + assert restored.read_bytes() == b"my essay" + + +def test_restore_can_target_a_different_folder(tmp_path, store): + source = tmp_path / "essay.docx" + source.write_bytes(b"content") + entry = store.quarantine(source) + elsewhere = tmp_path / "reviewed" + elsewhere.mkdir() + restored = store.restore(entry.entry_id, destination=elsewhere) + assert restored == elsewhere / "essay.docx" + + +def test_restore_refuses_to_overwrite(tmp_path, store): + source = tmp_path / "essay.docx" + source.write_bytes(b"original") + entry = store.quarantine(source) + source.write_bytes(b"a newer file with the same name") + with pytest.raises(QuarantineError, match="already exists"): + store.restore(entry.entry_id) + assert source.read_bytes() == b"a newer file with the same name" + + +def test_restore_twice_is_refused(tmp_path, store): + source = tmp_path / "essay.docx" + source.write_bytes(b"content") + entry = store.quarantine(source) + store.restore(entry.entry_id) + with pytest.raises(QuarantineError, match="already restored"): + store.restore(entry.entry_id) + + +def test_restore_detects_tampering(tmp_path, store): + source = tmp_path / "essay.docx" + source.write_bytes(b"content") + entry = store.quarantine(source) + stored = store.root / entry.stored_name + stored.chmod(0o600) + stored.write_bytes(b"something else entirely") + with pytest.raises(QuarantineError, match="no longer matches"): + store.restore(entry.entry_id) + + +def test_manifest_is_append_only_jsonl(tmp_path, store): + for name in ("a.txt", "b.txt"): + path = tmp_path / name + path.write_bytes(name.encode()) + store.quarantine(path, reason="test") + lines = (store.root / MANIFEST_NAME).read_text().strip().splitlines() + assert len(lines) == 2 + for line in lines: + record = json.loads(line) + assert record["sha256"] and record["original_path"] and record["quarantined_at"] + + +def test_manifest_survives_a_corrupt_line(tmp_path, store): + path = tmp_path / "a.txt" + path.write_bytes(b"a") + store.quarantine(path) + with (store.root / MANIFEST_NAME).open("a") as handle: + handle.write("{not json at all\n") + assert len(store.entries()) == 1 + + +def test_hash_recorded_matches_the_stored_file(tmp_path, store): + path = tmp_path / "a.bin" + path.write_bytes(b"\x00\x01\x02") + entry = store.quarantine(path) + assert sha256_file(store.root / entry.stored_name) == entry.sha256 + + +def test_quarantine_directory_is_not_world_readable(store): + if os.name == "nt": # pragma: no cover + pytest.skip("POSIX permissions only") + mode = store.root.stat().st_mode + assert not mode & stat.S_IROTH + assert not mode & stat.S_IRGRP + + +def test_unknown_entry_id_is_an_error(store): + with pytest.raises(QuarantineError, match="No quarantine entry"): + store.restore("deadbeef") diff --git a/tests/test_reporting_and_cli.py b/tests/test_reporting_and_cli.py new file mode 100644 index 0000000..877839f --- /dev/null +++ b/tests/test_reporting_and_cli.py @@ -0,0 +1,216 @@ +"""Reporting, the triage summary, the CLI, and the GUI presentation model.""" +from __future__ import annotations + +import io +import json + +import pytest + +from scanner import __version__ +from scanner import main as cli +from scanner.findings import Verdict +from scanner.gui_model import detail_for, parse_dropped_paths, rows_for, summarise_for_email +from scanner.limits import DEFAULT_LIMITS +from scanner.reporters import ( + generate_html_report, + print_console_report, + sanitize_display, + write_html_report, + write_json_report, +) +from scanner.scanner_core import ScanConfig, scan +from scanner.triage import build_summary, summary_from_dict + + +@pytest.fixture() +def summary(corpus): + results = scan(corpus, ScanConfig(limits=DEFAULT_LIMITS, threads=2)) + return build_summary(results, roots=[corpus], scanner_version=__version__, duration_ms=42) + + +# ------------------------------------------------------------------ triage +def test_headline_names_the_number_of_blocked_files(summary): + assert "should not be opened" in summary.headline() + assert str(summary.blocked) in summary.headline() + + +def test_counts_add_up(summary): + assert sum(summary.counts.values()) == summary.total_files == 6 + + +def test_exit_code_reflects_the_worst_verdict(summary): + assert summary.exit_code() == 2 + + +def test_duplicate_detection(tmp_path): + root = tmp_path / "dupes" + root.mkdir() + for name in ("a.txt", "b.txt", "c.txt"): + (root / name).write_bytes(b"identical content") + results = scan(root, ScanConfig(limits=DEFAULT_LIMITS)) + built = build_summary(results, roots=[root], scanner_version=__version__) + assert built.duplicate_groups + assert built.duplicate_groups[0]["count"] == 3 + + +def test_json_round_trip_preserves_verdicts(summary, tmp_path): + path = tmp_path / "r.json" + write_json_report(summary, path) + restored = summary_from_dict(json.loads(path.read_text())) + assert restored.counts == summary.counts + assert [r.verdict for r in restored.results] == [r.verdict for r in summary.results] + assert restored.headline() == summary.headline() + + +# -------------------------------------------------------------------- html +def test_html_report_is_self_contained(summary): + html = generate_html_report(summary) + assert "alert(1).txt").write_bytes(b"hi") + results = scan(root, ScanConfig(limits=DEFAULT_LIMITS)) + html = generate_html_report(build_summary(results, roots=[root], scanner_version="t")) + assert "