Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions api/models/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,9 @@ class DayLeader(BaseModel):
class AuthorStat(BaseModel):
name: str
commits: int
hue: int = Field(
description="Stable 0-359 hue from the name hash; the display colour is built from it client-side"
)


class RepoStats(BaseModel):
Expand Down Expand Up @@ -253,6 +256,12 @@ class Manifest(BaseModel):
busyness: BusynessThresholds
dateRanges: DateRanges
stats: RepoStats
readmePath: Optional[str] = Field(
description="Absolute path of the root README, or null if there isn't one"
)
readmeModified: Optional[str] = Field(
description="That README's mtime, for cache-busting the fetch"
)


class SignatureResponse(BaseModel):
Expand All @@ -261,6 +270,13 @@ class SignatureResponse(BaseModel):
content_signature: str


class DateRangeMs(BaseModel):
minCreated: int
maxCreated: int
minModified: int
maxModified: int


class TimelineChange(BaseModel):
path: str
sha: Optional[str] = Field(description="New blob sha, or null when deleted")
Expand All @@ -280,6 +296,7 @@ class TimelineBundle(BaseModel):
blobLines: dict[str, int]
blobSizes: dict[str, int]
commitLineRanges: list[RangeStat]
commitDateRanges: list[DateRangeMs]
note: Optional[str]


Expand Down
6 changes: 4 additions & 2 deletions api/services/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ class BlobEntry(TypedDict):
_FILE_CACHE_VERSION = 3 # v3: exact line counts (dropped the >5MB sampling estimate)
_BLOB_STATS_CACHE_VERSION = 4 # v4: git-lfs pointers resolved to real content
_GIT_HISTORY_CACHE_VERSION = 15 # v15: commit dates carry a time, not just a day
_TIMELINE_CACHE_VERSION = 6 # v6: bundle ships blobSizes
_TIMELINE_CACHE_VERSION = 7 # v7: bundle ships commitDateRanges
_MANIFEST_SCHEMA_VERSION = (
# v12: per-dir descendants_created_min / descendants_modified_max
# v13: ext_breakdown `ext` is null (was "(none)") for extensionless files
Expand All @@ -94,7 +94,9 @@ class BlobEntry(TypedDict):
# v19: FileNode.binaryType + RepoStats.binaryCount/maxBinaryBytesFile/
# minBinaryBytesFile (binary files as a first-class "data" category)
# v20: exact line counts (was sampled >5MB) — values change, bump to rebuild
20
# v21: readmePath / readmeModified resolved server-side
# v22: AuthorStat.hue resolved server-side
22
)
# Composite: invalidates when EITHER the manifest schema OR the git-history
# shape changes. Stored as a string in the cache file's `version` field.
Expand Down
18 changes: 17 additions & 1 deletion api/services/manifest_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ class CommitEntry(TypedDict):
scene tree-color both read one consistent value instead of each
recomputing the per-day grouping."""

date: str # "YYYY-MM-DD"
date: str # ISO-8601 UTC
files: int
sha: str
authors: list[str]
Expand Down Expand Up @@ -236,6 +236,8 @@ class AuthorStat(TypedDict):

name: str
commits: int
# 0-359, hashed from the name so the firefly orb and commit dot agree.
hue: int


class RepoStats(TypedDict):
Expand Down Expand Up @@ -295,6 +297,19 @@ class Manifest(TypedDict):
busyness: BusynessThresholds
dateRanges: DateRanges
stats: RepoStats
# Root README, resolved server-side so the client doesn't re-scan for it.
readmePath: str | None
readmeModified: str | None


class DateRangeMs(TypedDict):
"""Created/modified epoch-ms spread over the files present at one commit.
Zeroes when nothing is present, which the client reads as no spread."""

minCreated: int
maxCreated: int
minModified: int
maxModified: int


class TimelineChange(TypedDict):
Expand Down Expand Up @@ -326,6 +341,7 @@ class TimelineBundle(TypedDict):
blobLines: dict[str, int]
blobSizes: dict[str, int]
commitLineRanges: list[RangeStat]
commitDateRanges: list[DateRangeMs]
note: str | None


Expand Down
15 changes: 15 additions & 0 deletions api/services/scan.py
Original file line number Diff line number Diff line change
Expand Up @@ -1424,6 +1424,7 @@ def _wrap_manifest(
from api.services.source import label_from_source

tree["name"] = label_from_source(repo_info["remote_url"]) or tree["name"]
readme_path, readme_modified = _find_root_readme(tree)
return {
"root": root_abs,
"scanned_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
Expand All @@ -1436,9 +1437,23 @@ def _wrap_manifest(
"busyness": _compute_busyness(commits),
"dateRanges": signals.date_ranges,
"stats": compute_repo_stats(tree, commits),
"readmePath": readme_path,
"readmeModified": readme_modified,
}


def _find_root_readme(tree: DirNode) -> tuple[str | None, str | None]:
"""(fullPath, modified) of the root README, or (None, None). Matches any
direct child whose stem is "readme" — the rule GitHub and VSCode use."""
for child in tree.get("children") or []:
if child.get("type") != NodeKind.FILE:
continue
name = (child.get("name") or "").lower()
if name == "readme" or name.startswith("readme."):
return child.get("fullPath"), child.get("modified")
return None, None


def _annotate_same_day_totals(commits: list[CommitEntry]) -> None:
"""In-place: set each commit's same_day_total to the number of commits
sharing its calendar date. A derived aggregate (like busyness), so it's
Expand Down
12 changes: 11 additions & 1 deletion api/services/stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,16 @@ def _longest_streak(dates: list[str]) -> int:
return best


def _author_hue(name: str) -> int:
"""FNV-1a over the name's UTF-8 bytes, mod 360. Mirrors the 32-bit unsigned
arithmetic of the JS original so a name keeps the hue it already had."""
h = 0x811C9DC5
for byte in name.encode("utf-8"):
h ^= byte
h = (h * 0x01000193) & 0xFFFFFFFF
return h % 360


def compute_repo_stats(tree: DirNode, commits: list[CommitEntry]) -> RepoStats:
# One pass over files, one over dirs, one over commits — each updates every
# winner/range/count for that pool as it goes (first-seen wins ties, via
Expand Down Expand Up @@ -204,7 +214,7 @@ def compute_repo_stats(tree: DirNode, commits: list[CommitEntry]) -> RepoStats:
busiest_day = {"date": best_date, "count": best_count}

authors: list[AuthorStat] = [
{"name": name, "commits": n}
{"name": name, "commits": n, "hue": _author_hue(name)}
for name, n in sorted(author_counts.items(), key=lambda kv: (-kv[1], kv[0]))
]

Expand Down
97 changes: 96 additions & 1 deletion api/services/timeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,20 @@
import hashlib
import subprocess
import time
from datetime import datetime, timezone
from pathlib import Path
from typing import Callable, NamedTuple

from .cache import BlobEntry, cache_load_blobs, cache_save_blobs
from .gitobj import _git_argv, blob_sizes_batch, blob_stats_batch
from .manifest_types import CommitEntry, FileNode, Manifest, RangeStat, TimelineBundle
from .manifest_types import (
CommitEntry,
DateRangeMs,
FileNode,
Manifest,
RangeStat,
TimelineBundle,
)
from .media import media_kind
from .scan import (
SCAN_PROGRESS_THROTTLE_S,
Expand Down Expand Up @@ -291,6 +299,89 @@ def compute_commit_line_ranges(
return ranges


def _iso_ms(value: str | None) -> int | None:
"""Epoch ms for a Z-suffixed UTC stamp, or None. Matches JS Date.parse,
which the client used before this moved server-side."""
if not value:
return None
try:
parsed = datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ")
except ValueError:
return None
return int(parsed.replace(tzinfo=timezone.utc).timestamp() * 1000)


def compute_commit_date_ranges(
deltas: list[CommitDelta],
commits: list[CommitEntry],
git_created: dict[str, str],
git_modified: dict[str, str],
) -> list[DateRangeMs]:
"""Per-commit created/modified ms ranges over the files present at each
commit — what the weathering (color, lit windows, grime) normalizes against,
so range[HEAD] equals the live manifest's dateRanges.

Mirrors the client replay it replaces: a file uses its own full-precision
date once it has reached its final change, and the date of its latest change
commit before that; creation is fixed at its own date, falling back to its
genesis commit."""
commit_ms = [_iso_ms(c["date"]) or 0 for c in commits]

final_idx: dict[str, int] = {}
genesis_idx: dict[str, int] = {}
for i, delta in enumerate(deltas):
for path, sha in delta.changes:
final_idx[path] = i
if sha is not None and path not in genesis_idx:
genesis_idx[path] = i

present: set[str] = set()
last_change: dict[str, int] = {}
ranges: list[DateRangeMs] = []
for i, delta in enumerate(deltas):
for path, sha in delta.changes:
last_change[path] = i
if sha is None:
present.discard(path)
else:
present.add(path)

min_created = min_modified = None
max_created = max_modified = None
for path in present:
lm = last_change.get(path, 0)
modified = None
if lm >= final_idx.get(path, 0):
modified = _iso_ms(git_modified.get(path))
if modified is None:
modified = commit_ms[lm] if lm < len(commit_ms) else 0
created = _iso_ms(git_created.get(path))
if created is None:
gi = genesis_idx.get(path, 0)
created = commit_ms[gi] if gi < len(commit_ms) else 0

if min_created is None or created < min_created:
min_created = created
if max_created is None or created > max_created:
max_created = created
if min_modified is None or modified < min_modified:
min_modified = modified
if max_modified is None or modified > max_modified:
max_modified = modified

# An empty present set collapses to a zero span, which the client already
# treats as "no spread" (freshest / newest).
ranges.append(
{
"minCreated": min_created or 0,
"maxCreated": max_created or 0,
"minModified": min_modified or 0,
"maxModified": max_modified or 0,
}
)
return ranges


def build_timeline_bundle(
root: str,
*,
Expand Down Expand Up @@ -374,6 +465,9 @@ def build_timeline_bundle(
git_modified,
)
commit_line_ranges = compute_commit_line_ranges(deltas, blob_lines)
commit_date_ranges = compute_commit_date_ranges(
deltas, commits, git_created, git_modified
)
_log("timeline bundle complete")

return {
Expand All @@ -386,5 +480,6 @@ def build_timeline_bundle(
"blobLines": blob_lines,
"blobSizes": blob_sizes,
"commitLineRanges": commit_line_ranges,
"commitDateRanges": commit_date_ranges,
"note": note,
}
15 changes: 13 additions & 2 deletions api/tests/services/test_stats.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from api.services.stats import compute_repo_stats
from api.services.stats import _author_hue, compute_repo_stats


def _file(
Expand Down Expand Up @@ -169,7 +169,7 @@ def test_commit_leaders_authors_and_streak():
assert s["commitDates"] == {"oldest": "2022-01-01", "newest": "2022-02-10"}
assert s["maxCommitsPerDay"]["count"] == 2
assert s["maxCommitStreakDays"] == 3
assert s["authors"][0] == {"name": "Ada", "commits": 3}
assert s["authors"][0] == {"name": "Ada", "commits": 3, "hue": _author_hue("Ada")}
assert [a["name"] for a in s["authors"]] == ["Ada", "Bo"]


Expand Down Expand Up @@ -225,3 +225,14 @@ def test_empty_files_only_ranges():
s = compute_repo_stats(_dir("repo", "", [empty]), [])
assert s["lineCountRange"] == {"min": 0, "max": 0}
assert s["byteSizeRange"] == {"min": 0, "max": 0}


def test_author_hue_matches_the_javascript_hash_it_replaced():
"""Golden values from the original client-side FNV-1a, so moving the hash
to Python can't silently recolor everyone's orbs."""
assert _author_hue("Alice") == 143
assert _author_hue("Bob") == 220
assert _author_hue("") == 61
# Unicode hashes over UTF-8 bytes, not UTF-16 code units.
assert _author_hue("Yan \U0001f680 M\u00fcller") == 181
assert all(0 <= _author_hue(f"user-{i}") < 360 for i in range(200))
2 changes: 2 additions & 0 deletions api/tests/test_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@ def test_file_node_media_one_only_rejected(self) -> None:

def test_manifest_serializes(self) -> None:
m = Manifest(
readmePath=None,
readmeModified=None,
root="/r",
scanned_at="2020",
content_signature="s",
Expand Down
26 changes: 26 additions & 0 deletions api/tests/test_scan.py
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,32 @@ def test_ext_breakdown_extensionless_files_use_null(self):
self.assertNotIn("(none)", exts)
self.assertIn(".py", exts)

def test_readme_path_resolves_root_readme_case_insensitively(self):
with tempfile.TemporaryDirectory() as td:
root = Path(td)
_init_repo(root)
(root / "ReadMe.MD").write_text("# hi")
(root / "main.py").write_text("print()")
_commit_all(root)

m = _final_manifest(str(root))
self.assertIsNotNone(m["readmePath"])
self.assertTrue(m["readmePath"].endswith("ReadMe.MD"))
# Paired with the file's mtime so the client can cache-bust the fetch.
names = {c["name"]: c for c in m["tree"]["children"]}
self.assertEqual(m["readmeModified"], names["ReadMe.MD"]["modified"])

def test_readme_path_is_none_without_one_and_ignores_nested(self):
with tempfile.TemporaryDirectory() as td:
root = Path(td)
_init_repo(root)
(root / "readme_notes.py").write_text("x") # stem is not "readme"
(root / "docs").mkdir()
(root / "docs" / "README.md").write_text("# nested")
_commit_all(root)

self.assertIsNone(_final_manifest(str(root))["readmePath"])

def test_descendant_date_range_matches_repo_ranges(self):
m = _final_manifest(str(FIXTURE))
tree = m["tree"]
Expand Down
Loading