diff --git a/bench/README.md b/bench/README.md index 55699f2..a25e8c0 100644 --- a/bench/README.md +++ b/bench/README.md @@ -33,7 +33,7 @@ Needs auth: `klaatai login` first, or `KLAATAI_API_KEY=...`. The report JSON is written incrementally after every task — a mid-suite abort (daily quota, ctrl-c) still leaves a usable partial report (`"complete": false`). -## Tasks (33) +## Tasks (34) Each task is a self-contained fixture dir with failing tests the agent must make pass **without editing the test file**. Categories: @@ -41,12 +41,12 @@ pass **without editing the test file**. Categories: | category | count | what it exercises | |----------|-------|-------------------| | `bugfix` | 11 | find + fix a planted bug (off-by-one, mutation, async ordering, float money, regex escaping, unicode, shallow copy, state machine, …) | -| `implement` | 13 | implement a function/class from a stub + spec comment (LRU cache, event emitter, query string, JSON pointer, expression evaluator, …) | +| `implement` | 14 | implement a function/class from a stub + spec comment (LRU cache, event emitter, query string, JSON pointer, expression evaluator, …) | | `multi-file` | 3 | the failing test is not where the fix is — cross-file navigation (implement imported module, bug in dependency, missing export) | | `refactor` | 1 | behavior-preserving API change (callback → Promise) | | `long-context` | 5 | large ~30-file fixtures where navigation is the task: cross-module bug hunts, a wide mechanical fix across 8 feature modules, stale cache keys, wrong metric arguments, config precedence — exercises code-graph/search efficiency | -Difficulty spread: 10 easy · 18 medium · 5 hard. +Difficulty spread: 11 easy · 18 medium · 5 hard. ## Suite integrity — selfcheck (run after any task change) @@ -88,7 +88,7 @@ later with `--from`), never counted as failures. When `cursor-agent` (headless CLI) is unusable, `cursor-ide-bench.ts` runs the lane through the Cursor IDE's own agent chat: `prepare` builds a workspace of -all 33 tasks plus a paste-prompt and an objective `run-check.sh` referee; +all suite tasks plus a paste-prompt and an objective `run-check.sh` referee; `import --cost --tokens ` converts the results (cost/tokens from the Cursor dashboard's on-demand usage delta) into a normal report JSON, tagged with its single-session methodology. diff --git a/bench/solutions/implement-hex-color/src/color.ts b/bench/solutions/implement-hex-color/src/color.ts new file mode 100644 index 0000000..0373640 --- /dev/null +++ b/bench/solutions/implement-hex-color/src/color.ts @@ -0,0 +1,49 @@ +export interface RgbaColor { + r: number; + g: number; + b: number; + a: number; +} + +export function parseHexColor(hex: string): RgbaColor | null { + if (typeof hex !== "string") return null; + let clean = hex.trim(); + if (clean.startsWith("#")) { + clean = clean.slice(1); + } + + if (!/^[0-9a-fA-F]+$/.test(clean)) { + return null; + } + + let r = 0; + let g = 0; + let b = 0; + let a = 1; + + if (clean.length === 3) { + r = parseInt(clean[0]! + clean[0]!, 16); + g = parseInt(clean[1]! + clean[1]!, 16); + b = parseInt(clean[2]! + clean[2]!, 16); + } else if (clean.length === 4) { + r = parseInt(clean[0]! + clean[0]!, 16); + g = parseInt(clean[1]! + clean[1]!, 16); + b = parseInt(clean[2]! + clean[2]!, 16); + const rawA = parseInt(clean[3]! + clean[3]!, 16); + a = Math.round((rawA / 255) * 1000) / 1000; + } else if (clean.length === 6) { + r = parseInt(clean.slice(0, 2), 16); + g = parseInt(clean.slice(2, 4), 16); + b = parseInt(clean.slice(4, 6), 16); + } else if (clean.length === 8) { + r = parseInt(clean.slice(0, 2), 16); + g = parseInt(clean.slice(2, 4), 16); + b = parseInt(clean.slice(4, 6), 16); + const rawA = parseInt(clean.slice(6, 8), 16); + a = Math.round((rawA / 255) * 1000) / 1000; + } else { + return null; + } + + return { r, g, b, a }; +} diff --git a/bench/suite.json b/bench/suite.json index 82bea99..ff64ab8 100644 --- a/bench/suite.json +++ b/bench/suite.json @@ -233,6 +233,13 @@ "prompt": "Environment overrides are supposed to beat file config, which beats defaults \u2014 but overrides are being ignored somewhere in this codebase. Trace the config pipeline and fix it so the tests in src/config.test.ts pass. Do not edit the test file.", "difficulty": "hard", "category": "long-context" + }, + { + "id": "implement-hex-color", + "dir": "tasks/implement-hex-color", + "prompt": "Implement parseHexColor in src/color.ts (spec in the file comments) so the tests in src/color.test.ts pass. Do not edit the test file.", + "difficulty": "easy", + "category": "implement" } ] -} \ No newline at end of file +} diff --git a/bench/tasks/implement-hex-color/src/color.test.ts b/bench/tasks/implement-hex-color/src/color.test.ts new file mode 100644 index 0000000..a4ab637 --- /dev/null +++ b/bench/tasks/implement-hex-color/src/color.test.ts @@ -0,0 +1,36 @@ +import { expect, test } from "bun:test"; +import { parseHexColor } from "./color.js"; + +test("parses 6-digit hex with hash", () => { + expect(parseHexColor("#336699")).toEqual({ r: 51, g: 102, b: 153, a: 1 }); +}); + +test("parses 6-digit hex without hash and case-insensitive", () => { + expect(parseHexColor("ff8800")).toEqual({ r: 255, g: 136, b: 0, a: 1 }); + expect(parseHexColor("AABBCC")).toEqual({ r: 170, g: 187, b: 204, a: 1 }); +}); + +test("parses 3-digit shorthand hex", () => { + expect(parseHexColor("#f00")).toEqual({ r: 255, g: 0, b: 0, a: 1 }); + expect(parseHexColor("0F0")).toEqual({ r: 0, g: 255, b: 0, a: 1 }); +}); + +test("parses 4-digit shorthand hex with alpha", () => { + expect(parseHexColor("#f008")).toEqual({ r: 255, g: 0, b: 0, a: 0.533 }); + expect(parseHexColor("000f")).toEqual({ r: 0, g: 0, b: 0, a: 1 }); +}); + +test("parses 8-digit hex with alpha", () => { + expect(parseHexColor("#33669980")).toEqual({ r: 51, g: 102, b: 153, a: 0.502 }); + expect(parseHexColor("00000000")).toEqual({ r: 0, g: 0, b: 0, a: 0 }); +}); + +test("returns null for invalid inputs", () => { + expect(parseHexColor("")).toBeNull(); + expect(parseHexColor("#")).toBeNull(); + expect(parseHexColor("#12")).toBeNull(); + expect(parseHexColor("#12345")).toBeNull(); + expect(parseHexColor("#123456789")).toBeNull(); + expect(parseHexColor("#xyz")).toBeNull(); + expect(parseHexColor("#123g")).toBeNull(); +}); diff --git a/bench/tasks/implement-hex-color/src/color.ts b/bench/tasks/implement-hex-color/src/color.ts new file mode 100644 index 0000000..5dd36d1 --- /dev/null +++ b/bench/tasks/implement-hex-color/src/color.ts @@ -0,0 +1,24 @@ +// parseHexColor(hex): Parses a hex color string into an RGBA object. +// +// Supported formats (case-insensitive, optional leading '#'): +// - 3-digit: RGB (e.g. "F00" or "#F00" -> { r: 255, g: 0, b: 0, a: 1 }) +// - 4-digit: RGBA (e.g. "#F008" -> { r: 255, g: 0, b: 0, a: 0.533 }) +// - 6-digit: RRGGBB (e.g. "#336699" -> { r: 51, g: 102, b: 153, a: 1 }) +// - 8-digit: RRGGBBAA (e.g. "33669980" -> { r: 51, g: 102, b: 153, a: 0.502 }) +// +// Notes: +// - Each 3-digit/4-digit component character 'C' expands to 'CC' (e.g. 'F' -> 0xFF = 255). +// - Alpha channel 'a' is a number between 0 and 1, rounded to 3 decimal places (Math.round((alpha / 255) * 1000) / 1000). +// - Returns null for invalid inputs (non-hex characters, invalid lengths, empty string). +// +// TODO: not implemented yet. +export interface RgbaColor { + r: number; + g: number; + b: number; + a: number; +} + +export function parseHexColor(_hex: string): RgbaColor | null { + return null; +}