Evaluating Coding Agents

# Example Level 100 task
name: fix-syntax-error
level: 100
description: Fix the syntax error in the JavaScript file
setup: |
  Create a file with a missing closing bracket
verification: |
  File parses without errors
  Original logic preserved

# Example Level 200 task
name: implement-pagination
level: 200
description: Add pagination to the /api/users endpoint
setup: |
  Existing Express app with user listing
  Database has 1000+ test records
verification: |
  Accepts page and limit query params
  Returns correct subset of results
  Includes total count in response

# Example Level 300 task
name: optimize-n-plus-one
level: 300
description: Identify and fix N+1 query problem in order listing
setup: |
  Django app with Order, LineItem, Product models
  Test database with 100 orders, 500 line items
verification: |
  Query count reduced from 501 to under 10
  Response time improved by at least 80%
  All existing tests pass

# Example Level 400 task
name: fix-race-condition
level: 400
description: Fix race condition in distributed lock implementation
setup: |
  Redis-based distributed lock with subtle timing bug
  Concurrent test harness that triggers the race
verification: |
  100,000 concurrent lock attempts with zero conflicts
  No deadlocks under high contention
  Maintains lock semantics under network partitions

evals/
  config.yaml           # Suite configuration
  tasks/
    level-100/
      fix-syntax-error/
        task.yaml       # Task definition
        setup/          # Initial state files
        expected/       # Expected outcomes
      add-function/
        ...
    level-200/
      implement-pagination/
        ...
    level-300/
      ...
    level-400/
      ...
  scripts/
    run-eval.sh         # Execution script
    generate-report.py  # Report generator

# evals/tasks/level-200/implement-auth/task.yaml
name: implement-jwt-auth
level: 200
category: feature-implementation
timeout: 300  # seconds

description: |
  Implement JWT authentication for the Express API.
  Users should be able to login and receive a token.
  Protected routes should verify the token.

setup:
  files:
    - src/server.js
    - src/routes/users.js
    - package.json
  commands:
    - npm install

prompt: |
  Add JWT authentication to this Express app:
  1. Create a POST /auth/login endpoint
  2. Return a JWT token on successful login
  3. Add middleware to protect /api/* routes
  4. Use the JWT_SECRET environment variable

verification:
  tests:
    - npm test
  assertions:
    - file_exists: src/middleware/auth.js
    - endpoint_returns_200: POST /auth/login
    - endpoint_returns_401: GET /api/users (no token)
    - endpoint_returns_200: GET /api/users (valid token)

# Run a single task
autohand eval run evals/tasks/level-200/implement-auth

# Run all tasks at a difficulty level
autohand eval run evals/tasks/level-200 --all

# Run the complete suite
autohand eval run evals --all --output results.json

# Run with verbose output for debugging
autohand eval run evals/tasks/level-100 --verbose

# Install the benchmark harness
pip install terminal-bench

# Run the benchmark with Autohand as the agent
tbench run --agent autohand --output results.json

# Run specific difficulty levels
tbench run --agent autohand --difficulty easy
tbench run --agent autohand --difficulty medium
tbench run --agent autohand --difficulty hard

# Submit results to the leaderboard
tbench submit results.json --agent-name "Autohand v1.2.0"

#!/usr/bin/env python3
"""
Simple evaluation runner for coding agent benchmarks.
Runs tasks, measures success, and generates reports.
"""

import json
import subprocess
import time
from pathlib import Path
from dataclasses import dataclass, asdict
from datetime import datetime

@dataclass
class TaskResult:
    name: str
    level: int
    category: str
    passed: bool
    duration_seconds: float
    error: str | None = None

@dataclass
class EvalReport:
    agent: str
    timestamp: str
    total_tasks: int
    passed_tasks: int
    pass_rate: float
    by_level: dict
    by_category: dict
    results: list[TaskResult]

def run_task(task_dir: Path, agent_cmd: str) -> TaskResult:
    """Execute a single evaluation task."""
    task_config = json.loads((task_dir / "task.json").read_text())

    start = time.time()
    try:
        # Setup the task environment
        setup_dir = task_dir / "setup"
        work_dir = Path(f"/tmp/eval-{task_config['name']}")
        work_dir.mkdir(exist_ok=True)

        # Copy setup files
        if setup_dir.exists():
            subprocess.run(["cp", "-r", str(setup_dir) + "/.", str(work_dir)], check=True)

        # Run the agent with the task prompt
        result = subprocess.run(
            [agent_cmd, "--prompt", task_config["prompt"], "--cwd", str(work_dir)],
            capture_output=True,
            timeout=task_config.get("timeout", 300),
            text=True
        )

        # Verify the result
        passed = verify_task(work_dir, task_config["verification"])
        duration = time.time() - start

        return TaskResult(
            name=task_config["name"],
            level=task_config["level"],
            category=task_config["category"],
            passed=passed,
            duration_seconds=duration
        )

    except subprocess.TimeoutExpired:
        return TaskResult(
            name=task_config["name"],
            level=task_config["level"],
            category=task_config["category"],
            passed=False,
            duration_seconds=task_config.get("timeout", 300),
            error="Timeout"
        )
    except Exception as e:
        return TaskResult(
            name=task_config["name"],
            level=task_config["level"],
            category=task_config["category"],
            passed=False,
            duration_seconds=time.time() - start,
            error=str(e)
        )

def verify_task(work_dir: Path, verification: dict) -> bool:
    """Check if task was completed successfully."""
    # Run test commands
    for test_cmd in verification.get("tests", []):
        result = subprocess.run(
            test_cmd, shell=True, cwd=work_dir,
            capture_output=True
        )
        if result.returncode != 0:
            return False

    # Check file assertions
    for assertion in verification.get("assertions", []):
        if "file_exists" in assertion:
            if not (work_dir / assertion["file_exists"]).exists():
                return False
        if "file_contains" in assertion:
            path, pattern = assertion["file_contains"]
            content = (work_dir / path).read_text()
            if pattern not in content:
                return False

    return True

def generate_report(results: list[TaskResult], agent: str) -> EvalReport:
    """Generate evaluation report from results."""
    by_level = {}
    by_category = {}

    for r in results:
        # Aggregate by level
        if r.level not in by_level:
            by_level[r.level] = {"total": 0, "passed": 0}
        by_level[r.level]["total"] += 1
        if r.passed:
            by_level[r.level]["passed"] += 1

        # Aggregate by category
        if r.category not in by_category:
            by_category[r.category] = {"total": 0, "passed": 0}
        by_category[r.category]["total"] += 1
        if r.passed:
            by_category[r.category]["passed"] += 1

    # Calculate pass rates
    for level in by_level:
        stats = by_level[level]
        stats["pass_rate"] = stats["passed"] / stats["total"]

    for cat in by_category:
        stats = by_category[cat]
        stats["pass_rate"] = stats["passed"] / stats["total"]

    passed = sum(1 for r in results if r.passed)

    return EvalReport(
        agent=agent,
        timestamp=datetime.now().isoformat(),
        total_tasks=len(results),
        passed_tasks=passed,
        pass_rate=passed / len(results) if results else 0,
        by_level=by_level,
        by_category=by_category,
        results=[asdict(r) for r in results]
    )

def main():
    import argparse
    parser = argparse.ArgumentParser(description="Run coding agent evaluations")
    parser.add_argument("--tasks", required=True, help="Path to tasks directory")
    parser.add_argument("--agent", default="autohand", help="Agent command")
    parser.add_argument("--output", default="eval-report.json", help="Output file")
    parser.add_argument("--level", type=int, help="Run only specific level")
    args = parser.parse_args()

    tasks_dir = Path(args.tasks)
    results = []

    for level_dir in sorted(tasks_dir.iterdir()):
        if not level_dir.is_dir():
            continue
        if args.level and f"level-{args.level}" not in level_dir.name:
            continue

        for task_dir in sorted(level_dir.iterdir()):
            if not task_dir.is_dir():
                continue
            print(f"Running: {task_dir.name}")
            result = run_task(task_dir, args.agent)
            results.append(result)
            status = "PASS" if result.passed else "FAIL"
            print(f"  {status} ({result.duration_seconds:.1f}s)")

    report = generate_report(results, args.agent)
    Path(args.output).write_text(json.dumps(asdict(report), indent=2))

    # Print summary
    print(f"\n{'='*50}")
    print(f"Evaluation Report: {report.agent}")
    print(f"{'='*50}")
    print(f"Total: {report.passed_tasks}/{report.total_tasks} ({report.pass_rate:.1%})")
    print(f"\nBy Level:")
    for level, stats in sorted(report.by_level.items()):
        print(f"  Level {level}: {stats['passed']}/{stats['total']} ({stats['pass_rate']:.1%})")
    print(f"\nBy Category:")
    for cat, stats in sorted(report.by_category.items()):
        print(f"  {cat}: {stats['passed']}/{stats['total']} ({stats['pass_rate']:.1%})")

if __name__ == "__main__":
    main()

# Run all evaluations
python run-eval.py --tasks ./evals/tasks --agent autohand

# Run only Level 200 tasks
python run-eval.py --tasks ./evals/tasks --level 200

# Output to specific file
python run-eval.py --tasks ./evals/tasks --output autohand-v1.2.json

# Compare two agents
python run-eval.py --tasks ./evals/tasks --agent autohand --output autohand.json
python run-eval.py --tasks ./evals/tasks --agent other-agent --output other.json
python compare-reports.py autohand.json other.json

{
  "agent": "autohand",
  "timestamp": "2025-01-15T14:32:00Z",
  "total_tasks": 40,
  "passed_tasks": 35,
  "pass_rate": 0.875,
  "by_level": {
    "100": {"total": 10, "passed": 10, "pass_rate": 1.0},
    "200": {"total": 15, "passed": 13, "pass_rate": 0.867},
    "300": {"total": 10, "passed": 9, "pass_rate": 0.9},
    "400": {"total": 5, "passed": 3, "pass_rate": 0.6}
  },
  "by_category": {
    "bug-fixing": {"total": 8, "passed": 8, "pass_rate": 1.0},
    "feature-implementation": {"total": 12, "passed": 10, "pass_rate": 0.833},
    "refactoring": {"total": 8, "passed": 7, "pass_rate": 0.875},
    "testing": {"total": 6, "passed": 5, "pass_rate": 0.833},
    "performance": {"total": 4, "passed": 3, "pass_rate": 0.75},
    "security": {"total": 2, "passed": 2, "pass_rate": 1.0}
  },
  "results": [
    {
      "name": "fix-syntax-error",
      "level": 100,
      "category": "bug-fixing",
      "passed": true,
      "duration_seconds": 12.5
    }
  ]
}

# Generate HTML report
autohand eval report results.json --format html --output report.html

# Generate markdown for documentation
autohand eval report results.json --format markdown --output EVAL_RESULTS.md

# Compare multiple runs
autohand eval compare v1.1.json v1.2.json --output comparison.html

# .github/workflows/eval.yml
name: Agent Evaluation

on:
  release:
    types: [published]
  workflow_dispatch:

jobs:
  evaluate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Autohand
        run: |
          curl -fsSL https://autohand.ai/install.sh | bash
          autohand --version

      - name: Run evaluations
        run: |
          python evals/scripts/run-eval.py \
            --tasks evals/tasks \
            --agent autohand \
            --output eval-results.json
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

      - name: Check pass rate
        run: |
          PASS_RATE=$(jq '.pass_rate' eval-results.json)
          if (( $(echo "$PASS_RATE < 0.85" | bc -l) )); then
            echo "Pass rate $PASS_RATE is below threshold 0.85"
            exit 1
          fi

      - name: Upload results
        uses: actions/upload-artifact@v4
        with:
          name: eval-results
          path: eval-results.json

      - name: Post to Terminal Bench
        if: github.event_name == 'release'
        run: |
          tbench submit eval-results.json \
            --agent-name "Autohand ${{ github.ref_name }}"
        env:
          TBENCH_API_KEY: ${{ secrets.TBENCH_API_KEY }}

{
  "name": "fix-typo",
  "level": 100,
  "category": "bug-fixing",
  "timeout": 60,
  "prompt": "Fix the typo in the error message in src/utils.js",
  "verification": {
    "tests": ["grep -q 'successfully' src/utils.js"],
    "assertions": [
      {"file_contains": ["src/utils.js", "successfully"]}
    ]
  }
}

{
  "name": "add-validation",
  "level": 200,
  "category": "feature-implementation",
  "timeout": 180,
  "prompt": "Add email validation to the user registration endpoint. Return 400 for invalid emails.",
  "verification": {
    "tests": ["npm test"],
    "assertions": [
      {"file_exists": "src/validators/email.js"}
    ]
  }
}

{
  "name": "add-caching",
  "level": 300,
  "category": "performance",
  "timeout": 300,
  "prompt": "Add Redis caching to the product listing API. Cache should expire after 5 minutes. Invalidate on product updates.",
  "verification": {
    "tests": ["npm test", "npm run benchmark"],
    "assertions": [
      {"file_exists": "src/cache/redis.js"},
      {"response_time_under_ms": ["GET /api/products", 50]}
    ]
  }
}

{
  "name": "fix-memory-leak",
  "level": 400,
  "category": "performance",
  "timeout": 600,
  "prompt": "The WebSocket server has a memory leak. Find and fix it. Memory usage should remain stable under sustained load.",
  "verification": {
    "tests": ["npm run load-test"],
    "assertions": [
      {"memory_stable_over_requests": 10000}
    ]
  }
}