AI Coding Assistants: How to Use Them Effectively

import json
import re
import subprocess
import sys
import urllib.request

# Six small functions, each with examples a developer might show and hidden tests that include edge cases
TASKS = {
    "slugify": ("slugify(title: str) -> str: lower-case the title, drop every character that is not a letter, "
                "digit, space or hyphen, and join the remaining words with single hyphens",
                ['slugify("Hello, World!") == "hello-world"', 'slugify("Data Science 101") == "data-science-101"'],
                ['slugify("  Many   spaces  ") == "many-spaces"', 'slugify("C\'est la vie") == "cest-la-vie"',
                 'slugify("already-a-slug") == "already-a-slug"']),
    "parse_duration": ("parse_duration(text: str) -> int: convert a duration such as '1h30m' or '45s' into seconds; "
                       "units are h, m and s; raise ValueError for anything else",
                       ['parse_duration("1h30m") == 5400', 'parse_duration("45s") == 45'],
                       ['parse_duration("1h2m3s") == 3723', 'parse_duration("2h") == 7200',
                        'raises(ValueError, parse_duration, "abc")']),
    "merge_intervals": ("merge_intervals(intervals: list[tuple[int, int]]) -> list[tuple[int, int]]: merge "
                        "overlapping or touching intervals and return them sorted",
                        ['merge_intervals([(1, 3), (2, 6), (8, 10)]) == [(1, 6), (8, 10)]'],
                        ['merge_intervals([(1, 2), (2, 3)]) == [(1, 3)]', 'merge_intervals([]) == []',
                         'merge_intervals([(5, 7), (1, 2)]) == [(1, 2), (5, 7)]']),
    "roman_to_int": ("roman_to_int(numeral: str) -> int: convert a Roman numeral to an integer",
                     ['roman_to_int("XIV") == 14'],
                     ['roman_to_int("MCMXCIV") == 1994', 'roman_to_int("IX") == 9', 'roman_to_int("LVIII") == 58']),
    "top_words": ("top_words(text: str, n: int) -> list[tuple[str, int]]: the n most frequent lower-case words "
                  "with their counts, most frequent first, ties broken alphabetically",
                  ['top_words("a b b c c c", 2) == [("c", 3), ("b", 2)]'],
                  ['top_words("x y z y z", 3) == [("y", 2), ("z", 2), ("x", 1)]', 'top_words("", 2) == []',
                   'top_words("A a b", 1) == [("a", 2)]']),
    "is_valid_isbn10": ("is_valid_isbn10(code: str) -> bool: check an ISBN-10, ignoring hyphens; the last "
                        "character may be X meaning 10",
                        ['is_valid_isbn10("0306406152") is True', 'is_valid_isbn10("0306406153") is False'],
                        ['is_valid_isbn10("0-306-40615-2") is True', 'is_valid_isbn10("080442957X") is True',
                         'is_valid_isbn10("12345") is False']),
}


FENCE = "`" * 3      # the Markdown code fence the model wraps its code in


def ask(prompt):
    body = {"model": "llama3.2", "stream": False, "options": {"temperature": 0, "seed": 42, "num_predict": 400},
            "messages": [{"role": "user", "content": prompt}]}
    req = urllib.request.Request("http://localhost:11434/api/chat", json.dumps(body).encode(),
                                 {"Content-Type": "application/json"})
    with urllib.request.urlopen(req, timeout=900) as r:
        reply = json.load(r)["message"]["content"]
    block = re.search(FENCE + r"(?:python)?\n(.*?)" + FENCE, reply, re.S)
    return block.group(1) if block else reply


def run_tests(code, tests):
    # run the generated code in a separate process; report each failing assertion and why
    harness = code + "\n\ndef raises(err, fn, *args):\n    try:\n        fn(*args)\n    except err:\n        return True\n    return False\n"
    harness += "\nimport json\nfailed = []\n" + "".join(
        f"try:\n    assert {t}\nexcept Exception as e:\n    failed.append([{t!r}, type(e).__name__])\n" for t in tests)
    harness += "print(json.dumps(failed))\n"
    try:
        out = subprocess.run([sys.executable, "-c", harness], capture_output=True, text=True, timeout=20)
        return json.loads(out.stdout.strip().splitlines()[-1])
    except Exception:
        return [[t, "did not run"] for t in tests]


labels = ["description only", "with examples", "after one repair"]
totals = dict.fromkeys(labels, 0)
print(f"{'function':<17}" + "".join(f"{label:>19}" for label in labels) + "   (tests passed)")
shown_repair = None
for name, (spec, examples, hidden) in TASKS.items():
    tests = examples + hidden
    results = []
    for shown in ([], examples):
        prompt = f"Write a Python function {spec}.\n"
        if shown:
            prompt += "It must pass these examples:\n" + "\n".join(f"assert {t}" for t in shown) + "\n"
        prompt += "Reply with the code only, in one python code block."
        code = ask(prompt)
        results.append(run_tests(code, tests))
    # the coding-agent loop: show the model which tests failed, and ask for a fix
    failed = results[-1]
    if failed:
        report = "\n".join(f"assert {t}   -> {err}" for t, err in failed)
        code = ask(f"This Python function fails some tests.\n\n{FENCE}python\n{code}\n{FENCE}\n\nFailing tests:\n{report}\n\n"
                   "Fix the function so all tests pass. Reply with the code only, in one python code block.")
        results.append(run_tests(code, tests))
        if shown_repair is None:
            shown_repair = (name, report, results[-1])
    else:
        results.append(failed)
    for label, failed_tests in zip(labels, results):
        totals[label] += len(tests) - len(failed_tests)
    print(f"{name:<17}" + "".join(f"{f'{len(tests) - len(r)} of {len(tests)}':>19}" for r in results))

n_tests = sum(len(e) + len(h) for _, e, h in TASKS.values())
print(f"{'total':<17}" + "".join(f"{f'{totals[label]} of {n_tests}':>19}" for label in labels))

if shown_repair:
    name, report, still = shown_repair
    print(f"\nfeedback sent for {name}:\n{report}")
    print("still failing after the repair:", [t for t, _ in still] or "none")

function            description only      with examples   after one repair   (tests passed)
slugify                       5 of 5             5 of 5             5 of 5
parse_duration                1 of 5             1 of 5             1 of 5
merge_intervals               3 of 4             3 of 4             4 of 4
roman_to_int                  4 of 4             4 of 4             4 of 4
top_words                     4 of 4             4 of 4             4 of 4
is_valid_isbn10               2 of 5             2 of 5             2 of 5
total                       19 of 27           19 of 27           20 of 27

feedback sent for parse_duration:
assert parse_duration("1h30m") == 5400   -> ValueError
assert parse_duration("45s") == 45   -> ValueError
assert parse_duration("1h2m3s") == 3723   -> ValueError
assert parse_duration("2h") == 7200   -> ValueError
still failing after the repair: ['parse_duration("1h30m") == 5400', 'parse_duration("45s") == 45', 'parse_duration("1h2m3s") == 3723', 'parse_duration("2h") == 7200']