Exercise

from pathlib import Path
from workshop.llm import chat

INSTRUCTIONS = ("Extract the bibliographic record from this catalogue card into the JSON schema. A card has these lines: "
                "the creator (surname first); the title; the imprint as 'Place : Publisher, year'; the extent; sometimes an identifier. "
                "Copy each value as written into its own field. Use an empty string for a field the card does not state; "
                "never guess a place or year.")

def extract(card_text: str, model: str = MODEL) -> Record:
    response = chat(
        [{"role": "user", "content": f"{INSTRUCTIONS}\n\nCard:\n{card_text}"}],
        model=model, format=Record.model_json_schema(), options={"num_predict": 300},
    )
    return Record.model_validate_json(response.message.content)

card = Path("data/cards/card-01.txt").read_text(encoding="utf-8")
print(card)
print(extract(card).model_dump())

Goethe, Johann Wolfgang von
Die Leiden des jungen Werthers : Roman
Leipzig : Weygandsche Buchhandlung, 1774
2 Teile in 1 Bd. ; 8°

{'title': 'Die Leiden des jungen Werthers : Roman', 'creators': ['Goethe, Johann Wolfgang von'], 'place': 'Leipzig', 'publisher': 'Weygandsche Buchhandlung', 'date': '1774', 'language': 'de', 'identifier': ''}

card3 = Path("data/cards/card-03.txt").read_text(encoding="utf-8")
print(card3)
rec3 = extract(card3)
print(rec3.model_dump())
print("place should be empty; the model said:", repr(rec3.place))

Beauvoir, Simone de
Le deuxième sexe. I, Les faits et les mythes
Gallimard, 1949
395 p. ; 19 cm
Collection Folio essais

{'title': 'Le deuxième sexe. I, Les faits et les mythes', 'creators': ['Beauvoir, Simone de'], 'place': 'Gallimard', 'publisher': 'Gallimard', 'date': '1949', 'language': 'en', 'identifier': ''}
place should be empty; the model said: 'Gallimard'

from pydantic import ValidationError, model_validator

class CheckedRecord(Record):
    @model_validator(mode="after")
    def place_is_not_publisher(self):
        if self.place and self.publisher and self.place.strip().lower() == self.publisher.strip().lower():
            raise ValueError("place equals publisher; the card probably has no place")
        return self

def extract_checked(card_text: str, model: str = MODEL, retries: int = 1) -> CheckedRecord:
    messages = [{"role": "user", "content": f"{INSTRUCTIONS}\n\nCard:\n{card_text}"}]
    last_error: ValidationError | None = None
    for attempt in range(retries + 1):
        response = chat(messages, model=model, format=CheckedRecord.model_json_schema(), options={"num_predict": 300})
        try:
            return CheckedRecord.model_validate_json(response.message.content)
        except ValidationError as err:
            last_error = err
            print(f"attempt {attempt + 1}: validation failed: {err.errors()[0]['msg']}")
            messages += [{"role": "assistant", "content": response.message.content},
                         {"role": "user", "content": f"The record failed validation: {err.errors()[0]['msg']}. Fix it and return the JSON again. Use an empty string where the card is silent."}]
    raise last_error

print(extract_checked(card3).model_dump())

attempt 1: validation failed: Value error, place equals publisher; the card probably has no place

{'title': 'Le deuxième sexe. I, Les faits et les mythes', 'creators': ['Beauvoir, Simone de'], 'place': '', 'publisher': 'Gallimard', 'date': '1949', 'language': 'en', 'identifier': ''}

import csv, json
gold = {g["file"]: g for g in json.loads(Path("data/cards/cards-gold.json").read_text(encoding="utf-8"))}
rows, correct, total = [], 0, 0
for path in sorted(Path("data/cards").glob("card-*.txt")):
    rec = extract(path.read_text(encoding="utf-8"))
    rows.append({"file": path.name, **rec.model_dump()})
    g = gold[path.name]
    for field in ("title", "date", "place", "publisher", "language", "identifier"):
        total += 1
        correct += (str(getattr(rec, field) or "").strip().lower() == str(g[field] or "").strip().lower())
Path("runs").mkdir(exist_ok=True)
with open("runs/cards-extracted.csv", "w", newline="", encoding="utf-8") as fh:
    writer = csv.DictWriter(fh, fieldnames=list(rows[0].keys())); writer.writeheader(); writer.writerows(rows)
print(f"{len(rows)} cards, field accuracy {correct}/{total} = {correct/total:.0%}; wrote runs/cards-extracted.csv")

20 cards, field accuracy 97/120 = 81%; wrote runs/cards-extracted.csv

from collections import Counter
misses = Counter()
for row in rows:
    g = gold[row["file"]]
    for field in ("title", "date", "place", "publisher", "language", "identifier"):
        if str(row[field] or "").strip().lower() != str(g[field] or "").strip().lower():
            misses[field] += 1
print(misses.most_common())

[('title', 7), ('language', 7), ('identifier', 4), ('place', 4), ('publisher', 1)]