breadcrumb-the-shire/bin/fix-md-umlauts.py

#!/usr/bin/env python3
"""
Replace ASCII-fallback German umlauts (ue, ae, oe, ss) with proper umlauts
inside .md prose, while preserving:
  - fenced code blocks (``` ... ```)
  - inline code spans (` ... `)
  - markdown link targets ([label](path)) — only the (path) part is preserved
  - HTML comments (<!-- ... -->)

Usage: python3 bin/fix-md-umlauts.py <file.md> [<file.md> ...]
       --dry-run        print proposed changes per file (default: write in place)
       --check          exit 1 if any file would change (CI mode)
"""

from __future__ import annotations

import re
import sys
from pathlib import Path

# Order matters: longer patterns first to prevent partial matches
# (e.g. "aenderung" must be tried before "aender" so prose-ending stays clean).
SUBSTITUTIONS: list[tuple[str, str]] = [
    # ß / ss
    ("massgeblich", "maßgeblich"),
    ("Massgeblich", "Maßgeblich"),
    ("gemaess", "gemäß"),
    ("Gemaess", "Gemäß"),
    ("regelmaessig", "regelmäßig"),
    ("Regelmaessig", "Regelmäßig"),
    ("groesst", "größt"),
    ("Groesst", "Größt"),
    # ä – longer first
    ("aenderung", "änderung"),
    ("Aenderung", "Änderung"),
    ("aenderbar", "änderbar"),
    ("aendert", "ändert"),
    ("aendern", "ändern"),
    ("Aender", "Änder"),
    ("aender", "änder"),
    ("naechst", "nächst"),
    ("Naechst", "Nächst"),
    ("naemlich", "nämlich"),
    ("Naemlich", "Nämlich"),
    ("waehrend", "während"),
    ("Waehrend", "Während"),
    ("waehl", "wähl"),
    ("Waehl", "Wähl"),
    ("haeufig", "häufig"),
    ("Haeufig", "Häufig"),
    ("spaeter", "später"),
    ("Spaeter", "Später"),
    ("aufzaehl", "aufzähl"),
    ("Aufzaehl", "Aufzähl"),
    ("gefaehr", "gefähr"),
    ("Gefaehr", "Gefähr"),
    ("erlaeut", "erläut"),
    ("Erlaeut", "Erläut"),
    ("bestaetig", "bestätig"),
    ("Bestaetig", "Bestätig"),
    ("Geschaeft", "Geschäft"),
    ("geschaeft", "geschäft"),
    # ü – longer first
    ("ausfuehr", "ausführ"),
    ("Ausfuehr", "Ausführ"),
    ("durchfuehr", "durchführ"),
    ("Durchfuehr", "Durchführ"),
    ("einfuehr", "einführ"),
    ("Einfuehr", "Einführ"),
    ("fuehrun", "führun"),
    ("Fuehrun", "Führun"),
    ("fuehrt", "führt"),
    ("fuehren", "führen"),
    ("Fuehren", "Führen"),
    ("verfuegb", "verfügb"),
    ("Verfuegb", "Verfügb"),
    ("unterstuetz", "unterstütz"),
    ("Unterstuetz", "Unterstütz"),
    ("nuetzlich", "nützlich"),
    ("Nuetzlich", "Nützlich"),
    ("Schluess", "Schlüss"),
    ("schluess", "schlüss"),
    ("wuerde", "würde"),
    ("Wuerde", "Würde"),
    ("wuerden", "würden"),
    ("MUESSEN", "MÜSSEN"),
    ("muessen", "müssen"),
    ("Muessen", "Müssen"),
    ("muess", "müss"),
    ("Muess", "Müss"),
    ("KOENNEN", "KÖNNEN"),
    ("koennen", "können"),
    ("Koennen", "Können"),
    ("koenn", "könn"),
    ("Koenn", "Könn"),
    ("natuerli", "natürli"),
    ("Natuerli", "Natürli"),
    ("tatsaechli", "tatsächli"),
    ("Tatsaechli", "Tatsächli"),
    ("aehnlich", "ähnlich"),
    ("Aehnlich", "Ähnlich"),
    ("standardmaessig", "standardmäßig"),
    ("Standardmaessig", "Standardmäßig"),
    ("gleichmaessig", "gleichmäßig"),
    ("Gleichmaessig", "Gleichmäßig"),
    ("zugaengli", "zugängli"),
    ("Zugaengli", "Zugängli"),
    ("uebersicht", "übersicht"),
    ("Uebersicht", "Übersicht"),
    ("uebersetzun", "übersetzun"),
    ("Uebersetzun", "Übersetzun"),
    ("uebernahme", "übernahme"),
    ("Uebernahme", "Übernahme"),
    ("uebernehm", "übernehm"),
    ("Uebernehm", "Übernehm"),
    ("hoeher", "höher"),
    ("Hoeher", "Höher"),
    ("pruefen", "prüfen"),
    ("Pruefen", "Prüfen"),
    ("prueft", "prüft"),
    ("Prueft", "Prüft"),
    ("Pruefung", "Prüfung"),
    ("pruefung", "prüfung"),
    ("pruef", "prüf"),
    ("Pruef", "Prüf"),
    ("ueber", "über"),
    ("Ueber", "Über"),
    ("fuer", "für"),
    # ö
    ("oeffn", "öffn"),
    ("Oeffn", "Öffn"),
    ("loescht", "löscht"),
    ("Loescht", "Löscht"),
    ("loeschen", "löschen"),
    ("Loeschen", "Löschen"),
    ("loesung", "lösung"),
    ("Loesung", "Lösung"),
    ("moegli", "mögli"),
    ("Moegli", "Mögli"),
    ("moeglich", "möglich"),
    ("Moeglich", "Möglich"),
    ("noetig", "nötig"),
    ("Noetig", "Nötig"),
    ("hoer", "hör"),  # gehör, höher
    ("Hoer", "Hör"),
]

# Regex that splits a line into "preserve" tokens and replaceable text.
# Order matters: longer / more specific tokens first.
TOKENIZER = re.compile(
    r"""
    ( `[^`\n]*` )                               # inline code: `...`
  | ( !?\[[^\]]*\]\([^)]*\) )                   # full markdown link [label](target)
  | ( <!--.*?--> )                              # HTML comment
    """,
    re.VERBOSE | re.DOTALL,
)


def transform_text(text: str) -> str:
    """Apply substitutions to non-preserved spans only."""
    for needle, replacement in SUBSTITUTIONS:
        text = text.replace(needle, replacement)
    return text


def transform_line(line: str) -> str:
    """Tokenise a line into [preserved | text | preserved | text | ...] and replace only text segments."""
    out: list[str] = []
    pos = 0
    for match in TOKENIZER.finditer(line):
        # Replaceable run before the preserved span
        if match.start() > pos:
            out.append(transform_text(line[pos : match.start()]))
        out.append(match.group(0))  # keep preserved span verbatim
        pos = match.end()
    # Tail after last preserved span
    if pos < len(line):
        out.append(transform_text(line[pos:]))
    return "".join(out)


def transform_file(content: str) -> str:
    """Walk lines, skip fenced code blocks, transform the rest line-by-line."""
    out_lines: list[str] = []
    in_fence = False
    fence_marker = ""

    for line in content.splitlines(keepends=True):
        stripped = line.lstrip()

        if in_fence:
            out_lines.append(line)
            if stripped.startswith(fence_marker):
                in_fence = False
                fence_marker = ""
            continue

        # Detect fenced code start (``` or ~~~)
        if stripped.startswith("```"):
            in_fence = True
            fence_marker = "```"
            out_lines.append(line)
            continue
        if stripped.startswith("~~~"):
            in_fence = True
            fence_marker = "~~~"
            out_lines.append(line)
            continue

        out_lines.append(transform_line(line))

    return "".join(out_lines)


def main(argv: list[str]) -> int:
    args = list(argv[1:])
    dry_run = False
    check = False
    while args and args[0].startswith("--"):
        flag = args.pop(0)
        if flag == "--dry-run":
            dry_run = True
        elif flag == "--check":
            check = True
        elif flag == "--":
            break
        else:
            print(f"Unknown flag: {flag}", file=sys.stderr)
            return 2

    if not args:
        print(__doc__.strip(), file=sys.stderr)
        return 2

    changed_files: list[Path] = []
    for raw in args:
        path = Path(raw)
        if not path.is_file():
            print(f"Skipping (not a file): {path}", file=sys.stderr)
            continue
        original = path.read_text(encoding="utf-8")
        transformed = transform_file(original)
        if transformed != original:
            changed_files.append(path)
            if dry_run or check:
                # Brief summary; full diff is what `git diff` shows after a real run.
                added = sum(1 for c in transformed if c in "äöüÄÖÜß") - sum(
                    1 for c in original if c in "äöüÄÖÜß"
                )
                print(f"WOULD CHANGE  {path}  (+{added} umlauts)")
            else:
                path.write_text(transformed, encoding="utf-8")
                print(f"updated  {path}")

    if check and changed_files:
        return 1
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv))