Choose between C and UTF-8 punctuation classification

[jakewilliami]

The first check I do, string.punctuation, is what they do in C. However, this is not the same as checking its character code. I think I should choose between C or UTF-8 (leaning towards the latter). This would be a breaking change and would require changing the major version. I would need to be very clear about this distinction in the docstrings and other documentation.

[jakewilliami]

Currently all ASCII characters tested with our library are congruent with C; see vibe-coded tests below.

ispunct.c (gcc -o ispunct_c ispunct.c)

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <wctype.h>
#include <locale.h>
#include <errno.h>

static void usage(const char *prog) {
    fprintf(stderr, "Usage: %s <char>\n", prog);
    fprintf(stderr, "       %s -n <codepoint>   (decimal or 0x-prefixed hex)\n", prog);
}

int main(int argc, char *argv[]) {
    setlocale(LC_ALL, "");  // needed for iswpunct to behave sensibly on non-ASCII

    long code;

    if (argc == 3 && strcmp(argv[1], "-n") == 0) {
        // Parse as integer, base 0 => auto-detect "0x.." as hex, else decimal
        char *end;
        errno = 0;
        code = strtol(argv[2], &end, 0);

        if (errno != 0 || *end != '\0' || end == argv[2]) {
            fprintf(stderr, "Invalid integer: %s\n", argv[2]);
            return 1;
        }
    } else if (argc == 2 && strlen(argv[1]) == 1) {
        // Single literal character, as before
        code = (unsigned char)argv[1][0];
    } else {
        usage(argv[0]);
        return 1;
    }

    int result;
    if (code >= 0 && code <= 255) {
        // Safe to use the byte-oriented ispunct
        result = ispunct((unsigned char)code);
    } else {
        // Wider code point: use the wide-character version
        result = iswpunct((wint_t)code);
    }

    printf("U+%04lX: %s\n", code, result ? "punctuation" : "not punctuation");

    return 0;
}

ispunct_py (chmod +x ispunct_py`)

#! /usr/bin/env -S uv run --script
# /// script
# dependencies = [
#   "ispunct==1.0.5",
# ]
# ///

import argparse
import sys

from ispunct import ispunct


def main() -> int:
    parser = argparse.ArgumentParser(
        description="Check whether a character is punctuation."
    )
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument(
        "char", nargs="?", help="A single literal character, e.g. '$'"
    )
    group.add_argument(
        "-n",
        dest="codepoint",
        help="Code point as decimal or hex (e.g. 36 or 0x24)",
    )
    args = parser.parse_args()

    if args.codepoint is not None:
        try:
            code = int(args.codepoint, 0)  # base 0 => auto-detect 0x/0/decimal
        except ValueError:
            parser.error(f"Invalid integer: {args.codepoint!r}")

        try:
            c = chr(code)
        except (ValueError, OverflowError):
            parser.error(f"Invalid code point: {args.codepoint!r}")
    else:
        if len(args.char) != 1:
            parser.error("Expected exactly one character")
        c = args.char
        code = ord(c)

    result = ispunct(c)
    print(f"U+{code:04X}: {'punctuation' if result else 'not punctuation'}")

    return 0


if __name__ == "__main__":
    raise SystemExit(main())

ispunct (Bash comparison; chmod +x ispunct)

#! /usr/bin/env bash

set -euo pipefail

PY="./ispunct_py"
C="./ispunct_c"

usage() {
    echo "Usage: $0 [char]" >&2
    echo "       $0 -n <codepoint>       (decimal or 0x-prefixed hex)" >&2
    echo "       $0                      (sweep code points 0-255 and compare)" >&2
    exit 1
}

# Extract just the "punctuation" / "not punctuation" verdict from a line like:
#   U+0024: punctuation
verdict() {
    sed -E 's/^U\+[0-9A-Fa-f]+: //'
}

compare_one() {
    local args=("$@")
    local py_out c_out py_verdict c_verdict

    py_out=$("$PY" "${args[@]}")
    c_out=$("$C" "${args[@]}")

    py_verdict=$(echo "$py_out" | verdict)
    c_verdict=$(echo "$c_out" | verdict)

    if [[ "$py_verdict" == "$c_verdict" ]]; then
        printf "OK    %-30s py=%-16s c=%-16s\n" "${args[*]}" "$py_out" "$c_out"
        return 0
    else
        printf "DIFF  %-30s py=%-16s c=%-16s\n" "${args[*]}" "$py_out" "$c_out"
        return 1
    fi
}

main() {
    local mismatches=0
    local total=0

    if [[ $# -eq 0 ]]; then
        # No argument: sweep code points 0-255
        echo "Sweeping code points 0-255..."
        for ((code = 0; code <= 255; code++)); do
            total=$((total + 1))
            if ! compare_one -n "$code"; then
                mismatches=$((mismatches + 1))
            fi
        done
    elif [[ "$1" == "-n" ]]; then
        [[ $# -eq 2 ]] || usage
        total=1
        compare_one -n "$2" || mismatches=1
    elif [[ $# -eq 1 ]]; then
        total=1
        compare_one "$1" || mismatches=1
    else
        usage
    fi

    echo
    echo "Compared: $total   Mismatches: $mismatches"
    [[ $mismatches -eq 0 ]]
}

main "$@"