diff --git a/.github/workflows/c-cpp.yml b/.github/workflows/c-cpp.yml index b49b2bf..ad7d200 100644 --- a/.github/workflows/c-cpp.yml +++ b/.github/workflows/c-cpp.yml @@ -40,3 +40,17 @@ jobs: if: always() with: files: build/test/*.xml + + text-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Check comments and invisible characters + run: python3 tools/check_text.py + - name: Check commit messages + env: + BASE: ${{ github.event.pull_request.base.sha || github.event.before }} + HEAD: ${{ github.event.pull_request.head.sha || github.sha }} + run: python3 tools/check_commits.py "$BASE..$HEAD" diff --git a/tools/check_commits.py b/tools/check_commits.py new file mode 100755 index 0000000..5c27358 --- /dev/null +++ b/tools/check_commits.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Commit message check. + +Subject and body: only ASCII printable characters, Russian letters and №. +Co-authored-by trailers naming a bot or an AI assistant are forbidden, and so +are commit authors and committers that name one. + +Usage: check_commits.py REVISION_RANGE + check_commits.py --message-file FILE +""" + +import re +import subprocess +import sys +import unicodedata + +from check_text import is_comment_char + +CO_AUTHOR = re.compile(r'^\s*co-authored-by:(.*)$', re.IGNORECASE | re.MULTILINE) +BOT = re.compile(r'\[bot\]|\bbot\b|anthropic|claude|copilot|cursor|codex|openai|chatgpt|gemini|devin|aider', + re.IGNORECASE) + + +def check_message(name, message): + errors = [] + seen = set() + for line_no, line in enumerate(message.split('\n'), 1): + for column, ch in enumerate(line, 1): + if not is_comment_char(ch) and ch not in seen: + seen.add(ch) + errors.append(f'{name}:{line_no}:{column}: character not allowed' + f' U+{ord(ch):04X} {unicodedata.name(ch, "unnamed")}') + for match in CO_AUTHOR.finditer(message): + if BOT.search(match.group(1)): + errors.append(f'{name}: bot co-authorship not allowed:{match.group(1)}') + return errors + + +def check_identity(name, role, identity): + if BOT.search(identity): + return [f'{name}: bot {role} not allowed: {identity}'] + return [] + + +def git(*args): + return subprocess.run(['git', *args], check=True, capture_output=True).stdout.decode('utf-8') + + +def main(argv): + if len(argv) == 2 and argv[0] == '--message-file': + with open(argv[1], encoding='utf-8') as f: + lines = [l for l in f.read().split('\n') if not l.startswith('#')] + errors = check_message('commit message', '\n'.join(lines).strip()) + # In a commit-msg hook git exports the identities of the commit being made. + for role, var in (('author', 'GIT_AUTHOR_IDENT'), ('committer', 'GIT_COMMITTER_IDENT')): + identity = git('var', var).strip().rsplit('>', 1)[0] + '>' + errors.extend(check_identity('commit', role, identity)) + elif len(argv) == 1: + errors = [] + for sha in git('rev-list', argv[0]).split(): + errors.extend(check_message(sha[:12], git('log', '-1', '--format=%B', sha).strip())) + author, committer = git('log', '-1', '--format=%an <%ae>%n%cn <%ce>', sha).strip().split('\n') + errors.extend(check_identity(sha[:12], 'author', author)) + errors.extend(check_identity(sha[:12], 'committer', committer)) + else: + print(__doc__) + return 2 + + for error in errors: + print(error) + if errors: + print(f'{len(errors)} problem(s); commit messages allow only ASCII, Russian letters and №,' + ' bot authors, committers and co-authors are forbidden') + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/tools/check_text.py b/tools/check_text.py new file mode 100755 index 0000000..59263f8 --- /dev/null +++ b/tools/check_text.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Text hygiene check. + +Every text file: no control, format or invisible characters except tab and newline. +Comments in C/C++ and .kum files: only ASCII printable characters, Russian letters and №. + +Usage: check_text.py [--staged] [FILE...] +Without arguments all tracked files are checked; --staged checks files staged for commit. +""" + +import os +import subprocess +import sys +import unicodedata + +C_EXTENSIONS = ('.c', '.cc', '.cpp', '.cxx', '.h', '.hh', '.hpp') +KUM_EXTENSIONS = ('.kum',) + +# Invisible or indistinguishable from ASCII, yet not in the Cc/Cf/Z* categories. +EXTRA_INVISIBLE = {0x034F, 0x115F, 0x1160, 0x2011, 0x3164, 0xFFA0} + + +def is_invisible(ch): + if ch in '\t\n': + return False + code = ord(ch) + if 0xFE00 <= code <= 0xFE0F or 0xE0100 <= code <= 0xE01EF or code in EXTRA_INVISIBLE: + return True + category = unicodedata.category(ch) + if category == 'Zs': + return ch != ' ' + return category in ('Cc', 'Cf', 'Co', 'Zl', 'Zp') + + +def is_comment_char(ch): + return ' ' <= ch <= '~' or ch in '\t\n' or 'А' <= ch <= 'я' or ch in 'ёЁ№' + + +def skip_c_literal(text, i): + start = i + while start > 0 and (text[start - 1].isalnum() or text[start - 1] in "_'."): + start -= 1 + prefix = text[start:i] + if text[i] == "'" and prefix[:1].isdigit(): + return i + 1 # digit separator: 1'000'000 + if text[i] == '"' and prefix in ('R', 'u8R', 'uR', 'UR', 'LR'): + paren = text.find('(', i) + if paren >= 0: + closing = ')' + text[i + 1:paren] + '"' + end = text.find(closing, paren) + return len(text) if end < 0 else end + len(closing) + quote = text[i] + j = i + 1 + while j < len(text) and text[j] != quote and text[j] != '\n': + j += 2 if text[j] == '\\' else 1 + return j + 1 + + +def c_comments(text): + i = 0 + while i < len(text): + if text.startswith('//', i): + end = text.find('\n', i) + end = len(text) if end < 0 else end + yield i, end + i = end + elif text.startswith('/*', i): + end = text.find('*/', i + 2) + end = len(text) if end < 0 else end + 2 + yield i, end + i = end + elif text[i] in '"\'': + i = skip_c_literal(text, i) + else: + i += 1 + + +def kum_comments(text): + i = 0 + while i < len(text): + ch = text[i] + if ch == '|': + end = text.find('\n', i) + end = len(text) if end < 0 else end + yield i, end + i = end + elif ch in '"\'': + # The lexer ends a string at the first matching quote, backslash included. + end = text.find(ch, i + 1) + i = len(text) if end < 0 else end + 1 + else: + i += 1 + + +def comment_spans(path, text): + if path.endswith(C_EXTENSIONS): + return c_comments(text) + if path.endswith(KUM_EXTENSIONS): + return kum_comments(text) + return () + + +def check_file(path): + with open(path, 'rb') as f: + data = f.read() + if b'\0' in data: + return [] + try: + text = data.decode('utf-8') + except UnicodeDecodeError as e: + return [f'{path}: not valid UTF-8 at byte {e.start}'] + + bad = {} + for pos, ch in enumerate(text): + if is_invisible(ch): + bad[pos] = 'invisible character' + for start, end in comment_spans(path, text): + for pos in range(start, end): + if pos not in bad and not is_comment_char(text[pos]): + bad[pos] = 'character not allowed in comment' + + errors = [] + seen = set() + for pos in sorted(bad): + ch = text[pos] + line = text.count('\n', 0, pos) + 1 + if (line, ch) in seen: + continue + seen.add((line, ch)) + column = pos - text.rfind('\n', 0, pos) + name = unicodedata.name(ch, 'unnamed') + errors.append(f'{path}:{line}:{column}: {bad[pos]} U+{ord(ch):04X} {name}') + return errors + + +def git_files(args): + output = subprocess.run(['git', *args, '-z'], check=True, capture_output=True).stdout + return [f for f in output.decode('utf-8').split('\0') if f] + + +def main(argv): + if argv == ['--staged']: + files = git_files(['diff', '--cached', '--name-only', '--diff-filter=ACMR']) + elif argv: + files = argv + else: + files = git_files(['ls-files']) + + errors = [] + for path in files: + if os.path.isfile(path): + errors.extend(check_file(path)) + for error in errors: + print(error) + if errors: + print(f'{len(errors)} problem(s); comments allow only ASCII, Russian letters and №,' + ' invisible characters are forbidden everywhere') + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:]))