gh-153569: normalize prepared tokenizer input once

Prepared string input is split, normalized, and appended one line at a time,
while the source object keeps line tables that the lexer no longer queries.

Normalize prepared input once and transfer its storage to the tokenizer source.
The reader scans lines from that byte buffer and keeps the implicit-newline flag
with the prepared input.

Author: pablogsal

Modules/_testinternalcapi/tokenizer.c

@@ -12,64 +12,30 @@ check(int condition, const char *message)
     return -1;
 }
 
-static int
-check_system_error(int failed, const char *message)
-{
-    if (!failed || !PyErr_ExceptionMatches(PyExc_SystemError)) {
-        PyErr_SetString(PyExc_AssertionError, message);
-        return -1;
-    }
-    PyErr_Clear();
-    return 0;
-}
-
 static PyObject *
 test_tokenizer_source(PyObject *Py_UNUSED(module),
                       PyObject *Py_UNUSED(args))
 {
-    _PyTok_SourceText source;
-    _PyTok_SourceInit(&source);
+    _PyTok_SourceText source = {0};
 
-    if (check_system_error(
-            _PyTok_SourceAppendLine(&source, "", 0, 0) < 0,
-            "accepted empty source line") < 0 ||
-            check_system_error(
-                _PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0,
-                "accepted multiple source lines") < 0 ||
-            check_system_error(
-                _PyTok_SourceAppendLine(&source, "a", 1, 1) < 0,
-                "accepted missing implicit newline") < 0 ||
-            check(_PyTok_SourceAppendLine(
-                      &source, "alpha\n", 6, 0) == 0,
+    if (check(_PyTok_SourceAppend(
+                      &source, "alpha\n", 6) == 0,
                   "wrong first source offset") < 0 ||
-            check(_PyTok_SourceAppendLine(
-                      &source, "\xce\xb2\n", 3, 1) == 6,
+            check(_PyTok_SourceAppend(
+                      &source, "\xce\xb2\n", 3) == 6,
                   "wrong second source offset") < 0 ||
-            check(!_PyTok_SourceLineIsImplicit(&source, 1) &&
-                      _PyTok_SourceLineIsImplicit(&source, 2),
-                  "wrong implicit newline flags") < 0) {
-        goto error;
-    }
-
-    Py_ssize_t view_len;
-    const char *view = _PyTok_SourceSpanView(
-        &source, _PyTok_SpanFromBounds(6, 8), &view_len);
-    if (check(view != NULL && view_len == 2 &&
-                  memcmp(view, "\xce\xb2", 2) == 0,
-              "wrong source span view") < 0 ||
-            check_system_error(
-                _PyTok_SourceSpanView(
-                    &source, _PyTok_SpanFromBounds(0, source.len + 1),
-                    &view_len) == NULL,
-                "accepted invalid source span") < 0) {
+            check(source.len == 9 &&
+                      memcmp(source.bytes, "alpha\n\xce\xb2\n", 9) == 0,
+                  "wrong source contents") < 0) {
         goto error;
     }
 
     _PyTok_SourceClear(&source);
-    if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 ||
-            check_system_error(
-                _PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0,
-                "appended after unterminated source line") < 0) {
+    if (check(_PyTok_SourceAppend(&source, "tail", 4) == 0,
+              "cannot reuse source") < 0 ||
+            check(source.len == 4 &&
+                      memcmp(source.bytes, "tail\0", 5) == 0,
+                  "wrong reused source contents") < 0) {
         goto error;
     }
 

Parser/lexer/state.h

@@ -74,7 +74,6 @@ struct tok_state {
     _PyTok_Off buf_offset; /* Logical offset of buf[0]. */
     const char *start;  /* Start of current token if not NULL */
     int done;           /* E_OK normally, E_EOF at EOF, otherwise error code */
-    /* NB If done != E_OK, cur must be == inp!!! */
     FILE *fp;           /* Rest of input; NULL if tokenizing a string */
     int indent;         /* Current indentation index */
     int indstack[MAXINDENT];            /* Stack of indents */
@@ -146,7 +145,7 @@ _PyLexer_BufferOffset(const struct tok_state *tok, const char *position)
     return tok->buf_offset + offset;
 }
 
-static inline char *
+static inline const char *
 _PyLexer_BufferPointer(const struct tok_state *tok, _PyTok_Off offset)
 {
     assert(tok->buf != NULL);

Parser/tokenizer/decoder.c

@@ -103,7 +103,9 @@ _PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, int preserve_crlf,
     }
     result[write] = '\0';
     *out_len = write;
-    *implicit_newline = implicit;
+    if (implicit_newline != NULL) {
+        *implicit_newline = implicit;
+    }
     return result;
 }
 
@@ -297,51 +299,35 @@ static int
 store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len,
                       int preserve_crlf, int add_final_newline)
 {
-    Py_ssize_t pos = 0;
-    while (pos < len) {
-        Py_ssize_t raw_line_len;
-        if (preserve_crlf) {
-            const char *newline = memchr(data + pos, '\n', len - pos);
-            raw_line_len = newline == NULL
-                ? len - pos : newline - data - pos + 1;
-        }
-        else {
-            raw_line_len = raw_line_length(data + pos, len - pos);
-        }
-        int terminated = preserve_crlf
-            ? data[pos + raw_line_len - 1] == '\n'
-            : data[pos + raw_line_len - 1] == '\n' ||
-              data[pos + raw_line_len - 1] == '\r';
-        int add_newline = add_final_newline &&
-            pos + raw_line_len == len && !terminated;
-        int normalize = add_newline ||
-            (!preserve_crlf &&
-             memchr(data + pos, '\r', raw_line_len) != NULL);
-
-        const char *line = data + pos;
-        Py_ssize_t line_len = raw_line_len;
-        char *normalized = NULL;
-        int implicit = 0;
-        if (normalize) {
-            normalized = _PyTok_NormalizeNewlines(
-                line, line_len, preserve_crlf, add_newline,
-                &line_len, &implicit);
-            if (normalized == NULL) {
-                tok->done = E_NOMEM;
-                return -1;
-            }
-            line = normalized;
+    int normalize = (!preserve_crlf && memchr(data, '\r', len) != NULL) ||
+        (add_final_newline && len > 0 && data[len - 1] != '\n');
+    char *normalized = NULL;
+    int implicit = 0;
+    if (normalize) {
+        normalized = _PyTok_NormalizeNewlines(
+            data, len, preserve_crlf, add_final_newline,
+            &len, &implicit);
+        if (normalized == NULL) {
+            tok->done = E_NOMEM;
+            return -1;
         }
-        _PyTok_Off appended = _PyTok_SourceAppendLine(
-            &tok->source, line, line_len, implicit);
-        PyMem_Free(normalized);
-        if (appended < 0) {
-            tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
-                ? E_NOMEM : E_ERROR;
+    }
+    char *stored = normalized;
+    if (stored == NULL && len > 0) {
+        stored = _PyTok_CopyBytes(data, len);
+        if (stored == NULL) {
+            tok->done = E_NOMEM;
             return -1;
         }
-        pos += raw_line_len;
     }
+    assert(tok->source.bytes == NULL);
+    assert(tok->source.len == 0 && tok->source.cap == 0);
+    tok->source = (_PyTok_SourceText){
+        .bytes = stored,
+        .len = len,
+        .cap = stored != NULL ? len + 1 : 0,
+    };
+    tok->reader->prepared_implicit_newline = implicit;
     return 0;
 }
 

Parser/tokenizer/reader.c

@@ -191,20 +191,22 @@ chunk_is_line(const _PyTok_Chunk *chunk)
 static _PyTok_ReadResult
 next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk)
 {
-    int lineno = tok->lineno + 1;
-    if (lineno > tok->source.nlines) {
+    if (tok->source.len == 0) {
+        return _PYTOK_READ_EOF;
+    }
+    assert(tok->source.bytes != NULL);
+    const char *source_end = tok->source.bytes + tok->source.len;
+    if (tok->inp == source_end) {
         return _PYTOK_READ_EOF;
     }
     const char *start = tok->inp;
-    const char *newline = memchr(
-        start, '\n', tok->source.bytes + tok->source.len - start);
-    _PyTok_Off end = newline != NULL
-        ? newline - tok->source.bytes + 1 : tok->source.len;
+    const char *newline = memchr(start, '\n', source_end - start);
+    const char *end = newline != NULL ? newline + 1 : source_end;
     chunk->data = (char *)start;
-    chunk->len = tok->source.bytes + end - start;
+    chunk->len = end - start;
     chunk->ownership = _PYTOK_CHUNK_BORROWED;
-    chunk->implicit_newline = _PyTok_SourceLineIsImplicit(
-        &tok->source, lineno);
+    chunk->implicit_newline = end == source_end &&
+        tok->reader->prepared_implicit_newline;
     return _PYTOK_READ_LINE;
 }
 
@@ -557,10 +559,9 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk)
     }
     chunk->data = _PyTok_NormalizeNewlines(
         decoded.data, decoded.len, 0, 0,
-        &chunk->len, &chunk->implicit_newline);
+        &chunk->len, NULL);
     _PyTok_ChunkClear(&decoded);
     if (chunk->data == NULL) {
-        PyErr_NoMemory();
         tok->done = E_NOMEM;
         return _PYTOK_READ_ERROR;
     }
@@ -647,12 +648,15 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
         }
         return 0;
     }
+    if (tok->lineno == INT_MAX) {
+        _PyTok_ChunkClear(&chunk);
+        PyErr_SetString(PyExc_OverflowError,
+                        "too many tokenizer source lines");
+        tok->done = E_ERROR;
+        return 0;
+    }
 
     Py_ssize_t scan_len = chunk.len;
-    if (kind == _PYTOK_READER_INTERACTIVE &&
-            chunk.implicit_newline) {
-        scan_len--;
-    }
     if (streaming) {
         if (reset_buffer) {
             reset_streaming_buffer(tok);
@@ -680,9 +684,8 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
         if (!reset_buffer && source_will_grow) {
             offsets = save_buffer_offsets(tok, tok->source.bytes);
         }
-        _PyTok_Off source_start = _PyTok_SourceAppendLine(
-            &tok->source, chunk.data, chunk.len,
-            chunk.implicit_newline);
+        _PyTok_Off source_start = _PyTok_SourceAppend(
+            &tok->source, chunk.data, chunk.len);
         if (source_start < 0) {
             _PyTok_ChunkClear(&chunk);
             tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)

Parser/tokenizer/reader_internal.h

@@ -60,6 +60,7 @@ typedef struct _PyTok_Reader {
     unsigned char file_eof;
     unsigned char decoder_finalized;
     unsigned char stop_interactive;
+    unsigned char prepared_implicit_newline;
 } _PyTok_Reader;
 
 struct tok_state;