address review: revert _PyTokenizer_From* changes

Author: skirpichev

Parser/pegen.c

@@ -1055,8 +1055,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
                              PyObject **interactive_src, PyArena *arena)
 {
     int parser_flags = compute_parser_flags(flags);
-    struct tok_state *tok = _PyTokenizer_FromFile(fp, enc, ps1, ps2,
-                                                  parser_flags & PyPARSE_BARRY_AS_BDFL);
+    struct tok_state *tok = _PyTokenizer_FromFile(fp, enc, ps1, ps2);
     if (tok == NULL) {
         if (PyErr_Occurred()) {
             _PyTokenizer_raise_init_error(filename_ob);
@@ -1067,6 +1066,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
         }
         return NULL;
     }
+    tok->barry_as_bdfl = parser_flags & PyPARSE_BARRY_AS_BDFL;
     if (!tok->fp || ps1 != NULL || ps2 != NULL ||
         PyUnicode_CompareWithASCIIString(filename_ob, "<stdin>") == 0) {
         tok->fp_interactive = 1;
@@ -1114,9 +1114,9 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen
 
     struct tok_state *tok;
     if (flags != NULL && flags->cf_flags & PyCF_IGNORE_COOKIE) {
-        tok = _PyTokenizer_FromUTF8(str, exec_input, 0, parser_flags & PyPARSE_BARRY_AS_BDFL);
+        tok = _PyTokenizer_FromUTF8(str, exec_input, 0);
     } else {
-        tok = _PyTokenizer_FromString(str, exec_input, 0, parser_flags & PyPARSE_BARRY_AS_BDFL);
+        tok = _PyTokenizer_FromString(str, exec_input, 0);
     }
     if (tok == NULL) {
         if (PyErr_Occurred()) {
@@ -1128,6 +1128,7 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen
         }
         return NULL;
     }
+    tok->barry_as_bdfl = parser_flags & PyPARSE_BARRY_AS_BDFL;
     // This transfers the ownership to the tokenizer
     tok->filename = Py_NewRef(filename_ob);
     tok->module = Py_XNewRef(module);

Parser/tokenizer/file_tokenizer.c

@@ -371,8 +371,7 @@ tok_underflow_file(struct tok_state *tok)
 /* Set up tokenizer for file */
 struct tok_state *
 _PyTokenizer_FromFile(FILE *fp, const char* enc,
-                      const char *ps1, const char *ps2,
-                      int barry_as_bdfl)
+                      const char *ps1, const char *ps2)
 {
     struct tok_state *tok = _PyTokenizer_tok_new();
     if (tok == NULL)
@@ -402,7 +401,6 @@ _PyTokenizer_FromFile(FILE *fp, const char* enc,
         }
         tok->decoding_state = STATE_NORMAL;
     }
-    tok->barry_as_bdfl = barry_as_bdfl;
     return tok;
 }
 
@@ -459,7 +457,7 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
     if (fp == NULL) {
         return NULL;
     }
-    tok = _PyTokenizer_FromFile(fp, NULL, NULL, NULL, 0);
+    tok = _PyTokenizer_FromFile(fp, NULL, NULL, NULL);
     if (tok == NULL) {
         fclose(fp);
         return NULL;

Parser/tokenizer/string_tokenizer.c

@@ -128,7 +128,7 @@ decode_str(const char *input, int single, struct tok_state *tok, int preserve_cr
 
 /* Set up tokenizer for string */
 struct tok_state *
-_PyTokenizer_FromString(const char *str, int exec_input, int preserve_crlf, int barry_as_bdfl)
+_PyTokenizer_FromString(const char *str, int exec_input, int preserve_crlf)
 {
     struct tok_state *tok = _PyTokenizer_tok_new();
     char *decoded;
@@ -144,6 +144,5 @@ _PyTokenizer_FromString(const char *str, int exec_input, int preserve_crlf, int
     tok->buf = tok->cur = tok->inp = decoded;
     tok->end = decoded;
     tok->underflow = &tok_underflow_string;
-    tok->barry_as_bdfl = barry_as_bdfl;
     return tok;
 }

Parser/tokenizer/tokenizer.h

@@ -3,11 +3,11 @@
 
 #include "Python.h"
 
-struct tok_state *_PyTokenizer_FromString(const char *, int, int, int);
-struct tok_state *_PyTokenizer_FromUTF8(const char *, int, int, int);
+struct tok_state *_PyTokenizer_FromString(const char *, int, int);
+struct tok_state *_PyTokenizer_FromUTF8(const char *, int, int);
 struct tok_state *_PyTokenizer_FromReadline(PyObject*, const char*, int, int);
 struct tok_state *_PyTokenizer_FromFile(FILE *, const char*,
-                                              const char *, const char *, int);
+                                              const char *, const char *);
 
 #define tok_dump _Py_tok_dump
 

Parser/tokenizer/utf8_tokenizer.c

@@ -28,8 +28,7 @@ tok_underflow_string(struct tok_state *tok) {
 
 /* Set up tokenizer for UTF-8 string */
 struct tok_state *
-_PyTokenizer_FromUTF8(const char *str, int exec_input, int preserve_crlf,
-                      int barry_as_bdfl)
+_PyTokenizer_FromUTF8(const char *str, int exec_input, int preserve_crlf)
 {
     struct tok_state *tok = _PyTokenizer_tok_new();
     char *translated;
@@ -52,6 +51,5 @@ _PyTokenizer_FromUTF8(const char *str, int exec_input, int preserve_crlf,
     tok->buf = tok->cur = tok->inp = translated;
     tok->end = translated;
     tok->underflow = &tok_underflow_string;
-    tok->barry_as_bdfl = barry_as_bdfl;
     return tok;
 }