Skip to content

Commit 5d4f953

Browse files
miss-islingtonserhiy-storchaka
authored andcommitted
[3.11] gh-98820: Fix quadratic time in csv.Sniffer for quoted fields (GH-154867)
(cherry picked from commit b30c7fa) (cherry picked from commit 39dac55)
1 parent 274de10 commit 5d4f953

3 files changed

Lines changed: 18 additions & 5 deletions

File tree

Lib/csv.py

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -215,12 +215,16 @@ def _guess_quote_and_delimiter(self, data, delimiters):
215215
this way.
216216
"""
217217

218+
# The body of a quoted field ends at the first quote which is
219+
# not doubled, as it does for a reader. A lazy ".*?" scans to
220+
# the end of the sample instead, from every start: quadratically.
221+
body = r'(?:(?P=quote){2}|(?!(?P=quote)).)*+'
218222
matches = []
219-
for restr in (r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?P=delim)', # ,".*?",
220-
r'(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?P<delim>[^\w\n"\'])(?P<space> ?)', # ".*?",
221-
r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?:$|\n)', # ,".*?"
222-
r'(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?:$|\n)'): # ".*?" (no delim, no space)
223-
regexp = re.compile(restr, re.DOTALL | re.MULTILINE)
223+
for restr in (r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\'])%s(?P=quote)(?P=delim)', # ,"...",
224+
r'(?:^|\n)(?P<quote>["\'])%s(?P=quote)(?P<delim>[^\w\n"\'])(?P<space> ?)', # "...",
225+
r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\'])%s(?P=quote)(?:$|\n)', # ,"..."
226+
r'(?:^|\n)(?P<quote>["\'])%s(?P=quote)(?:$|\n)'): # "..." (no delim, no space)
227+
regexp = re.compile(restr % body, re.DOTALL | re.MULTILINE)
224228
matches = regexp.findall(data)
225229
if matches:
226230
break

Lib/test/test_csv.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1357,6 +1357,13 @@ def test_sniff_space_delimiter(self):
13571357
self.assertEqual(dialect.delimiter, ' ')
13581358
self.assertIs(dialect.doublequote, False)
13591359

1360+
def test_sniff_quoted_single_column(self):
1361+
# gh-98820: this sample used to take minutes.
1362+
sniffer = csv.Sniffer()
1363+
sample = '"abcdefghijklmnopqrstuvwxyz"\n' * 10000
1364+
with self.assertRaisesRegex(csv.Error, "Could not determine delimiter"):
1365+
sniffer.sniff(sample, delimiters=',:|\t')
1366+
13601367

13611368
class NUL:
13621369
def write(s, *args):
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix quadratic time in :meth:`csv.Sniffer.sniff` for a sample which contains
2+
quoted fields, in particular for a single column of quoted fields.

0 commit comments

Comments
 (0)