From 70b150a366e4d8e426f45d24a421fd70f833a8c5 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Fri, 15 Oct 2021 01:35:53 +0900 Subject: [3.10] no-issue: Make silence about warning '_POSIX_C_SOURCE redefined' (GH-28948) (GH-28951) --- Modules/expat/xmltok.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index f2b6b40606..5b93571874 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -42,16 +42,16 @@ USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include -#include /* memcpy */ -#include - #ifdef _WIN32 # include "winconfig.h" #endif #include +#include +#include /* memcpy */ +#include + #include "expat_external.h" #include "internal.h" #include "xmltok.h" -- cgit v1.2.1 From 0bff4ccbfd3297b0adf690655d3e9ddb0033bc69 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 14 Oct 2021 10:02:20 -0700 Subject: [3.10] bpo-45461: Fix IncrementalDecoder and StreamReader in the "unicode-escape" codec (GH-28939) (GH-28943) They support now splitting escape sequences between input chunks. Add the third parameter "final" in codecs.unicode_escape_decode(). It is True by default to match the former behavior. (cherry picked from commit c96d1546b11b4c282a7e21737cb1f5d16349656d) Co-authored-by: Serhiy Storchaka --- Doc/data/python3.10.abi | 18477 ++++++++++++------- Include/cpython/unicodeobject.h | 10 +- Lib/encodings/unicode_escape.py | 9 +- Lib/test/test_codecs.py | 50 +- .../2021-10-14-00-19-02.bpo-45461.4LB_tJ.rst | 2 + Modules/_codecsmodule.c | 13 +- Modules/clinic/_codecsmodule.c.h | 18 +- Objects/unicodeobject.c | 49 +- Parser/string_parser.c | 2 +- 9 files changed, 11842 insertions(+), 6788 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2021-10-14-00-19-02.bpo-45461.4LB_tJ.rst diff --git a/Doc/data/python3.10.abi b/Doc/data/python3.10.abi index 1dca3e1c2e..0e678f3aed 100644 --- a/Doc/data/python3.10.abi +++ b/Doc/data/python3.10.abi @@ -1046,6 +1046,7 @@ + @@ -1248,7 +1249,8 @@ - + + @@ -1647,333 +1649,1037 @@ + + + + - + - - + + - - - + + + - - - - + + + + - - - + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - + + + + + + + + - + - + - - - - + + + + - + - + - - - - - + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + + + - + - + - + @@ -1998,12 +2704,12 @@ - + - - + + @@ -2021,7 +2727,7 @@ - + @@ -2033,7 +2739,7 @@ - + @@ -2057,7 +2763,7 @@ - + @@ -2075,7 +2781,7 @@ - + @@ -2117,12 +2823,12 @@ - + - - + + - + @@ -2137,13 +2843,13 @@ - + - + @@ -2164,7 +2870,7 @@ - + @@ -2179,12 +2885,12 @@ - - + + - - - + + + @@ -2195,22 +2901,22 @@ - + - + - + - + - + - + @@ -2225,10 +2931,10 @@ - + - + @@ -2236,16 +2942,16 @@ - - + + - + - - - + + + @@ -2254,16 +2960,16 @@ - + - + - + @@ -2278,61 +2984,61 @@ - + - - - - - - - + + + + + + + - + - - - - - + + + + + - + - + - + - + - + - + - - - - - + + + + + - + @@ -2374,10 +3080,10 @@ - + - + @@ -2407,13 +3113,13 @@ - + - + @@ -2422,24 +3128,24 @@ - + - + - + - + - + @@ -2452,59 +3158,107 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + - - + + - - - + + + - - - - + + + + - - - - + + + + - + - + - - - + + + - - - + + + - - - - + + + + + - - + @@ -2513,695 +3267,931 @@ - + - - + + - - - - - + + + + + - - - + + + - - - - + + + + - - - - + + + + - + - - + + - + - - + + - - - - + + + + - + - - + + - - - + + + - - + + - + - - + + - - + + - + - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - + + - - - - - - + + - - + + - - - - - - + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - + + - - - + + + - + - + - + - - + + - + - + - - + + - - + + - - + + - - - + + + - - - - - - - - + + + - - - + + + - - - - + + + + - + - + - - + + - - + + - - - - + + + + - - - - - + + + + + - - - + + + - - - - + + + + - - - - + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - - + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - - + + + - + - - - - - - + + + + + + - + - + - + - - - + + + - - + + - + - + - + - + - + - + - + - - - - - - - - + - + - - + + - + - - + + - + - - + + - + - + - - + + - + - - + + - - + + - + - + - - - + + + - - - - + + + + - - - - - - - - + + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - - - - - + - - + + - + + + + + + + + + + + + + + - + - + - + - - + + + + + + + + + + + + + + - + @@ -3226,42 +4216,237 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - + + + - - - + + + - + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -3269,22 +4454,22 @@ - + - + - + - + - + - + @@ -3292,253 +4477,298 @@ - + - - - - + + + + - - + + - - - - - - - - + + - - - + + + - + - + - - + + - - - - - - - - - - - + + - - - + + + - - - + + + - - - + + + - + - + - + - - + + - + - + - + - + - - + + - - - - + + + + - + - + - + - + - + - + - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + - + - + - + - + - - + + - + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -3547,91 +4777,91 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -3639,7 +4869,7 @@ - + @@ -3651,13 +4881,13 @@ - + - + @@ -3666,73 +4896,73 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -3741,7 +4971,7 @@ - + @@ -3789,16 +5019,16 @@ - + - + - + - + @@ -3810,13 +5040,13 @@ - + - + @@ -3831,7 +5061,7 @@ - + @@ -3861,10 +5091,10 @@ - + - + @@ -3876,7 +5106,7 @@ - + @@ -3887,10 +5117,10 @@ - + - + @@ -3899,7 +5129,7 @@ - + @@ -3910,7 +5140,7 @@ - + @@ -3930,7 +5160,7 @@ - + @@ -3939,7 +5169,7 @@ - + @@ -3976,10 +5206,10 @@ - + - + @@ -3996,20 +5226,20 @@ - + - + - - + + - + @@ -4022,24 +5252,24 @@ - + - + - + - + - + - + @@ -4064,13 +5294,13 @@ - + - + @@ -4084,44 +5314,44 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -4134,7 +5364,7 @@ - + @@ -4146,13 +5376,13 @@ - + - + @@ -4172,16 +5402,16 @@ - + - + - + @@ -4189,27 +5419,27 @@ - + - + - + - + - + - + @@ -4224,19 +5454,19 @@ - + - + - + - + - + @@ -4244,10 +5474,10 @@ - + - + @@ -4258,69 +5488,69 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -4332,7 +5562,7 @@ - + @@ -4347,43 +5577,43 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -4395,7 +5625,7 @@ - + @@ -4410,7 +5640,7 @@ - + @@ -4434,7 +5664,7 @@ - + @@ -4446,20 +5676,20 @@ - + - + - + - + - + @@ -4472,7 +5702,7 @@ - + @@ -4481,22 +5711,22 @@ - + - + - + - + - + @@ -4505,16 +5735,16 @@ - + - + - + - + @@ -4529,67 +5759,67 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -4598,7 +5828,7 @@ - + @@ -4614,24 +5844,24 @@ - + - + - + - + - - + + @@ -4639,22 +5869,22 @@ - - + + - + - + - + - + @@ -4662,21 +5892,21 @@ - + - + - + - + - + @@ -4686,12 +5916,12 @@ - + - + @@ -4699,7 +5929,7 @@ - + @@ -4712,12 +5942,12 @@ - + - + @@ -4726,15 +5956,15 @@ - + - - + + @@ -4742,7 +5972,7 @@ - + @@ -4766,7 +5996,7 @@ - + @@ -4774,7 +6004,7 @@ - + @@ -4782,7 +6012,7 @@ - + @@ -4793,16 +6023,16 @@ - + - + - + - + @@ -4817,7 +6047,7 @@ - + @@ -4827,12 +6057,12 @@ - + - - + + @@ -4840,25 +6070,25 @@ - + - + - + - + @@ -4866,24 +6096,24 @@ - + - + - + - + - + @@ -4899,12 +6129,12 @@ - + - + @@ -4912,7 +6142,7 @@ - + @@ -4920,27 +6150,27 @@ - + - + - + - + @@ -4948,7 +6178,7 @@ - + @@ -4956,33 +6186,33 @@ - + - + - + - + - + - + - + - + @@ -4992,709 +6222,709 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -5704,205 +6934,238 @@ - + - + - + - + - - - - - + + + + + - - - + + + - - - - + + + + - + - - + + - - - - - - - - - - + + + + - - - - + + + + - - - - + + + + - + - - - + + + - + - - + + - + - - + + - - + + - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + - + - + - - - + + + - - + + - + - - + + - - - + + + - + - + - - - + + + - - - + + + - - + + - + - + - + - - - - - - + - + - + + + + + + - - - + + + - - + + - - + + + + + @@ -5911,13 +7174,13 @@ - + - + - + @@ -5925,246 +7188,303 @@ - - - - - - - - - - - + + - - + + - - + + - - - - - - - - + + + + - - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - - - - - + + + + + - - - + + + - - + + - - + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -6179,25 +7499,25 @@ - + - + - + - + - + @@ -6232,6 +7552,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -6242,28 +7604,28 @@ - - - - + + + + - - - + + + - - - + + + - + - + @@ -6272,312 +7634,407 @@ - + - + - - - - - + + + + + - - - - + + + + - - - - + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + - + - + - - - - - - - - - - - - - - - - + - + - + - + - + - + - - - - - + - - - - - - - - - - - - - - + + - + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + @@ -6590,100 +8047,145 @@ - - - + + + - - - + + + - + - - - + + + - + + + + + - - + + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - + - - + + - - + + - + - + - - - + + + - - + + - - + + - - - - + + + + - - - + + + - + - + - - + + + + + + + + + + + + + + + + + + + + + + + + @@ -6692,55 +8194,51 @@ - + - + - + - - + + - - - - + + + + - - + + - - - - - + - - + + - + - + @@ -6748,6 +8246,82 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -6761,20 +8335,20 @@ - + - - + + - + - + @@ -6783,21 +8357,54 @@ - - - - + + + + - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -6805,71 +8412,80 @@ - - + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - - - + + + + + + + + + + + + + @@ -6879,7 +8495,7 @@ - + @@ -6891,13 +8507,13 @@ - + - + @@ -6906,73 +8522,73 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -6981,7 +8597,7 @@ - + @@ -7030,23 +8646,50 @@ - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -7054,13 +8697,25 @@ - - - + + + - - + + + + + + + + + + + + + + @@ -7068,104 +8723,115 @@ - - + + - - + + - - + + - - + + - - - - - + + + + + - - - - + + + + - - - + + + - - - - + + + + - - - - + + + + - - - + + + - - + + - - + + - + + + + + + + + + + + + + + + + - + - + - - - + + + - - - + + + - + - + - + - - - - - + @@ -7173,36 +8839,36 @@ - - - + + + - - - + + + - - + + - - - + + + - + - + @@ -7211,13 +8877,13 @@ - + - + - + @@ -7241,155 +8907,174 @@ - - - - + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - + - + - - - - - + - + - - - - - - - - - - - + + + - - + + - - - + + + - - - + + + - + - - + + - - + + - + - - + + - - + + - - - - - - + + - + - + - + - + - - - - - - - - - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -7403,79 +9088,74 @@ - - - + + + - - - - - - - - + + + - - - + + + - + - + - - - - + + + + - + - - + + - - - + + + - + - + - - + + - + - - - + + + - + - + - + - + @@ -7485,169 +9165,216 @@ - + - - - - + + + + - - + + - - + + - - + + - - + + - - + + - - - - + + + + - - - - + + + + - - - + + + - - - - - - - - - - + + + + - + - - - - - - - - - - - - - - + - - + + - + - - - - + + + + - - - + + + - - - + + + - - - - - - - + - - - + + + - + - - - - - - - - - - - - + - - - + + + - - + + - - + + - - + + - + - - - + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + @@ -7657,103 +9384,157 @@ - - - + + + - - - - + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - - + + - - - - - + - + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - + + - + - - - - + + + + - - - + + + - - + + + + + - - + + - - + + - - + + - - + + @@ -7766,7 +9547,7 @@ - + @@ -7775,181 +9556,195 @@ - + - + - + - + - + - + - + - + - + - + - - - - - + - - - - - - + + - - - + + + - + - + - + - + - - - + + + - - + + - - + + - + - - + + - + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + + + + - - + + - + - + - - + + - - + + - - + + - - - + + + - - - - - - + + - + - - + + - + - + - + - - - + + + - + @@ -7958,29 +9753,29 @@ - + - + - + - + - + - + @@ -7989,273 +9784,342 @@ - + - - + + - + - + - - + + - - + + - - + + - - + + - + - - + + - - + + - - + + - - - - - - + + - - - - + + + + - - - - + + + + - - - - - + + + + + - - - + + + - - - - - + + + + + - - + + - + - - - - - - + + - + - - - - + + + + - - - + + + - + - + - - + + - + - - - + + + - + - - + + - + - + - - + + - - - - + + + + - + - - - - - - + - + - + - - - - + + + + - - - - + + + + - - + + - - + + - - - - - - - - - - - - - - + + - - + + - + - + - - + + - - + + - - + + - + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -8264,10 +10128,10 @@ - + - + @@ -8277,37 +10141,33 @@ - - - - - + - + - + - - + + - + - + - + - + @@ -8324,44 +10184,44 @@ - - + + - + - + - + - + - - + + - + - + - + - + - + @@ -8376,242 +10236,311 @@ - + - + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - + - + - + - + - - - - - - - - - + + + + + + + + + - - - + + + - - + + + + - - + + + - - - - - - - - - - - - - - + + - - - - - + + - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + - + - + - - + + - + - - - + + + - + - - + + - - + + - - + + + + + - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - - + + - + - + - - - + + + - - - + + + - - - + + + - - + + - - + + - - + + - - + + + + + + + + + + + - + - + - + - - + + - + - + - - + + - + - + - - + + - + - - - + + + - - - - + + + + - + @@ -8623,7 +10552,7 @@ - + @@ -8639,38 +10568,44 @@ - + - + - + - + - + - + - - + + - - - + + + - - - - + + + + - - + + + + + + + + @@ -8678,97 +10613,89 @@ - - + + - - - - + + + + - + - + - - + + - - - - + + + + - - - + + + - - - - - - + + - + + + + + - - - + + + + - - - - - - - - - + + + + - - - - - - - - + + + + - - - + + + - - - - + + + + - + - + - + @@ -8776,10 +10703,10 @@ - + - + @@ -8788,895 +10715,1184 @@ - + - + - - + + - - + + - - - + + + - - + + - + - - - - - - + - - - - + + + + - - - + + + - + - - - - - + - - - - + + + - + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + + - - + + - + - - + + - + - - + + - + - + - + - + - - + + - + - - + + - + - - + + - + - + - + - + - - + + - - - - + + + + - - - + + + - + - + - - - - + + + + - - - + + + - + - - - + + + - + - + - + - - - - + + + + - - + + - + - - + + - - + + - - - - - - + + - + - + - + - + - - - + + + - - - + + + - - + + - - + + - - - + + + - + - + - + - - - - - - - + - - + + - + - + - - - - + + + + - + - + - - - - + + + + - + - - - - - - - - + - + - - - - - - - - - + - - - + + + - - - + + + - + - + - - + + - + - + - + - + - - + + - - + + - - - + + + - - - - - - - - - - - - + - - + + - - + + - - + + - - - - - - - + + - - + + - - - - - - + + + + + + - - - - - - + + + + + + - + - + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - - - + + + - - - + + + - - - + + + @@ -9688,47 +11904,47 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -9737,51 +11953,59 @@ - + - - - - - + + + + + + + + + + + + + - - + + - - + + - - - + + + - - - + + + - + - + - + @@ -9797,160 +12021,376 @@ - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - - + + - - - - - - - + + + + + + + - + - - + + - - + + - + - - + + - + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + - - - + + + - - - - + + + + - - - - + + + + - - + + - - + + - + - + - - - + + + - - + + - - + + - - - + + + - - + + - + - + - + - + - + - + - + @@ -9959,16 +12399,16 @@ - + - + - + - + @@ -9982,32 +12422,32 @@ - + - - + + - - - + + + - - + + - - - + + + - - + + - + - + @@ -10016,91 +12456,91 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -10108,7 +12548,7 @@ - + @@ -10120,13 +12560,13 @@ - + - + @@ -10135,73 +12575,73 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -10210,7 +12650,7 @@ - + @@ -10257,16 +12697,16 @@ - + - + - + - + @@ -10278,13 +12718,13 @@ - + - + @@ -10299,7 +12739,7 @@ - + @@ -10315,290 +12755,489 @@ - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + - - - - - - - - - + + + + - - + + - + - + - - + + - - + + - + - + - + - + - - + + - + - + - + - + - + - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - + + - - + + - - + + - - - - - - + + - - - - - - - + + - + - + - + - + - + - - - - + + + + - - - - + + + + - + - + - + - + - + - + - + - + - - - + + + - - - + + + - + - + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + + + - - - + + + - - - - + + + + @@ -10639,10 +13278,10 @@ - + - + @@ -10656,16 +13295,16 @@ - + - + - + - + @@ -10788,7 +13427,7 @@ - + @@ -10814,10 +13453,10 @@ - + - + @@ -10834,16 +13473,16 @@ - + - + - + - + @@ -10854,16 +13493,16 @@ - + - + - + - + @@ -10995,10 +13634,10 @@ - + - + @@ -11008,7 +13647,7 @@ - + @@ -11109,10 +13748,10 @@ - + - + @@ -11129,14 +13768,14 @@ - + - + @@ -11170,18 +13809,18 @@ - + - + - - + + @@ -11199,10 +13838,10 @@ - + - + @@ -11216,23 +13855,23 @@ - + - + - + - + - + @@ -11242,7 +13881,7 @@ - + @@ -11261,8 +13900,8 @@ - - + + @@ -11331,7 +13970,7 @@ - + @@ -11393,7 +14032,7 @@ - + @@ -11437,10 +14076,10 @@ - + - + @@ -11458,7 +14097,7 @@ - + @@ -11473,10 +14112,10 @@ - + - + @@ -11501,16 +14140,16 @@ - + - + - + - + @@ -11562,10 +14201,10 @@ - + - + @@ -11575,7 +14214,7 @@ - + @@ -11607,10 +14246,10 @@ - + - + @@ -11635,7 +14274,7 @@ - + @@ -11664,10 +14303,10 @@ - + - + @@ -11681,16 +14320,16 @@ - + - + - + - + @@ -11717,7 +14356,7 @@ - + @@ -11737,10 +14376,10 @@ - + - + @@ -11754,23 +14393,23 @@ - + - + - + - + - + @@ -11783,7 +14422,7 @@ - + @@ -11795,15 +14434,15 @@ - + - + - + @@ -11828,7 +14467,7 @@ - + @@ -11838,7 +14477,7 @@ - + @@ -11868,901 +14507,312 @@ - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + - - + + - + - - + + - - + + - - + + - + - - + + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - - + + - - - + + + - - - - - - + + + + + + - - - - + + + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - + - + - + - + - + - + - + - + + + + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - + + - - - - + + - - - - - - - - - + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + @@ -12774,13 +14824,13 @@ - + - + @@ -12789,73 +14839,73 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -12864,7 +14914,7 @@ - + @@ -12909,2762 +14959,5581 @@ - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + - - + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + - - - - - - - + + + - - - - - - - + + - - - - - + + - - - - + + - - - - - + + - - - - - + + - - - - - + + - - - + + - - - + + - - - + + - - - + + + + + + - - - - - + + - - + + - - + + - - - - + + - - - - - + + - - - - + + - - - - + + - - - - - + + - - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - + + - - - + + - - + + - - - + + + - - - - + + + + + + - - - - + + + + - - - + + + + - - - + + + + + + + + - - - + + + + + + + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - + + + - - - - - + + + + + + - - - - - - - + + + + + + - - - - - + + + + + + + - - - - - + + + + + + + - - - - - + + + + + + - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - + + + + + + + - - - - + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + - + - - + + + + - + - - - - + + + + + - - - - - - - - + + + + + - - - - + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + - - - - - + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - + + - - + + - + - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - + - + + + + + + - + - - - - - - + - - - - - - - - - - - + - - + + - - - + + + + + + + + + + + + + - + - - - - - - - + - - - - - - - + + + + + - - - - + + + - - - + + + + - - - + + + + + + + + + + + + + + + + - - - - + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - - - - + + + + - - - - + + + + + - - - + + + + + + - - - + + - - - + + - - - - - + + + + - - - - + + + + - - - + + + - - - - + + + - - - + + - - + + - - - + + - - + + - - + + - - + + - - + + - - - + + - - - + + - - - - + + - - - - - + + - - - - - + + - - - - - + + - - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - + + - - + + - - + + + + - - + + - - + + - - - - - - - - - + + - - + + - - - + + + + + + + + + + + + + + + + + + + - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + + + + + - - - - + + + + + + + - - - + + + + + - - - + + + + - - - - + + + + + - - - - - + + + + + - - - + + + - - - - + + + - - - + + + - - - - - - - - - + + + - - + + + + + - - + + - - + + - - + + + + - - + + + + + - - + + + + - - - + + + + - - - + + + + + - - - + + + + - - + + + - - + + + - - - - + + + - - - + + + - - - + + + - - - + + - - + + + - - - + + - - - + + + + + - - - - - + + - - + + - - + + - - - - - - + + - - + + - - - + + - - - + + - - - + + - - - + + - - - - + + - - - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - + + - - - + + - - - + + - - + + - - - + + - - - + + + + - - - + + + + - - - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - - - + + + - - - + + + + + - - - - - - - - - - - - - - - - - - - - - + + + + + - - - - + + - - + + - - - - + + - - + + - - - - - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - - - + + - - - - + + - - - - + + - - - + + - - - + + - - - - - + + + + + + + - - - - - + + + + + - - - - + + + + + - - - - + + + + + - - - - + + + + + + + - - - + + + + + + + - - - + + + + - - - - - - - + + + + - - - - - - + + + + - - - - - + + + + - - - + + + + - - - + + + + + - - - - - + + - - - - - + + + + + + + - - - - + + + + + + + + + - - - - - - + + + + + + - - - - + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - + + + + - - - - - - + + - - - - + + - - - - + + - - - - + + - - - - + + - - - - - + + + - - - + + + - - + + + - - - + + + - - + + - - - + + - - - + + - - + + - - - + + - - - - + + - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + - - - - + + - - - + + - - - + + - - - - + + - - - - + + - - - + + - - + + - - - + + - - + + - - - + + - - - - + + - - - + + - - - + + - - + + - - - + + - - - - + + - - - + + - - - + + - - + + - - - - + + - - - - + + - - - + + - - - - - - - - - - - - + + - - - + + - - + + - - + + - - + + - - + + - - - - + + - - + + - - - - - - - + + - - - - - - + + - - - - - + + - - - + + - - - - - - - - - - - + + - - - - - + + - - - - - - - - - + + - - - - - - - - - + + - - - - - + + - - - - + + - - - - - - - - - - - - - - - - + + - - - - - - + + - - - - + + - - - - + + - - - - - - - - - - + + - - - + + - - - - + + - - - + + - - - + + - - - - - - - - - + + - - - - - - - + + - - - - - - - + + - - - - - - - + + - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - - + + - - - + + - - - + + - - - - + + - - - - + + - - - + + - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - - + + - - - - + + - - - - + + - - - - - + + - - - - - + + - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - - - - - - - - - - - - + + - - - - + + - - - - + + - - - - - - - - + + - - + + - - + + - - - + + - - - - - - - + + - - - - - - - - - - - - - - + + - - - - + + - - + + - - - - - + + - - - + + - - - + + - - - - + + - - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - - + + - - + + - - + + - - + + - - + + - - + + - - - - + + - - - + + - - - - + + - - - - - - + + - - - - - - + + - - - + + - - - + + - - + + - - + + - - + + - - + + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + - + + + + + + + + + + + + + - + + + + + + + + + + - + + + + + + + + + + - + + + + + + + + + + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - - + + - - - - - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -15672,65 +20541,161 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + - + + + + - + - + - + - + - + - + diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 0b5067f5d3..a004895346 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -834,12 +834,20 @@ PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF16( /* --- Unicode-Escape Codecs ---------------------------------------------- */ +/* Variant of PyUnicode_DecodeUnicodeEscape that supports partial decoding. */ +PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeStateful( + const char *string, /* Unicode-Escape encoded string */ + Py_ssize_t length, /* size of string */ + const char *errors, /* error handling */ + Py_ssize_t *consumed /* bytes consumed */ +); /* Helper for PyUnicode_DecodeUnicodeEscape that detects invalid escape chars. */ -PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscape( +PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeInternal( const char *string, /* Unicode-Escape encoded string */ Py_ssize_t length, /* size of string */ const char *errors, /* error handling */ + Py_ssize_t *consumed, /* bytes consumed */ const char **first_invalid_escape /* on return, points to first invalid escaped char in string. */ diff --git a/Lib/encodings/unicode_escape.py b/Lib/encodings/unicode_escape.py index 817f93265a..9b1ce99b33 100644 --- a/Lib/encodings/unicode_escape.py +++ b/Lib/encodings/unicode_escape.py @@ -21,15 +21,16 @@ class IncrementalEncoder(codecs.IncrementalEncoder): def encode(self, input, final=False): return codecs.unicode_escape_encode(input, self.errors)[0] -class IncrementalDecoder(codecs.IncrementalDecoder): - def decode(self, input, final=False): - return codecs.unicode_escape_decode(input, self.errors)[0] +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def _buffer_decode(self, input, errors, final): + return codecs.unicode_escape_decode(input, errors, final) class StreamWriter(Codec,codecs.StreamWriter): pass class StreamReader(Codec,codecs.StreamReader): - pass + def decode(self, input, errors='strict'): + return codecs.unicode_escape_decode(input, errors, False) ### encodings module API diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 328a47b2e3..153912b241 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -114,7 +114,7 @@ class ReadTest(MixInCheckStateHandling): q = Queue(b"") r = codecs.getreader(self.encoding)(q) result = "" - for (c, partialresult) in zip(input.encode(self.encoding), partialresults): + for (c, partialresult) in zip(input.encode(self.encoding), partialresults, strict=True): q.write(bytes([c])) result += r.read() self.assertEqual(result, partialresult) @@ -125,7 +125,7 @@ class ReadTest(MixInCheckStateHandling): # do the check again, this time using an incremental decoder d = codecs.getincrementaldecoder(self.encoding)() result = "" - for (c, partialresult) in zip(input.encode(self.encoding), partialresults): + for (c, partialresult) in zip(input.encode(self.encoding), partialresults, strict=True): result += d.decode(bytes([c])) self.assertEqual(result, partialresult) # check that there's nothing left in the buffers @@ -135,7 +135,7 @@ class ReadTest(MixInCheckStateHandling): # Check whether the reset method works properly d.reset() result = "" - for (c, partialresult) in zip(input.encode(self.encoding), partialresults): + for (c, partialresult) in zip(input.encode(self.encoding), partialresults, strict=True): result += d.decode(bytes([c])) self.assertEqual(result, partialresult) # check that there's nothing left in the buffers @@ -2341,7 +2341,11 @@ class TypesTest(unittest.TestCase): (r"\x5c\x55\x30\x30\x31\x31\x30\x30\x30\x30", 10)) -class UnicodeEscapeTest(unittest.TestCase): +class UnicodeEscapeTest(ReadTest, unittest.TestCase): + encoding = "unicode-escape" + + test_lone_surrogates = None + def test_empty(self): self.assertEqual(codecs.unicode_escape_encode(""), (b"", 0)) self.assertEqual(codecs.unicode_escape_decode(b""), ("", 0)) @@ -2428,6 +2432,44 @@ class UnicodeEscapeTest(unittest.TestCase): self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10)) self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10)) + def test_partial(self): + self.check_partial( + "\x00\t\n\r\\\xff\uffff\U00010000", + [ + '', + '', + '', + '\x00', + '\x00', + '\x00\t', + '\x00\t', + '\x00\t\n', + '\x00\t\n', + '\x00\t\n\r', + '\x00\t\n\r', + '\x00\t\n\r\\', + '\x00\t\n\r\\', + '\x00\t\n\r\\', + '\x00\t\n\r\\', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff\U00010000', + ] + ) class RawUnicodeEscapeTest(unittest.TestCase): def test_empty(self): diff --git a/Misc/NEWS.d/next/Library/2021-10-14-00-19-02.bpo-45461.4LB_tJ.rst b/Misc/NEWS.d/next/Library/2021-10-14-00-19-02.bpo-45461.4LB_tJ.rst new file mode 100644 index 0000000000..c1c4ed1ace --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-14-00-19-02.bpo-45461.4LB_tJ.rst @@ -0,0 +1,2 @@ +Fix incremental decoder and stream reader in the "unicode-escape" codec. +Previously they failed if the escape sequence was split. diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index 2e8cb97fe7..fc74127ce5 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -489,17 +489,20 @@ _codecs_utf_32_ex_decode_impl(PyObject *module, Py_buffer *data, _codecs.unicode_escape_decode data: Py_buffer(accept={str, buffer}) errors: str(accept={str, NoneType}) = None + final: bool(accept={int}) = True / [clinic start generated code]*/ static PyObject * _codecs_unicode_escape_decode_impl(PyObject *module, Py_buffer *data, - const char *errors) -/*[clinic end generated code: output=3ca3c917176b82ab input=8328081a3a569bd6]*/ + const char *errors, int final) +/*[clinic end generated code: output=b284f97b12c635ee input=6154f039a9f7c639]*/ { - PyObject *decoded = PyUnicode_DecodeUnicodeEscape(data->buf, data->len, - errors); - return codec_tuple(decoded, data->len); + Py_ssize_t consumed = data->len; + PyObject *decoded = _PyUnicode_DecodeUnicodeEscapeStateful(data->buf, data->len, + errors, + final ? NULL : &consumed); + return codec_tuple(decoded, consumed); } /*[clinic input] diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 43378f94f9..a7086dd6e1 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -1063,7 +1063,7 @@ exit: } PyDoc_STRVAR(_codecs_unicode_escape_decode__doc__, -"unicode_escape_decode($module, data, errors=None, /)\n" +"unicode_escape_decode($module, data, errors=None, final=True, /)\n" "--\n" "\n"); @@ -1072,7 +1072,7 @@ PyDoc_STRVAR(_codecs_unicode_escape_decode__doc__, static PyObject * _codecs_unicode_escape_decode_impl(PyObject *module, Py_buffer *data, - const char *errors); + const char *errors, int final); static PyObject * _codecs_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) @@ -1080,8 +1080,9 @@ _codecs_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_ PyObject *return_value = NULL; Py_buffer data = {NULL, NULL}; const char *errors = NULL; + int final = 1; - if (!_PyArg_CheckPositional("unicode_escape_decode", nargs, 1, 2)) { + if (!_PyArg_CheckPositional("unicode_escape_decode", nargs, 1, 3)) { goto exit; } if (PyUnicode_Check(args[0])) { @@ -1122,8 +1123,15 @@ _codecs_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_ _PyArg_BadArgument("unicode_escape_decode", "argument 2", "str or None", args[1]); goto exit; } + if (nargs < 3) { + goto skip_optional; + } + final = _PyLong_AsInt(args[2]); + if (final == -1 && PyErr_Occurred()) { + goto exit; + } skip_optional: - return_value = _codecs_unicode_escape_decode_impl(module, &data, errors); + return_value = _codecs_unicode_escape_decode_impl(module, &data, errors, final); exit: /* Cleanup for data */ @@ -2801,4 +2809,4 @@ exit: #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=557c3b37e4c492ac input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9e9fb1d5d81577e0 input=a9049054013a1b77]*/ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 11a3a6d0a4..e660834b47 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6408,9 +6408,10 @@ PyUnicode_AsUTF16String(PyObject *unicode) static _PyUnicode_Name_CAPI *ucnhash_capi = NULL; PyObject * -_PyUnicode_DecodeUnicodeEscape(const char *s, +_PyUnicode_DecodeUnicodeEscapeInternal(const char *s, Py_ssize_t size, const char *errors, + Py_ssize_t *consumed, const char **first_invalid_escape) { const char *starts = s; @@ -6423,6 +6424,9 @@ _PyUnicode_DecodeUnicodeEscape(const char *s, *first_invalid_escape = NULL; if (size == 0) { + if (consumed) { + *consumed = 0; + } _Py_RETURN_UNICODE_EMPTY(); } /* Escaped strings will always be longer than the resulting @@ -6473,7 +6477,7 @@ _PyUnicode_DecodeUnicodeEscape(const char *s, /* \ - Escapes */ if (s >= end) { message = "\\ at end of string"; - goto error; + goto incomplete; } c = (unsigned char) *s++; @@ -6527,7 +6531,10 @@ _PyUnicode_DecodeUnicodeEscape(const char *s, count = 8; message = "truncated \\UXXXXXXXX escape"; hexescape: - for (ch = 0; count && s < end; ++s, --count) { + for (ch = 0; count; ++s, --count) { + if (s >= end) { + goto incomplete; + } c = (unsigned char)*s; ch <<= 4; if (c >= '0' && c <= '9') { @@ -6540,12 +6547,9 @@ _PyUnicode_DecodeUnicodeEscape(const char *s, ch += c - ('A' - 10); } else { - break; + goto error; } } - if (count) { - goto error; - } /* when we get here, ch is a 32-bit unicode character */ if (ch > MAX_UNICODE) { @@ -6572,14 +6576,20 @@ _PyUnicode_DecodeUnicodeEscape(const char *s, } message = "malformed \\N character escape"; - if (s < end && *s == '{') { + if (s >= end) { + goto incomplete; + } + if (*s == '{') { const char *start = ++s; size_t namelen; /* look for the closing brace */ while (s < end && *s != '}') s++; + if (s >= end) { + goto incomplete; + } namelen = s - start; - if (namelen && s < end) { + if (namelen) { /* found a name. look it up in the unicode database */ s++; ch = 0xffffffff; /* in case 'getcode' messes up */ @@ -6605,6 +6615,11 @@ _PyUnicode_DecodeUnicodeEscape(const char *s, continue; } + incomplete: + if (consumed) { + *consumed = startinpos; + break; + } error: endinpos = s-starts; writer.min_length = end - s + writer.pos; @@ -6633,12 +6648,14 @@ _PyUnicode_DecodeUnicodeEscape(const char *s, } PyObject * -PyUnicode_DecodeUnicodeEscape(const char *s, +_PyUnicode_DecodeUnicodeEscapeStateful(const char *s, Py_ssize_t size, - const char *errors) + const char *errors, + Py_ssize_t *consumed) { const char *first_invalid_escape; - PyObject *result = _PyUnicode_DecodeUnicodeEscape(s, size, errors, + PyObject *result = _PyUnicode_DecodeUnicodeEscapeInternal(s, size, errors, + consumed, &first_invalid_escape); if (result == NULL) return NULL; @@ -6653,6 +6670,14 @@ PyUnicode_DecodeUnicodeEscape(const char *s, return result; } +PyObject * +PyUnicode_DecodeUnicodeEscape(const char *s, + Py_ssize_t size, + const char *errors) +{ + return _PyUnicode_DecodeUnicodeEscapeStateful(s, size, errors, NULL); +} + /* Return a Unicode-Escape string version of the Unicode object. */ PyObject * diff --git a/Parser/string_parser.c b/Parser/string_parser.c index fb37d37553..dcd298cb35 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -115,7 +115,7 @@ decode_unicode_with_escapes(Parser *parser, const char *s, size_t len, Token *t) s = buf; const char *first_invalid_escape; - v = _PyUnicode_DecodeUnicodeEscape(s, len, NULL, &first_invalid_escape); + v = _PyUnicode_DecodeUnicodeEscapeInternal(s, len, NULL, NULL, &first_invalid_escape); if (v != NULL && first_invalid_escape != NULL) { if (warn_invalid_escape_sequence(parser, *first_invalid_escape, t) < 0) { -- cgit v1.2.1 From 4641afef661e6a22bc64194bd334b161c95edfe2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 14 Oct 2021 21:23:39 +0300 Subject: [3.10] bpo-45467: Fix IncrementalDecoder and StreamReader in the "raw-unicode-escape" codec (GH-28944) (GH-28952) They support now splitting escape sequences between input chunks. Add the third parameter "final" in codecs.raw_unicode_escape_decode(). It is True by default to match the former behavior. (cherry picked from commit 39aa98346d5dd8ac591a7cafb467af21c53f1e5d) Co-authored-by: Serhiy Storchaka --- Doc/data/python3.10.abi | 586 +++++++++++---------- Include/cpython/unicodeobject.h | 9 + Lib/encodings/raw_unicode_escape.py | 9 +- Lib/test/test_codecs.py | 35 +- .../2021-10-14-13-31-19.bpo-45467.Q7Ma6A.rst | 2 + Modules/_codecsmodule.c | 13 +- Modules/clinic/_codecsmodule.c.h | 18 +- Objects/unicodeobject.c | 64 ++- 8 files changed, 412 insertions(+), 324 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2021-10-14-13-31-19.bpo-45467.Q7Ma6A.rst diff --git a/Doc/data/python3.10.abi b/Doc/data/python3.10.abi index 0e678f3aed..94dcf74411 100644 --- a/Doc/data/python3.10.abi +++ b/Doc/data/python3.10.abi @@ -1249,6 +1249,7 @@ + @@ -2357,7 +2358,7 @@ - + @@ -3376,10 +3377,10 @@ - + - + @@ -3991,7 +3992,7 @@ - + @@ -4648,7 +4649,7 @@ - + @@ -7573,7 +7574,7 @@ - + @@ -8021,7 +8022,7 @@ - + @@ -9066,7 +9067,7 @@ - + @@ -10097,7 +10098,7 @@ - + @@ -10480,7 +10481,7 @@ - + @@ -10892,351 +10893,358 @@ - + - + - - + + - - + + - - + + - - - + + + - - - - - + + + + + - - + + - - - - + + + + - - - - + + + + - - - + + + - - - + + + - - - + + + - - - - + + + + - - + + - - - - + + + + - - - - + + + + - - - - - + + + + + - - - - + + + + - - - - + + + + - - + + - - + + - + - - - + + + - - - + + + - + - - - + + + - - - + + + - - - + + + - - - + + + - - - - - + + + + + - - - - - + + + + + - - - - + + + + - - - + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - + + + + + - - - - - - - - - - + + + + + + + + + + - - - - - + + + + + - - - + + + - - + + - - - - + + + + - - - - - + + + + + - - - + + + - - - - - + + + + + - - - - + + + + - - + + - - - - - + + + + + - + - - - - + + + + - - - - + + + + - + - - - - + + + + + + + + + + - - - - + + + - - - + + - - + + + + - - - - + + + + + - - - + + + - - + + - - - - + + + + - - - - - + + + + + @@ -11316,9 +11324,9 @@ - - - + + + @@ -11334,9 +11342,9 @@ - - - + + + @@ -11362,9 +11370,9 @@ - - - + + + @@ -11592,15 +11600,15 @@ - - - - + + + + - - - + + + @@ -11631,14 +11639,14 @@ - - - + + + - - - + + + @@ -11647,30 +11655,30 @@ - - - + + + - - - + + + - - - + + + - - - - - + + + + + - - + + @@ -11704,28 +11712,28 @@ - + - + - + - + - + - + - + @@ -11809,37 +11817,37 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -12936,7 +12944,7 @@ - + @@ -18810,7 +18818,7 @@ - + @@ -20053,10 +20061,10 @@ - + - + @@ -20070,7 +20078,7 @@ - + @@ -20082,10 +20090,10 @@ - + - + @@ -20106,7 +20114,7 @@ - + diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index a004895346..0761f01efb 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -841,6 +841,7 @@ PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeStateful( const char *errors, /* error handling */ Py_ssize_t *consumed /* bytes consumed */ ); + /* Helper for PyUnicode_DecodeUnicodeEscape that detects invalid escape chars. */ PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeInternal( @@ -865,6 +866,14 @@ Py_DEPRECATED(3.3) PyAPI_FUNC(PyObject*) PyUnicode_EncodeRawUnicodeEscape( Py_ssize_t length /* Number of Py_UNICODE chars to encode */ ); +/* Variant of PyUnicode_DecodeRawUnicodeEscape that supports partial decoding. */ +PyAPI_FUNC(PyObject*) _PyUnicode_DecodeRawUnicodeEscapeStateful( + const char *string, /* Unicode-Escape encoded string */ + Py_ssize_t length, /* size of string */ + const char *errors, /* error handling */ + Py_ssize_t *consumed /* bytes consumed */ +); + /* --- Latin-1 Codecs ----------------------------------------------------- */ PyAPI_FUNC(PyObject*) _PyUnicode_AsLatin1String( diff --git a/Lib/encodings/raw_unicode_escape.py b/Lib/encodings/raw_unicode_escape.py index 2b919b40d3..46c8e070dd 100644 --- a/Lib/encodings/raw_unicode_escape.py +++ b/Lib/encodings/raw_unicode_escape.py @@ -21,15 +21,16 @@ class IncrementalEncoder(codecs.IncrementalEncoder): def encode(self, input, final=False): return codecs.raw_unicode_escape_encode(input, self.errors)[0] -class IncrementalDecoder(codecs.IncrementalDecoder): - def decode(self, input, final=False): - return codecs.raw_unicode_escape_decode(input, self.errors)[0] +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def _buffer_decode(self, input, errors, final): + return codecs.raw_unicode_escape_decode(input, errors, final) class StreamWriter(Codec,codecs.StreamWriter): pass class StreamReader(Codec,codecs.StreamReader): - pass + def decode(self, input, errors='strict'): + return codecs.raw_unicode_escape_decode(input, errors, False) ### encodings module API diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 153912b241..f7310fbb14 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -2471,7 +2471,11 @@ class UnicodeEscapeTest(ReadTest, unittest.TestCase): ] ) -class RawUnicodeEscapeTest(unittest.TestCase): +class RawUnicodeEscapeTest(ReadTest, unittest.TestCase): + encoding = "raw-unicode-escape" + + test_lone_surrogates = None + def test_empty(self): self.assertEqual(codecs.raw_unicode_escape_encode(""), (b"", 0)) self.assertEqual(codecs.raw_unicode_escape_decode(b""), ("", 0)) @@ -2520,6 +2524,35 @@ class RawUnicodeEscapeTest(unittest.TestCase): self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10)) self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10)) + def test_partial(self): + self.check_partial( + "\x00\t\n\r\\\xff\uffff\U00010000", + [ + '\x00', + '\x00\t', + '\x00\t\n', + '\x00\t\n\r', + '\x00\t\n\r', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff', + '\x00\t\n\r\\\xff\uffff\U00010000', + ] + ) + class EscapeEncodeTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2021-10-14-13-31-19.bpo-45467.Q7Ma6A.rst b/Misc/NEWS.d/next/Library/2021-10-14-13-31-19.bpo-45467.Q7Ma6A.rst new file mode 100644 index 0000000000..f2c0ae4ae5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-14-13-31-19.bpo-45467.Q7Ma6A.rst @@ -0,0 +1,2 @@ +Fix incremental decoder and stream reader in the "raw-unicode-escape" codec. +Previously they failed if the escape sequence was split. diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index fc74127ce5..50afc097b3 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -509,17 +509,20 @@ _codecs_unicode_escape_decode_impl(PyObject *module, Py_buffer *data, _codecs.raw_unicode_escape_decode data: Py_buffer(accept={str, buffer}) errors: str(accept={str, NoneType}) = None + final: bool(accept={int}) = True / [clinic start generated code]*/ static PyObject * _codecs_raw_unicode_escape_decode_impl(PyObject *module, Py_buffer *data, - const char *errors) -/*[clinic end generated code: output=c98eeb56028070a6 input=d2f5159ce3b3392f]*/ + const char *errors, int final) +/*[clinic end generated code: output=11dbd96301e2879e input=2d166191beb3235a]*/ { - PyObject *decoded = PyUnicode_DecodeRawUnicodeEscape(data->buf, data->len, - errors); - return codec_tuple(decoded, data->len); + Py_ssize_t consumed = data->len; + PyObject *decoded = _PyUnicode_DecodeRawUnicodeEscapeStateful(data->buf, data->len, + errors, + final ? NULL : &consumed); + return codec_tuple(decoded, consumed); } /*[clinic input] diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index a7086dd6e1..855ac77a7f 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -1143,7 +1143,7 @@ exit: } PyDoc_STRVAR(_codecs_raw_unicode_escape_decode__doc__, -"raw_unicode_escape_decode($module, data, errors=None, /)\n" +"raw_unicode_escape_decode($module, data, errors=None, final=True, /)\n" "--\n" "\n"); @@ -1152,7 +1152,7 @@ PyDoc_STRVAR(_codecs_raw_unicode_escape_decode__doc__, static PyObject * _codecs_raw_unicode_escape_decode_impl(PyObject *module, Py_buffer *data, - const char *errors); + const char *errors, int final); static PyObject * _codecs_raw_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) @@ -1160,8 +1160,9 @@ _codecs_raw_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ss PyObject *return_value = NULL; Py_buffer data = {NULL, NULL}; const char *errors = NULL; + int final = 1; - if (!_PyArg_CheckPositional("raw_unicode_escape_decode", nargs, 1, 2)) { + if (!_PyArg_CheckPositional("raw_unicode_escape_decode", nargs, 1, 3)) { goto exit; } if (PyUnicode_Check(args[0])) { @@ -1202,8 +1203,15 @@ _codecs_raw_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ss _PyArg_BadArgument("raw_unicode_escape_decode", "argument 2", "str or None", args[1]); goto exit; } + if (nargs < 3) { + goto skip_optional; + } + final = _PyLong_AsInt(args[2]); + if (final == -1 && PyErr_Occurred()) { + goto exit; + } skip_optional: - return_value = _codecs_raw_unicode_escape_decode_impl(module, &data, errors); + return_value = _codecs_raw_unicode_escape_decode_impl(module, &data, errors, final); exit: /* Cleanup for data */ @@ -2809,4 +2817,4 @@ exit: #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=9e9fb1d5d81577e0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=814dae36b6f885cb input=a9049054013a1b77]*/ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index e660834b47..c72871074b 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6445,8 +6445,6 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, unsigned char c = (unsigned char) *s++; Py_UCS4 ch; int count; - Py_ssize_t startinpos; - Py_ssize_t endinpos; const char *message; #define WRITE_ASCII_CHAR(ch) \ @@ -6473,7 +6471,7 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, continue; } - startinpos = s - starts - 1; + Py_ssize_t startinpos = s - starts - 1; /* \ - Escapes */ if (s >= end) { message = "\\ at end of string"; @@ -6620,8 +6618,8 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, *consumed = startinpos; break; } - error: - endinpos = s-starts; + error:; + Py_ssize_t endinpos = s-starts; writer.min_length = end - s + writer.pos; if (unicode_decode_call_errorhandler_writer( errors, &errorHandler, @@ -6816,9 +6814,10 @@ PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s, /* --- Raw Unicode Escape Codec ------------------------------------------- */ PyObject * -PyUnicode_DecodeRawUnicodeEscape(const char *s, - Py_ssize_t size, - const char *errors) +_PyUnicode_DecodeRawUnicodeEscapeStateful(const char *s, + Py_ssize_t size, + const char *errors, + Py_ssize_t *consumed) { const char *starts = s; _PyUnicodeWriter writer; @@ -6827,6 +6826,9 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s, PyObject *exc = NULL; if (size == 0) { + if (consumed) { + *consumed = 0; + } _Py_RETURN_UNICODE_EMPTY(); } @@ -6845,8 +6847,6 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s, unsigned char c = (unsigned char) *s++; Py_UCS4 ch; int count; - Py_ssize_t startinpos; - Py_ssize_t endinpos; const char *message; #define WRITE_CHAR(ch) \ @@ -6861,11 +6861,21 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s, } while(0) /* Non-escape characters are interpreted as Unicode ordinals */ - if (c != '\\' || s >= end) { + if (c != '\\' || (s >= end && !consumed)) { WRITE_CHAR(c); continue; } + Py_ssize_t startinpos = s - starts - 1; + /* \ - Escapes */ + if (s >= end) { + assert(consumed); + // Set message to silent compiler warning. + // Actually it is never used. + message = "\\ at end of string"; + goto incomplete; + } + c = (unsigned char) *s++; if (c == 'u') { count = 4; @@ -6881,10 +6891,12 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s, WRITE_CHAR(c); continue; } - startinpos = s - starts - 2; /* \uHHHH with 4 hex digits, \U00HHHHHH with 8 */ - for (ch = 0; count && s < end; ++s, --count) { + for (ch = 0; count; ++s, --count) { + if (s >= end) { + goto incomplete; + } c = (unsigned char)*s; ch <<= 4; if (c >= '0' && c <= '9') { @@ -6897,18 +6909,23 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s, ch += c - ('A' - 10); } else { - break; + goto error; } } - if (!count) { - if (ch <= MAX_UNICODE) { - WRITE_CHAR(ch); - continue; - } + if (ch > MAX_UNICODE) { message = "\\Uxxxxxxxx out of range"; + goto error; } + WRITE_CHAR(ch); + continue; - endinpos = s-starts; + incomplete: + if (consumed) { + *consumed = startinpos; + break; + } + error:; + Py_ssize_t endinpos = s-starts; writer.min_length = end - s + writer.pos; if (unicode_decode_call_errorhandler_writer( errors, &errorHandler, @@ -6930,7 +6947,14 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s, Py_XDECREF(errorHandler); Py_XDECREF(exc); return NULL; +} +PyObject * +PyUnicode_DecodeRawUnicodeEscape(const char *s, + Py_ssize_t size, + const char *errors) +{ + return _PyUnicode_DecodeRawUnicodeEscapeStateful(s, size, errors, NULL); } -- cgit v1.2.1 From 2b6eb8149656541044884e76212495175e061a0a Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 15 Oct 2021 05:14:35 -0700 Subject: bpo-45428: Fix reading filenames from stdin in py_compile (GH-28848) Strip trailing '\n'. (cherry picked from commit 59a633d3e2071d65aa6638da5cf767a5c1310271) Co-authored-by: Graham Inggs --- Lib/py_compile.py | 2 +- Misc/NEWS.d/next/Library/2021-10-14-18-04-17.bpo-45428.mM2War.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2021-10-14-18-04-17.bpo-45428.mM2War.rst diff --git a/Lib/py_compile.py b/Lib/py_compile.py index 0f9b59025c..388614e51b 100644 --- a/Lib/py_compile.py +++ b/Lib/py_compile.py @@ -190,7 +190,7 @@ def main(): ) args = parser.parse_args() if args.filenames == ['-']: - filenames = sys.stdin.readlines() + filenames = [filename.rstrip('\n') for filename in sys.stdin.readlines()] else: filenames = args.filenames for filename in filenames: diff --git a/Misc/NEWS.d/next/Library/2021-10-14-18-04-17.bpo-45428.mM2War.rst b/Misc/NEWS.d/next/Library/2021-10-14-18-04-17.bpo-45428.mM2War.rst new file mode 100644 index 0000000000..556eca43ed --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-14-18-04-17.bpo-45428.mM2War.rst @@ -0,0 +1 @@ +Fix a regression in py_compile when reading filenames from standard input. \ No newline at end of file -- cgit v1.2.1 From 855d6247adb39d4e38b698b89e519587318abd80 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 16 Oct 2021 07:01:54 -0700 Subject: [3.10]bpo-45463: Clarify that global statements allows multiple names (GH-28851) (GH-28989) The global statement allows specifying a list of identifiers (https://docs.python.org/3/reference/simple_stmts.htmlGH-the-global-statement). The "Execution model" chapter described the global statement as if it only allowed one single name. Pluralize "name" in the appropriate places. (cherry picked from commit 4ecd119b007cb766b8bede2dc78b70d29cd932dd) Co-authored-by: Luca Chiodini Co-authored-by: Luca Chiodini --- Doc/reference/executionmodel.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index 55ac01b6a8..5c85dd7052 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -119,14 +119,14 @@ is subtle. Python lacks declarations and allows name binding operations to occur anywhere within a code block. The local variables of a code block can be determined by scanning the entire text of the block for name binding operations. -If the :keyword:`global` statement occurs within a block, all uses of the name -specified in the statement refer to the binding of that name in the top-level +If the :keyword:`global` statement occurs within a block, all uses of the names +specified in the statement refer to the bindings of those names in the top-level namespace. Names are resolved in the top-level namespace by searching the global namespace, i.e. the namespace of the module containing the code block, and the builtins namespace, the namespace of the module :mod:`builtins`. The -global namespace is searched first. If the name is not found there, the +global namespace is searched first. If the names are not found there, the builtins namespace is searched. The :keyword:`!global` statement must precede -all uses of the name. +all uses of the listed names. The :keyword:`global` statement has the same scope as a name binding operation in the same block. If the nearest enclosing scope for a free variable contains -- cgit v1.2.1 From 5df35faf3611b459f7f32bfe9fd2ffa7fb2dc59e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 16 Oct 2021 10:51:05 -0700 Subject: bpo-45249: Fix caret location when end_offset is set to 0 (GH-28855) (cherry picked from commit fe0d9e22a52a10c4cbe52254b51f2d4e74d83568) Co-authored-by: Pablo Galindo Salgado --- Lib/test/test_traceback.py | 10 ++++++++++ Lib/traceback.py | 2 +- .../next/Library/2021-10-10-16-14-33.bpo-45249.xqLliz.rst | 2 ++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2021-10-10-16-14-33.bpo-45249.xqLliz.rst diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index fabe5d01a9..ba03ce4629 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -51,6 +51,9 @@ class TracebackCases(unittest.TestCase): def syntax_error_bad_indentation2(self): compile(" print(2)", "?", "exec") + def tokenizer_error_with_caret_range(self): + compile("blech ( ", "?", "exec") + def test_caret(self): err = self.get_exception_format(self.syntax_error_with_caret, SyntaxError) @@ -81,6 +84,13 @@ class TracebackCases(unittest.TestCase): self.assertEqual(err[1].find("y"), err[2].find("^")) # in the right place self.assertEqual(err[2].count("^"), len("y for y in range(30)")) + err = self.get_exception_format(self.tokenizer_error_with_caret_range, + SyntaxError) + self.assertIn("^", err[2]) # third line has caret + self.assertEqual(err[2].count('\n'), 1) # and no additional newline + self.assertEqual(err[1].find("("), err[2].find("^")) # in the right place + self.assertEqual(err[2].count("^"), 1) + def test_nocaret(self): exc = SyntaxError("error", ("x.py", 23, None, "bad syntax")) err = traceback.format_exception_only(SyntaxError, exc) diff --git a/Lib/traceback.py b/Lib/traceback.py index c7947f118b..901b99476a 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -633,7 +633,7 @@ class TracebackException: if self.offset is not None: offset = self.offset - end_offset = self.end_offset if self.end_offset is not None else offset + end_offset = self.end_offset if self.end_offset not in {None, 0} else offset if offset == end_offset or end_offset == -1: end_offset = offset + 1 diff --git a/Misc/NEWS.d/next/Library/2021-10-10-16-14-33.bpo-45249.xqLliz.rst b/Misc/NEWS.d/next/Library/2021-10-10-16-14-33.bpo-45249.xqLliz.rst new file mode 100644 index 0000000000..1d5a857e25 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-10-16-14-33.bpo-45249.xqLliz.rst @@ -0,0 +1,2 @@ +Fix the behaviour of :func:`traceback.print_exc` when displaying the caret +when the ``end_offset`` in the exception is set to 0. Patch by Pablo Galindo -- cgit v1.2.1 From a29470307308f64af9c55263cdd6e1984ba89925 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 16 Oct 2021 16:14:11 -0700 Subject: [3.10] bpo-45495: Add 'case' and 'match' to IDLE completions list. (GH-29000) (GH-29001) Since the keyword list is frozen, only compute it once per session. The colorizer already handles context keywords. (cherry picked from commit 42ac06dcd234bdda989dcfe854ac5173337024c9) Co-authored-by: Terry Jan Reedy Automerge-Triggered-By: GH:terryjreedy --- Lib/idlelib/autocomplete.py | 10 +++++++--- Lib/idlelib/idle_test/test_autocomplete.py | 5 +++++ Misc/NEWS.d/next/IDLE/2021-10-16-17-20-32.bpo-45495.ST8RFt.rst | 1 + 3 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2021-10-16-17-20-32.bpo-45495.ST8RFt.rst diff --git a/Lib/idlelib/autocomplete.py b/Lib/idlelib/autocomplete.py index bb7ee035c4..032d312253 100644 --- a/Lib/idlelib/autocomplete.py +++ b/Lib/idlelib/autocomplete.py @@ -9,6 +9,12 @@ import os import string import sys +# Modified keyword list is used in fetch_completions. +completion_kwds = [s for s in keyword.kwlist + if s not in {'True', 'False', 'None'}] # In builtins. +completion_kwds.extend(('match', 'case')) # Context keywords. +completion_kwds.sort() + # Two types of completions; defined here for autocomplete_w import below. ATTRS, FILES = 0, 1 from idlelib import autocomplete_w @@ -177,9 +183,7 @@ class AutoComplete: namespace = {**__main__.__builtins__.__dict__, **__main__.__dict__} bigl = eval("dir()", namespace) - kwds = (s for s in keyword.kwlist - if s not in {'True', 'False', 'None'}) - bigl.extend(kwds) + bigl.extend(completion_kwds) bigl.sort() if "__all__" in bigl: smalll = sorted(eval("__all__", namespace)) diff --git a/Lib/idlelib/idle_test/test_autocomplete.py b/Lib/idlelib/idle_test/test_autocomplete.py index 642bb5db64..a811363c18 100644 --- a/Lib/idlelib/idle_test/test_autocomplete.py +++ b/Lib/idlelib/idle_test/test_autocomplete.py @@ -218,6 +218,11 @@ class AutoCompleteTest(unittest.TestCase): self.assertTrue(acp.open_completions(ac.TAB)) self.text.delete('1.0', 'end') + def test_completion_kwds(self): + self.assertIn('and', ac.completion_kwds) + self.assertIn('case', ac.completion_kwds) + self.assertNotIn('None', ac.completion_kwds) + def test_fetch_completions(self): # Test that fetch_completions returns 2 lists: # For attribute completion, a large list containing all variables, and diff --git a/Misc/NEWS.d/next/IDLE/2021-10-16-17-20-32.bpo-45495.ST8RFt.rst b/Misc/NEWS.d/next/IDLE/2021-10-16-17-20-32.bpo-45495.ST8RFt.rst new file mode 100644 index 0000000000..3868f8d136 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2021-10-16-17-20-32.bpo-45495.ST8RFt.rst @@ -0,0 +1 @@ +Add context keywords 'case' and 'match' to completions list. -- cgit v1.2.1 From 65c1db794e0484e18142658c691141387c1dc2e4 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 17 Oct 2021 09:47:03 -0700 Subject: bpo-45229: Make test_http_cookiejar discoverable (GH-29004) (cherry picked from commit b3f0ceae919c1627094ff628c87184684a5cedd6) Co-authored-by: Serhiy Storchaka --- Lib/test/test_http_cookiejar.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py index fdf15efde1..9450104d0b 100644 --- a/Lib/test/test_http_cookiejar.py +++ b/Lib/test/test_http_cookiejar.py @@ -1920,14 +1920,5 @@ class LWPCookieTests(unittest.TestCase): self.assertNotEqual(counter["session_before"], 0) -def test_main(verbose=None): - test.support.run_unittest( - DateTimeTests, - HeaderTests, - CookieTests, - FileCookieJarTests, - LWPCookieTests, - ) - if __name__ == "__main__": - test_main(verbose=True) + unittest.main() -- cgit v1.2.1 From 7082abf53db0ce599eb7347bd7efab0a647efb18 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 17 Oct 2021 16:43:10 -0700 Subject: Improve multiserver queue recipe (GH-29012) (GH-29013) --- Doc/library/random.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Doc/library/random.rst b/Doc/library/random.rst index e924127d8b..b9c33af59c 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -512,7 +512,7 @@ between the effects of a drug versus a placebo:: Simulation of arrival times and service deliveries for a multiserver queue:: - from heapq import heappush, heappop + from heapq import heapify, heapreplace from random import expovariate, gauss from statistics import mean, quantiles @@ -524,14 +524,15 @@ Simulation of arrival times and service deliveries for a multiserver queue:: waits = [] arrival_time = 0.0 servers = [0.0] * num_servers # time when each server becomes available - for i in range(100_000): + heapify(servers) + for i in range(1_000_000): arrival_time += expovariate(1.0 / average_arrival_interval) - next_server_available = heappop(servers) + next_server_available = servers[0] wait = max(0.0, next_server_available - arrival_time) waits.append(wait) - service_duration = gauss(average_service_time, stdev_service_time) + service_duration = max(0.0, gauss(average_service_time, stdev_service_time)) service_completed = arrival_time + wait + service_duration - heappush(servers, service_completed) + heapreplace(servers, service_completed) print(f'Mean wait: {mean(waits):.1f} Max wait: {max(waits):.1f}') print('Quartiles:', [round(q, 1) for q in quantiles(waits)]) -- cgit v1.2.1 From b1949e0b58714724a3105cad3ad1b61384688da7 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 18 Oct 2021 11:49:28 -0700 Subject: [3.10] bpo-45221: Fix handling of LDFLAGS and CPPFLAGS options in setup.py (GH-29031) (GH-29037) (cherry picked from commit 6a533a423869e28d9086cf4d79029f59e9eec916) Co-authored-by: andrei kulakov Automerge-Triggered-By: GH:ned-deily --- .../next/Build/2021-10-18-10-25-56.bpo-45221.rnulhf.rst | 3 +++ setup.py | 12 ++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2021-10-18-10-25-56.bpo-45221.rnulhf.rst diff --git a/Misc/NEWS.d/next/Build/2021-10-18-10-25-56.bpo-45221.rnulhf.rst b/Misc/NEWS.d/next/Build/2021-10-18-10-25-56.bpo-45221.rnulhf.rst new file mode 100644 index 0000000000..cb981d96f3 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-10-18-10-25-56.bpo-45221.rnulhf.rst @@ -0,0 +1,3 @@ +Fixed regression in handling of ``LDFLAGS`` and ``CPPFLAGS`` options +where :meth:`argparse.parse_known_args` could interpret an option as +one of the built-in command line argument, for example ``-h`` for help. diff --git a/setup.py b/setup.py index a6fcc12b84..74d5cefc60 100644 --- a/setup.py +++ b/setup.py @@ -801,6 +801,18 @@ class PyBuildExt(build_ext): if env_val: parser = argparse.ArgumentParser() parser.add_argument(arg_name, dest="dirs", action="append") + + # To prevent argparse from raising an exception about any + # options in env_val that it mistakes for known option, we + # strip out all double dashes and any dashes followed by a + # character that is not for the option we are dealing with. + # + # Please note that order of the regex is important! We must + # strip out double-dashes first so that we don't end up with + # substituting "--Long" to "-Long" and thus lead to "ong" being + # used for a library directory. + env_val = re.sub(r'(^|\s+)-(-|(?!%s))' % arg_name[1], + ' ', env_val) options, _ = parser.parse_known_args(env_val.split()) if options.dirs: for directory in reversed(options.dirs): -- cgit v1.2.1 From 97ce855ca8ce437070424b43f5b41158685ac140 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 18 Oct 2021 20:15:48 -0700 Subject: bpo-45475: Revert `__iter__` optimization for GzipFile, BZ2File, and LZMAFile. (GH-29016) This reverts commit d2a8e69c2c605fbaa3656a5f99aa8d295f74c80e. (cherry picked from commit 0a4c82ddd34a3578684b45b76f49cd289a08740b) Co-authored-by: Inada Naoki --- Lib/bz2.py | 4 ---- Lib/gzip.py | 4 ---- Lib/lzma.py | 4 ---- Misc/NEWS.d/next/Library/2021-10-18-10-46-47.bpo-45475.sb9KDF.rst | 4 ++++ 4 files changed, 4 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2021-10-18-10-46-47.bpo-45475.sb9KDF.rst diff --git a/Lib/bz2.py b/Lib/bz2.py index 7f1d20632e..fabe4f73c8 100644 --- a/Lib/bz2.py +++ b/Lib/bz2.py @@ -197,10 +197,6 @@ class BZ2File(_compression.BaseStream): self._check_can_read() return self._buffer.readline(size) - def __iter__(self): - self._check_can_read() - return self._buffer.__iter__() - def readlines(self, size=-1): """Read a list of lines of uncompressed bytes from the file. diff --git a/Lib/gzip.py b/Lib/gzip.py index 3d837b7448..475ec326c0 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -398,10 +398,6 @@ class GzipFile(_compression.BaseStream): self._check_not_closed() return self._buffer.readline(size) - def __iter__(self): - self._check_not_closed() - return self._buffer.__iter__() - class _GzipReader(_compression.DecompressReader): def __init__(self, fp): diff --git a/Lib/lzma.py b/Lib/lzma.py index 9abf06d91d..800f52198f 100644 --- a/Lib/lzma.py +++ b/Lib/lzma.py @@ -221,10 +221,6 @@ class LZMAFile(_compression.BaseStream): self._check_can_read() return self._buffer.readline(size) - def __iter__(self): - self._check_can_read() - return self._buffer.__iter__() - def write(self, data): """Write a bytes object to the file. diff --git a/Misc/NEWS.d/next/Library/2021-10-18-10-46-47.bpo-45475.sb9KDF.rst b/Misc/NEWS.d/next/Library/2021-10-18-10-46-47.bpo-45475.sb9KDF.rst new file mode 100644 index 0000000000..6fce894e6e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-18-10-46-47.bpo-45475.sb9KDF.rst @@ -0,0 +1,4 @@ +Reverted optimization of iterating :class:`gzip.GzipFile`, +:class:`bz2.BZ2File`, and :class:`lzma.LZMAFile` (see bpo-43787) because it +caused regression when user iterate them without having reference of them. +Patch by Inada Naoki. -- cgit v1.2.1 From 8cef526a871695f8df4fac167139bbf3292396db Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 19 Oct 2021 11:35:34 -0700 Subject: bpo-45515: Add zoneinfo to the datetime documentation (GH-29038) (GH-29065) We should have done this way back when 3.9 was released, but it fell off the radar. Co-authored-by: Paul Ganssle (cherry picked from commit 8e40ca127fa92d6113617c80710e0a077977a84d) --- Doc/library/datetime.rst | 10 ++++++---- .../next/Library/2021-10-18-14-52-48.bpo-45515.aXdvm_.rst | 3 +++ 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2021-10-18-14-52-48.bpo-45515.aXdvm_.rst diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 196aa84473..0f8b70cded 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -27,6 +27,9 @@ on efficient attribute extraction for output formatting and manipulation. Module :mod:`time` Time access and conversions. + Module :mod:`zoneinfo` + Concrete time zones representing the IANA time zone database. + Package `dateutil `_ Third-party library with expanded time zone and parsing support. @@ -2174,14 +2177,13 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. seealso:: - `dateutil.tz `_ + :mod:`zoneinfo` The :mod:`datetime` module has a basic :class:`timezone` class (for handling arbitrary fixed offsets from UTC) and its :attr:`timezone.utc` attribute (a UTC timezone instance). - *dateutil.tz* library brings the *IANA timezone database* - (also known as the Olson database) to Python, and its usage is - recommended. + ``zoneinfo`` brings the *IANA timezone database* (also known as the Olson + database) to Python, and its usage is recommended. `IANA timezone database `_ The Time Zone Database (often called tz, tzdata or zoneinfo) contains code diff --git a/Misc/NEWS.d/next/Library/2021-10-18-14-52-48.bpo-45515.aXdvm_.rst b/Misc/NEWS.d/next/Library/2021-10-18-14-52-48.bpo-45515.aXdvm_.rst new file mode 100644 index 0000000000..382733ff91 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-18-14-52-48.bpo-45515.aXdvm_.rst @@ -0,0 +1,3 @@ +Add references to :mod:`zoneinfo` in the :mod:`datetime` documentation, +mostly replacing outdated references to ``dateutil.tz``. Change by Paul +Ganssle. -- cgit v1.2.1 From 092ec4b9d144c493cb514df0978638fdd2d9622a Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 19 Oct 2021 11:41:33 -0700 Subject: bpo-45449: add note about PEP 585 in collections.abc's documentation (GH-29047) (GH-29067) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns Co-authored-by: Łukasz Langa (cherry picked from commit 7bafa0cf586227987d3d662264d491e3780024b7) --- Doc/library/collections.abc.rst | 3 +++ .../NEWS.d/next/Documentation/2021-10-19-01-41-40.bpo-45449.fjHZJc.rst | 1 + 2 files changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Documentation/2021-10-19-01-41-40.bpo-45449.fjHZJc.rst diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 0abc87f919..2c941b4774 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -104,6 +104,9 @@ example, knowing that a class supplies ``__getitem__``, ``__len__``, and ``__iter__`` is insufficient for distinguishing a :class:`Sequence` from a :class:`Mapping`. +.. versionadded:: 3.9 + These abstract classes now support ``[]``. See :ref:`types-genericalias` + and :pep:`585`. .. _collections-abstract-base-classes: diff --git a/Misc/NEWS.d/next/Documentation/2021-10-19-01-41-40.bpo-45449.fjHZJc.rst b/Misc/NEWS.d/next/Documentation/2021-10-19-01-41-40.bpo-45449.fjHZJc.rst new file mode 100644 index 0000000000..fb817757a1 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2021-10-19-01-41-40.bpo-45449.fjHZJc.rst @@ -0,0 +1 @@ +Add note about :pep:`585` in :mod:`collections.abc`. -- cgit v1.2.1 From 325b2c223453203b2fa9ce7b9bcebdbef03adf70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 19 Oct 2021 22:21:03 +0200 Subject: [3.10] bpo-45500: Rewrite test_dbm (GH-29002) (GH-29069) * Generate test classes at import time. It allows to filter them when run with unittest. E.g: "./python -m unittest test.test_dbm.TestCase_gnu -v". * Create a database class in a new directory which will be removed after test. It guarantees that all created files and directories be removed and will not conflict with other dbm tests. * Restore dbm._defaultmod after tests. Previously it was set to the last dbm module (dbm.dumb) which affected other tests. * Enable the whichdb test for dbm.dumb. * Move test_keys to the correct test class. It does not test whichdb(). * Remove some outdated code and comments.. (cherry picked from commit 975b94b9de969777218e96a9950c1dab2dab65a0) Co-authored-by: Serhiy Storchaka --- Lib/test/test_dbm.py | 114 ++++++++++++++++++++++----------------------------- 1 file changed, 50 insertions(+), 64 deletions(-) diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index e02d1e16ae..16a023084e 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -1,23 +1,21 @@ """Test script for the dbm.open function based on testdumbdbm.py""" import unittest -import glob +import dbm +import os from test.support import import_helper from test.support import os_helper -# Skip tests if dbm module doesn't exist. -dbm = import_helper.import_module('dbm') - try: from dbm import ndbm except ImportError: ndbm = None -_fname = os_helper.TESTFN +dirname = os_helper.TESTFN +_fname = os.path.join(dirname, os_helper.TESTFN) # -# Iterates over every database module supported by dbm currently available, -# setting dbm to use each in turn, and yielding that module +# Iterates over every database module supported by dbm currently available. # def dbm_iterator(): for name in dbm._names: @@ -31,11 +29,12 @@ def dbm_iterator(): # # Clean up all scratch databases we might have created during testing # -def delete_files(): - # we don't know the precise name the underlying database uses - # so we use glob to locate all names - for f in glob.glob(glob.escape(_fname) + "*"): - os_helper.unlink(f) +def cleaunup_test_dir(): + os_helper.rmtree(dirname) + +def setup_test_dir(): + cleaunup_test_dir() + os.mkdir(dirname) class AnyDBMTestCase: @@ -134,80 +133,67 @@ class AnyDBMTestCase: for key in self._dict: self.assertEqual(self._dict[key], f[key.encode("ascii")]) - def tearDown(self): - delete_files() + def test_keys(self): + with dbm.open(_fname, 'c') as d: + self.assertEqual(d.keys(), []) + a = [(b'a', b'b'), (b'12345678910', b'019237410982340912840198242')] + for k, v in a: + d[k] = v + self.assertEqual(sorted(d.keys()), sorted(k for (k, v) in a)) + for k, v in a: + self.assertIn(k, d) + self.assertEqual(d[k], v) + self.assertNotIn(b'xxx', d) + self.assertRaises(KeyError, lambda: d[b'xxx']) def setUp(self): + self.addCleanup(setattr, dbm, '_defaultmod', dbm._defaultmod) dbm._defaultmod = self.module - delete_files() + self.addCleanup(cleaunup_test_dir) + setup_test_dir() class WhichDBTestCase(unittest.TestCase): def test_whichdb(self): + self.addCleanup(setattr, dbm, '_defaultmod', dbm._defaultmod) for module in dbm_iterator(): # Check whether whichdb correctly guesses module name # for databases opened with "module" module. - # Try with empty files first name = module.__name__ - if name == 'dbm.dumb': - continue # whichdb can't support dbm.dumb - delete_files() - f = module.open(_fname, 'c') - f.close() + setup_test_dir() + dbm._defaultmod = module + # Try with empty files first + with module.open(_fname, 'c'): pass self.assertEqual(name, self.dbm.whichdb(_fname)) # Now add a key - f = module.open(_fname, 'w') - f[b"1"] = b"1" - # and test that we can find it - self.assertIn(b"1", f) - # and read it - self.assertEqual(f[b"1"], b"1") - f.close() + with module.open(_fname, 'w') as f: + f[b"1"] = b"1" + # and test that we can find it + self.assertIn(b"1", f) + # and read it + self.assertEqual(f[b"1"], b"1") self.assertEqual(name, self.dbm.whichdb(_fname)) @unittest.skipUnless(ndbm, reason='Test requires ndbm') def test_whichdb_ndbm(self): # Issue 17198: check that ndbm which is referenced in whichdb is defined - db_file = '{}_ndbm.db'.format(_fname) - with open(db_file, 'w'): - self.addCleanup(os_helper.unlink, db_file) - self.assertIsNone(self.dbm.whichdb(db_file[:-3])) - - def tearDown(self): - delete_files() + with open(_fname + '.db', 'wb'): pass + self.assertIsNone(self.dbm.whichdb(_fname)) def setUp(self): - delete_files() - self.filename = os_helper.TESTFN - self.d = dbm.open(self.filename, 'c') - self.d.close() + self.addCleanup(cleaunup_test_dir) + setup_test_dir() self.dbm = import_helper.import_fresh_module('dbm') - def test_keys(self): - self.d = dbm.open(self.filename, 'c') - self.assertEqual(self.d.keys(), []) - a = [(b'a', b'b'), (b'12345678910', b'019237410982340912840198242')] - for k, v in a: - self.d[k] = v - self.assertEqual(sorted(self.d.keys()), sorted(k for (k, v) in a)) - for k, v in a: - self.assertIn(k, self.d) - self.assertEqual(self.d[k], v) - self.assertNotIn(b'xxx', self.d) - self.assertRaises(KeyError, lambda: self.d[b'xxx']) - self.d.close() - - -def load_tests(loader, tests, pattern): - classes = [] - for mod in dbm_iterator(): - classes.append(type("TestCase-" + mod.__name__, - (AnyDBMTestCase, unittest.TestCase), - {'module': mod})) - suites = [unittest.makeSuite(c) for c in classes] - - tests.addTests(suites) - return tests + +for mod in dbm_iterator(): + assert mod.__name__.startswith('dbm.') + suffix = mod.__name__[4:] + testname = f'TestCase_{suffix}' + globals()[testname] = type(testname, + (AnyDBMTestCase, unittest.TestCase), + {'module': mod}) + if __name__ == "__main__": unittest.main() -- cgit v1.2.1 From 5c9cab595e56aeb118bff77ece784dbac30b4338 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 19 Oct 2021 22:31:18 +0200 Subject: [3.10] bpo-45494: Fix parser crash when reporting errors involving invalid continuation characters (GH-28993) (GH-29070) There are two errors that this commit fixes: * The parser was not correctly computing the offset and the string source for E_LINECONT errors due to the incorrect usage of strtok(). * The parser was not correctly unwinding the call stack when a tokenizer exception happened in rules involving optionals ('?', [...]) as we always make them return valid results by using the comma operator. We need to check first if we don't have an error before continuing.. (cherry picked from commit a106343f632a99c8ebb0136fa140cf189b4a6a57) Co-authored-by: Pablo Galindo Salgado --- Lib/test/test_exceptions.py | 4 + .../2021-10-16-17-27-48.bpo-45494.vMt1g4.rst | 2 + Parser/parser.c | 240 ++++++++++----------- Parser/pegen.c | 12 +- Tools/peg_generator/pegen/c_generator.py | 2 +- 5 files changed, 137 insertions(+), 123 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-10-16-17-27-48.bpo-45494.vMt1g4.rst diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 4213dabfd8..4930c57d3f 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -209,6 +209,10 @@ class ExceptionTests(unittest.TestCase): src = src.decode(encoding, 'replace') line = src.split('\n')[lineno-1] self.assertIn(line, cm.exception.text) + + def test_error_offset_continuation_characters(self): + check = self.check + check('"\\\n"(1 for c in I,\\\n\\', 2, 2) def testSyntaxErrorOffset(self): check = self.check diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-16-17-27-48.bpo-45494.vMt1g4.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-16-17-27-48.bpo-45494.vMt1g4.rst new file mode 100644 index 0000000000..97e29813ab --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-16-17-27-48.bpo-45494.vMt1g4.rst @@ -0,0 +1,2 @@ +Fix parser crash when reporting errors involving invalid continuation +characters. Patch by Pablo Galindo. diff --git a/Parser/parser.c b/Parser/parser.c index 2a437d5281..eccf104d67 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -948,7 +948,7 @@ file_rule(Parser *p) void *a; Token * endmarker_var; if ( - (a = statements_rule(p), 1) // statements? + (a = statements_rule(p), !p->error_indicator) // statements? && (endmarker_var = _PyPegen_expect_token(p, ENDMARKER)) // token='ENDMARKER' ) @@ -1087,7 +1087,7 @@ func_type_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = type_expressions_rule(p), 1) // type_expressions? + (a = type_expressions_rule(p), !p->error_indicator) // type_expressions? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && @@ -1689,7 +1689,7 @@ simple_stmts_rule(Parser *p) if ( (a = (asdl_stmt_seq*)_gather_12_rule(p)) // ';'.simple_stmt+ && - (_opt_var = _PyPegen_expect_token(p, 13), 1) // ';'? + (_opt_var = _PyPegen_expect_token(p, 13), !p->error_indicator) // ';'? && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -2309,7 +2309,7 @@ assignment_rule(Parser *p) && (b = expression_rule(p)) // expression && - (c = _tmp_19_rule(p), 1) // ['=' annotated_rhs] + (c = _tmp_19_rule(p), !p->error_indicator) // ['=' annotated_rhs] ) { D(fprintf(stderr, "%*c+ assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME ':' expression ['=' annotated_rhs]")); @@ -2351,7 +2351,7 @@ assignment_rule(Parser *p) && (b = expression_rule(p)) // expression && - (c = _tmp_21_rule(p), 1) // ['=' annotated_rhs] + (c = _tmp_21_rule(p), !p->error_indicator) // ['=' annotated_rhs] ) { D(fprintf(stderr, "%*c+ assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "('(' single_target ')' | single_subscript_attribute_target) ':' expression ['=' annotated_rhs]")); @@ -2392,7 +2392,7 @@ assignment_rule(Parser *p) && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? ) { D(fprintf(stderr, "%*c+ assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((star_targets '='))+ (yield_expr | star_expressions) !'=' TYPE_COMMENT?")); @@ -3047,7 +3047,7 @@ assert_stmt_rule(Parser *p) && (a = expression_rule(p)) // expression && - (b = _tmp_29_rule(p), 1) // [',' expression] + (b = _tmp_29_rule(p), !p->error_indicator) // [',' expression] ) { D(fprintf(stderr, "%*c+ assert_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'assert' expression [',' expression]")); @@ -3433,7 +3433,7 @@ import_from_targets_rule(Parser *p) && (a = import_from_as_names_rule(p)) // import_from_as_names && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -3602,7 +3602,7 @@ import_from_as_name_rule(Parser *p) if ( (a = _PyPegen_name_token(p)) // NAME && - (b = _tmp_35_rule(p), 1) // ['as' NAME] + (b = _tmp_35_rule(p), !p->error_indicator) // ['as' NAME] ) { D(fprintf(stderr, "%*c+ import_from_as_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME ['as' NAME]")); @@ -3705,7 +3705,7 @@ dotted_as_name_rule(Parser *p) if ( (a = dotted_name_rule(p)) // dotted_name && - (b = _tmp_38_rule(p), 1) // ['as' NAME] + (b = _tmp_38_rule(p), !p->error_indicator) // ['as' NAME] ) { D(fprintf(stderr, "%*c+ dotted_as_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name ['as' NAME]")); @@ -3943,7 +3943,7 @@ if_stmt_rule(Parser *p) && (b = block_rule(p)) // block && - (c = else_block_rule(p), 1) // else_block? + (c = else_block_rule(p), !p->error_indicator) // else_block? ) { D(fprintf(stderr, "%*c+ if_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' named_expression ':' block else_block?")); @@ -4081,7 +4081,7 @@ elif_stmt_rule(Parser *p) && (b = block_rule(p)) // block && - (c = else_block_rule(p), 1) // else_block? + (c = else_block_rule(p), !p->error_indicator) // else_block? ) { D(fprintf(stderr, "%*c+ elif_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'elif' named_expression ':' block else_block?")); @@ -4237,7 +4237,7 @@ while_stmt_rule(Parser *p) && (b = block_rule(p)) // block && - (c = else_block_rule(p), 1) // else_block? + (c = else_block_rule(p), !p->error_indicator) // else_block? ) { D(fprintf(stderr, "%*c+ while_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'while' named_expression ':' block else_block?")); @@ -4339,11 +4339,11 @@ for_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? && (b = block_rule(p)) // block && - (el = else_block_rule(p), 1) // else_block? + (el = else_block_rule(p), !p->error_indicator) // else_block? ) { D(fprintf(stderr, "%*c+ for_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'for' star_targets 'in' ~ star_expressions &&':' TYPE_COMMENT? block else_block?")); @@ -4403,11 +4403,11 @@ for_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? && (b = block_rule(p)) // block && - (el = else_block_rule(p), 1) // else_block? + (el = else_block_rule(p), !p->error_indicator) // else_block? ) { D(fprintf(stderr, "%*c+ for_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "ASYNC 'for' star_targets 'in' ~ star_expressions &&':' TYPE_COMMENT? block else_block?")); @@ -4527,7 +4527,7 @@ with_stmt_rule(Parser *p) && (a = (asdl_withitem_seq*)_gather_39_rule(p)) // ','.with_item+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && @@ -4576,7 +4576,7 @@ with_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? && (b = block_rule(p)) // block ) @@ -4627,7 +4627,7 @@ with_stmt_rule(Parser *p) && (a = (asdl_withitem_seq*)_gather_43_rule(p)) // ','.with_item+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && @@ -4679,7 +4679,7 @@ with_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? && (b = block_rule(p)) // block ) @@ -4931,9 +4931,9 @@ try_stmt_rule(Parser *p) && (ex = (asdl_excepthandler_seq*)_loop1_48_rule(p)) // except_block+ && - (el = else_block_rule(p), 1) // else_block? + (el = else_block_rule(p), !p->error_indicator) // else_block? && - (f = finally_block_rule(p), 1) // finally_block? + (f = finally_block_rule(p), !p->error_indicator) // finally_block? ) { D(fprintf(stderr, "%*c+ try_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'try' &&':' block except_block+ else_block? finally_block?")); @@ -5023,7 +5023,7 @@ except_block_rule(Parser *p) && (e = expression_rule(p)) // expression && - (t = _tmp_49_rule(p), 1) // ['as' NAME] + (t = _tmp_49_rule(p), !p->error_indicator) // ['as' NAME] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -5314,7 +5314,7 @@ subject_expr_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (values = star_named_expressions_rule(p), 1) // star_named_expressions? + (values = star_named_expressions_rule(p), !p->error_indicator) // star_named_expressions? ) { D(fprintf(stderr, "%*c+ subject_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions?")); @@ -5410,7 +5410,7 @@ case_block_rule(Parser *p) && (pattern = patterns_rule(p)) // patterns && - (guard = guard_rule(p), 1) // guard? + (guard = guard_rule(p), !p->error_indicator) // guard? && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -7163,7 +7163,7 @@ sequence_pattern_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' && - (patterns = maybe_sequence_pattern_rule(p), 1) // maybe_sequence_pattern? + (patterns = maybe_sequence_pattern_rule(p), !p->error_indicator) // maybe_sequence_pattern? && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' ) @@ -7202,7 +7202,7 @@ sequence_pattern_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (patterns = open_sequence_pattern_rule(p), 1) // open_sequence_pattern? + (patterns = open_sequence_pattern_rule(p), !p->error_indicator) // open_sequence_pattern? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -7260,7 +7260,7 @@ open_sequence_pattern_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (patterns = maybe_sequence_pattern_rule(p), 1) // maybe_sequence_pattern? + (patterns = maybe_sequence_pattern_rule(p), !p->error_indicator) // maybe_sequence_pattern? ) { D(fprintf(stderr, "%*c+ open_sequence_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern ',' maybe_sequence_pattern?")); @@ -7305,7 +7305,7 @@ maybe_sequence_pattern_rule(Parser *p) if ( (patterns = _gather_57_rule(p)) // ','.maybe_star_pattern+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ maybe_sequence_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.maybe_star_pattern+ ','?")); @@ -7556,7 +7556,7 @@ mapping_pattern_rule(Parser *p) && (rest = double_star_pattern_rule(p)) // double_star_pattern && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_1 = _PyPegen_expect_token(p, 26)) // token='}' ) @@ -7605,7 +7605,7 @@ mapping_pattern_rule(Parser *p) && (rest = double_star_pattern_rule(p)) // double_star_pattern && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_2 = _PyPegen_expect_token(p, 26)) // token='}' ) @@ -7648,7 +7648,7 @@ mapping_pattern_rule(Parser *p) && (items = items_pattern_rule(p)) // items_pattern && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_1 = _PyPegen_expect_token(p, 26)) // token='}' ) @@ -7891,7 +7891,7 @@ class_pattern_rule(Parser *p) && (patterns = positional_patterns_rule(p)) // positional_patterns && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -7937,7 +7937,7 @@ class_pattern_rule(Parser *p) && (keywords = keyword_patterns_rule(p)) // keyword_patterns && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -7989,7 +7989,7 @@ class_pattern_rule(Parser *p) && (keywords = keyword_patterns_rule(p)) // keyword_patterns && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_2 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -8196,7 +8196,7 @@ return_stmt_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 500)) // token='return' && - (a = star_expressions_rule(p), 1) // star_expressions? + (a = star_expressions_rule(p), !p->error_indicator) // star_expressions? ) { D(fprintf(stderr, "%*c+ return_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'return' star_expressions?")); @@ -8261,7 +8261,7 @@ raise_stmt_rule(Parser *p) && (a = expression_rule(p)) // expression && - (b = _tmp_66_rule(p), 1) // ['from' expression] + (b = _tmp_66_rule(p), !p->error_indicator) // ['from' expression] ) { D(fprintf(stderr, "%*c+ raise_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'raise' expression ['from' expression]")); @@ -8452,15 +8452,15 @@ function_def_raw_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (params = params_rule(p), 1) // params? + (params = params_rule(p), !p->error_indicator) // params? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (a = _tmp_67_rule(p), 1) // ['->' expression] + (a = _tmp_67_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && - (tc = func_type_comment_rule(p), 1) // func_type_comment? + (tc = func_type_comment_rule(p), !p->error_indicator) // func_type_comment? && (b = block_rule(p)) // block ) @@ -8512,15 +8512,15 @@ function_def_raw_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (params = params_rule(p), 1) // params? + (params = params_rule(p), !p->error_indicator) // params? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (a = _tmp_68_rule(p), 1) // ['->' expression] + (a = _tmp_68_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && - (tc = func_type_comment_rule(p), 1) // func_type_comment? + (tc = func_type_comment_rule(p), !p->error_indicator) // func_type_comment? && (b = block_rule(p)) // block ) @@ -8728,7 +8728,7 @@ parameters_rule(Parser *p) && (c = _loop0_71_rule(p)) // param_with_default* && - (d = star_etc_rule(p), 1) // star_etc? + (d = star_etc_rule(p), !p->error_indicator) // star_etc? ) { D(fprintf(stderr, "%*c+ parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default param_no_default* param_with_default* star_etc?")); @@ -8758,7 +8758,7 @@ parameters_rule(Parser *p) && (b = _loop0_72_rule(p)) // param_with_default* && - (c = star_etc_rule(p), 1) // star_etc? + (c = star_etc_rule(p), !p->error_indicator) // star_etc? ) { D(fprintf(stderr, "%*c+ parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default param_with_default* star_etc?")); @@ -8788,7 +8788,7 @@ parameters_rule(Parser *p) && (b = _loop0_74_rule(p)) // param_with_default* && - (c = star_etc_rule(p), 1) // star_etc? + (c = star_etc_rule(p), !p->error_indicator) // star_etc? ) { D(fprintf(stderr, "%*c+ parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default+ param_with_default* star_etc?")); @@ -8815,7 +8815,7 @@ parameters_rule(Parser *p) if ( (a = _loop1_75_rule(p)) // param_with_default+ && - (b = star_etc_rule(p), 1) // star_etc? + (b = star_etc_rule(p), !p->error_indicator) // star_etc? ) { D(fprintf(stderr, "%*c+ parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_with_default+ star_etc?")); @@ -9053,7 +9053,7 @@ star_etc_rule(Parser *p) && (b = _loop0_82_rule(p)) // param_maybe_default* && - (c = kwds_rule(p), 1) // kwds? + (c = kwds_rule(p), !p->error_indicator) // kwds? ) { D(fprintf(stderr, "%*c+ star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' param_no_default param_maybe_default* kwds?")); @@ -9086,7 +9086,7 @@ star_etc_rule(Parser *p) && (b = _loop1_83_rule(p)) // param_maybe_default+ && - (c = kwds_rule(p), 1) // kwds? + (c = kwds_rule(p), !p->error_indicator) // kwds? ) { D(fprintf(stderr, "%*c+ star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' ',' param_maybe_default+ kwds?")); @@ -9220,7 +9220,7 @@ param_no_default_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? ) { D(fprintf(stderr, "%*c+ param_no_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param ',' TYPE_COMMENT?")); @@ -9247,7 +9247,7 @@ param_no_default_rule(Parser *p) if ( (a = param_rule(p)) // param && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? && _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 8) // token=')' ) @@ -9299,7 +9299,7 @@ param_with_default_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? ) { D(fprintf(stderr, "%*c+ param_with_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param default ',' TYPE_COMMENT?")); @@ -9329,7 +9329,7 @@ param_with_default_rule(Parser *p) && (c = default_rule(p)) // default && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? && _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 8) // token=')' ) @@ -9379,11 +9379,11 @@ param_maybe_default_rule(Parser *p) if ( (a = param_rule(p)) // param && - (c = default_rule(p), 1) // default? + (c = default_rule(p), !p->error_indicator) // default? && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? ) { D(fprintf(stderr, "%*c+ param_maybe_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param default? ',' TYPE_COMMENT?")); @@ -9411,9 +9411,9 @@ param_maybe_default_rule(Parser *p) if ( (a = param_rule(p)) // param && - (c = default_rule(p), 1) // default? + (c = default_rule(p), !p->error_indicator) // default? && - (tc = _PyPegen_expect_token(p, TYPE_COMMENT), 1) // TYPE_COMMENT? + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? && _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 8) // token=')' ) @@ -9468,7 +9468,7 @@ param_rule(Parser *p) if ( (a = _PyPegen_name_token(p)) // NAME && - (b = annotation_rule(p), 1) // annotation? + (b = annotation_rule(p), !p->error_indicator) // annotation? ) { D(fprintf(stderr, "%*c+ param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME annotation?")); @@ -9746,7 +9746,7 @@ class_def_raw_rule(Parser *p) && (a = _PyPegen_name_token(p)) // NAME && - (b = _tmp_85_rule(p), 1) // ['(' arguments? ')'] + (b = _tmp_85_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -9912,7 +9912,7 @@ star_expressions_rule(Parser *p) && (b = _loop1_86_rule(p)) // ((',' star_expression))+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ star_expressions[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expression ((',' star_expression))+ ','?")); @@ -10107,7 +10107,7 @@ star_named_expressions_rule(Parser *p) if ( (a = (asdl_expr_seq*)_gather_87_rule(p)) // ','.star_named_expression+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ star_named_expressions[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.star_named_expression+ ','?")); @@ -10448,7 +10448,7 @@ expressions_rule(Parser *p) && (b = _loop1_89_rule(p)) // ((',' expression))+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ expressions[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ((',' expression))+ ','?")); @@ -10704,7 +10704,7 @@ lambdef_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 528)) // token='lambda' && - (a = lambda_params_rule(p), 1) // lambda_params? + (a = lambda_params_rule(p), !p->error_indicator) // lambda_params? && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -10827,7 +10827,7 @@ lambda_parameters_rule(Parser *p) && (c = _loop0_91_rule(p)) // lambda_param_with_default* && - (d = lambda_star_etc_rule(p), 1) // lambda_star_etc? + (d = lambda_star_etc_rule(p), !p->error_indicator) // lambda_star_etc? ) { D(fprintf(stderr, "%*c+ lambda_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default lambda_param_no_default* lambda_param_with_default* lambda_star_etc?")); @@ -10857,7 +10857,7 @@ lambda_parameters_rule(Parser *p) && (b = _loop0_92_rule(p)) // lambda_param_with_default* && - (c = lambda_star_etc_rule(p), 1) // lambda_star_etc? + (c = lambda_star_etc_rule(p), !p->error_indicator) // lambda_star_etc? ) { D(fprintf(stderr, "%*c+ lambda_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default lambda_param_with_default* lambda_star_etc?")); @@ -10887,7 +10887,7 @@ lambda_parameters_rule(Parser *p) && (b = _loop0_94_rule(p)) // lambda_param_with_default* && - (c = lambda_star_etc_rule(p), 1) // lambda_star_etc? + (c = lambda_star_etc_rule(p), !p->error_indicator) // lambda_star_etc? ) { D(fprintf(stderr, "%*c+ lambda_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default+ lambda_param_with_default* lambda_star_etc?")); @@ -10914,7 +10914,7 @@ lambda_parameters_rule(Parser *p) if ( (a = _loop1_95_rule(p)) // lambda_param_with_default+ && - (b = lambda_star_etc_rule(p), 1) // lambda_star_etc? + (b = lambda_star_etc_rule(p), !p->error_indicator) // lambda_star_etc? ) { D(fprintf(stderr, "%*c+ lambda_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default+ lambda_star_etc?")); @@ -11154,7 +11154,7 @@ lambda_star_etc_rule(Parser *p) && (b = _loop0_102_rule(p)) // lambda_param_maybe_default* && - (c = lambda_kwds_rule(p), 1) // lambda_kwds? + (c = lambda_kwds_rule(p), !p->error_indicator) // lambda_kwds? ) { D(fprintf(stderr, "%*c+ lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' lambda_param_no_default lambda_param_maybe_default* lambda_kwds?")); @@ -11187,7 +11187,7 @@ lambda_star_etc_rule(Parser *p) && (b = _loop1_103_rule(p)) // lambda_param_maybe_default+ && - (c = lambda_kwds_rule(p), 1) // lambda_kwds? + (c = lambda_kwds_rule(p), !p->error_indicator) // lambda_kwds? ) { D(fprintf(stderr, "%*c+ lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' ',' lambda_param_maybe_default+ lambda_kwds?")); @@ -11465,7 +11465,7 @@ lambda_param_maybe_default_rule(Parser *p) if ( (a = lambda_param_rule(p)) // lambda_param && - (c = default_rule(p), 1) // default? + (c = default_rule(p), !p->error_indicator) // default? && (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) @@ -11494,7 +11494,7 @@ lambda_param_maybe_default_rule(Parser *p) if ( (a = lambda_param_rule(p)) // lambda_param && - (c = default_rule(p), 1) // default? + (c = default_rule(p), !p->error_indicator) // default? && _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 11) // token=':' ) @@ -14024,7 +14024,7 @@ primary_raw(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (b = arguments_rule(p), 1) // arguments? + (b = arguments_rule(p), !p->error_indicator) // arguments? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -14176,7 +14176,7 @@ slices_rule(Parser *p) if ( (a = (asdl_expr_seq*)_gather_108_rule(p)) // ','.slice+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ slices[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.slice+ ','?")); @@ -14238,13 +14238,13 @@ slice_rule(Parser *p) void *b; void *c; if ( - (a = expression_rule(p), 1) // expression? + (a = expression_rule(p), !p->error_indicator) // expression? && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (b = expression_rule(p), 1) // expression? + (b = expression_rule(p), !p->error_indicator) // expression? && - (c = _tmp_110_rule(p), 1) // [':' expression?] + (c = _tmp_110_rule(p), !p->error_indicator) // [':' expression?] ) { D(fprintf(stderr, "%*c+ slice[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression? ':' expression? [':' expression?]")); @@ -14667,7 +14667,7 @@ list_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' && - (a = star_named_expressions_rule(p), 1) // star_named_expressions? + (a = star_named_expressions_rule(p), !p->error_indicator) // star_named_expressions? && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' ) @@ -14819,7 +14819,7 @@ tuple_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = _tmp_115_rule(p), 1) // [star_named_expression ',' star_named_expressions?] + (a = _tmp_115_rule(p), !p->error_indicator) // [star_named_expression ',' star_named_expressions?] && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -15191,7 +15191,7 @@ dict_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (a = double_starred_kvpairs_rule(p), 1) // double_starred_kvpairs? + (a = double_starred_kvpairs_rule(p), !p->error_indicator) // double_starred_kvpairs? && (_literal_1 = _PyPegen_expect_token(p, 26)) // token='}' ) @@ -15359,7 +15359,7 @@ double_starred_kvpairs_rule(Parser *p) if ( (a = _gather_118_rule(p)) // ','.double_starred_kvpair+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ','?")); @@ -15730,7 +15730,7 @@ yield_expr_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 504)) // token='yield' && - (a = star_expressions_rule(p), 1) // star_expressions? + (a = star_expressions_rule(p), !p->error_indicator) // star_expressions? ) { D(fprintf(stderr, "%*c+ yield_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'yield' star_expressions?")); @@ -15788,7 +15788,7 @@ arguments_rule(Parser *p) if ( (a = args_rule(p)) // args && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 8) // token=')' ) @@ -15865,7 +15865,7 @@ args_rule(Parser *p) if ( (a = (asdl_expr_seq*)_gather_123_rule(p)) // ','.(starred_expression | (assigment_expression | expression !':=') !'=')+ && - (b = _tmp_125_rule(p), 1) // [',' kwargs] + (b = _tmp_125_rule(p), !p->error_indicator) // [',' kwargs] ) { D(fprintf(stderr, "%*c+ args[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assigment_expression | expression !':=') !'=')+ [',' kwargs]")); @@ -16368,7 +16368,7 @@ star_targets_rule(Parser *p) && (b = _loop0_134_rule(p)) // ((',' star_target))* && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ star_targets[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_target ((',' star_target))* ','?")); @@ -16422,7 +16422,7 @@ star_targets_list_seq_rule(Parser *p) if ( (a = (asdl_expr_seq*)_gather_135_rule(p)) // ','.star_target+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ star_targets_list_seq[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.star_target+ ','?")); @@ -16470,7 +16470,7 @@ star_targets_tuple_seq_rule(Parser *p) && (b = _loop1_137_rule(p)) // ((',' star_target))+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ star_targets_tuple_seq[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_target ((',' star_target))+ ','?")); @@ -16833,7 +16833,7 @@ star_atom_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = star_targets_tuple_seq_rule(p), 1) // star_targets_tuple_seq? + (a = star_targets_tuple_seq_rule(p), !p->error_indicator) // star_targets_tuple_seq? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -16872,7 +16872,7 @@ star_atom_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' && - (a = star_targets_list_seq_rule(p), 1) // star_targets_list_seq? + (a = star_targets_list_seq_rule(p), !p->error_indicator) // star_targets_list_seq? && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' ) @@ -17131,7 +17131,7 @@ del_targets_rule(Parser *p) if ( (a = (asdl_expr_seq*)_gather_139_rule(p)) // ','.del_target+ && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ del_targets[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.del_target+ ','?")); @@ -17377,7 +17377,7 @@ del_t_atom_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = del_targets_rule(p), 1) // del_targets? + (a = del_targets_rule(p), !p->error_indicator) // del_targets? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -17416,7 +17416,7 @@ del_t_atom_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' && - (a = del_targets_rule(p), 1) // del_targets? + (a = del_targets_rule(p), !p->error_indicator) // del_targets? && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' ) @@ -17646,7 +17646,7 @@ t_primary_raw(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (b = arguments_rule(p), 1) // arguments? + (b = arguments_rule(p), !p->error_indicator) // arguments? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && @@ -17846,7 +17846,7 @@ invalid_arguments_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_opt_var = _tmp_141_rule(p), 1) // [args | expression for_if_clauses] + (_opt_var = _tmp_141_rule(p), !p->error_indicator) // [args | expression for_if_clauses] ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses ',' [args | expression for_if_clauses]")); @@ -19501,7 +19501,7 @@ invalid_for_target_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings expr_ty a; if ( - (_opt_var = _PyPegen_expect_token(p, ASYNC), 1) // ASYNC? + (_opt_var = _PyPegen_expect_token(p, ASYNC), !p->error_indicator) // ASYNC? && (_keyword = _PyPegen_expect_token(p, 517)) // token='for' && @@ -19679,7 +19679,7 @@ invalid_with_stmt_rule(Parser *p) void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings if ( - (_opt_var = _PyPegen_expect_token(p, ASYNC), 1) // ASYNC? + (_opt_var = _PyPegen_expect_token(p, ASYNC), !p->error_indicator) // ASYNC? && (_keyword = _PyPegen_expect_token(p, 519)) // token='with' && @@ -19712,7 +19712,7 @@ invalid_with_stmt_rule(Parser *p) void *_opt_var_1; UNUSED(_opt_var_1); // Silence compiler warnings if ( - (_opt_var = _PyPegen_expect_token(p, ASYNC), 1) // ASYNC? + (_opt_var = _PyPegen_expect_token(p, ASYNC), !p->error_indicator) // ASYNC? && (_keyword = _PyPegen_expect_token(p, 519)) // token='with' && @@ -19720,7 +19720,7 @@ invalid_with_stmt_rule(Parser *p) && (_gather_164_var = _gather_164_rule(p)) // ','.(expressions ['as' star_target])+ && - (_opt_var_1 = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && @@ -19767,7 +19767,7 @@ invalid_with_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, ASYNC), 1) // ASYNC? + (_opt_var = _PyPegen_expect_token(p, ASYNC), !p->error_indicator) // ASYNC? && (a = _PyPegen_expect_token(p, 519)) // token='with' && @@ -19810,7 +19810,7 @@ invalid_with_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, ASYNC), 1) // ASYNC? + (_opt_var = _PyPegen_expect_token(p, ASYNC), !p->error_indicator) // ASYNC? && (a = _PyPegen_expect_token(p, 519)) // token='with' && @@ -19818,7 +19818,7 @@ invalid_with_stmt_indent_rule(Parser *p) && (_gather_168_var = _gather_168_rule(p)) // ','.(expressions ['as' star_target])+ && - (_opt_var_1 = _PyPegen_expect_token(p, 12), 1) // ','? + (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && @@ -19965,7 +19965,7 @@ invalid_except_stmt_rule(Parser *p) && (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_171_rule(p), 1) // ['as' NAME] + (_opt_var = _tmp_171_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -19999,7 +19999,7 @@ invalid_except_stmt_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_172_rule(p), 1) // ['as' NAME] + (_opt_var = _tmp_172_rule(p), !p->error_indicator) // ['as' NAME] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -20129,7 +20129,7 @@ invalid_except_stmt_indent_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_173_rule(p), 1) // ['as' NAME] + (_opt_var = _tmp_173_rule(p), !p->error_indicator) // ['as' NAME] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -20300,7 +20300,7 @@ invalid_case_block_rule(Parser *p) && (patterns_var = patterns_rule(p)) // patterns && - (_opt_var = guard_rule(p), 1) // guard? + (_opt_var = guard_rule(p), !p->error_indicator) // guard? && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 11) // token=':' ) @@ -20335,7 +20335,7 @@ invalid_case_block_rule(Parser *p) && (patterns_var = patterns_rule(p)) // patterns && - (_opt_var = guard_rule(p), 1) // guard? + (_opt_var = guard_rule(p), !p->error_indicator) // guard? && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -20513,7 +20513,7 @@ invalid_class_argument_pattern_rule(Parser *p) asdl_pattern_seq* a; asdl_seq* keyword_patterns_var; if ( - (_opt_var = _tmp_174_rule(p), 1) // [positional_patterns ','] + (_opt_var = _tmp_174_rule(p), !p->error_indicator) // [positional_patterns ','] && (keyword_patterns_var = keyword_patterns_rule(p)) // keyword_patterns && @@ -20868,7 +20868,7 @@ invalid_for_stmt_rule(Parser *p) expr_ty star_expressions_var; expr_ty star_targets_var; if ( - (_opt_var = _PyPegen_expect_token(p, ASYNC), 1) // ASYNC? + (_opt_var = _PyPegen_expect_token(p, ASYNC), !p->error_indicator) // ASYNC? && (a = _PyPegen_expect_token(p, 517)) // token='for' && @@ -20935,7 +20935,7 @@ invalid_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, ASYNC), 1) // ASYNC? + (_opt_var = _PyPegen_expect_token(p, ASYNC), !p->error_indicator) // ASYNC? && (a = _PyPegen_expect_token(p, 526)) // token='def' && @@ -20943,11 +20943,11 @@ invalid_def_raw_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_opt_var_1 = params_rule(p), 1) // params? + (_opt_var_1 = params_rule(p), !p->error_indicator) // params? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (_opt_var_2 = _tmp_175_rule(p), 1) // ['->' expression] + (_opt_var_2 = _tmp_175_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -21003,7 +21003,7 @@ invalid_class_def_raw_rule(Parser *p) && (name_var = _PyPegen_name_token(p)) // NAME && - (_opt_var = _tmp_176_rule(p), 1) // ['(' arguments? ')'] + (_opt_var = _tmp_176_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -26246,7 +26246,7 @@ _tmp_85_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (z = arguments_rule(p), 1) // arguments? + (z = arguments_rule(p), !p->error_indicator) // arguments? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -27875,7 +27875,7 @@ _tmp_110_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (d = expression_rule(p), 1) // expression? + (d = expression_rule(p), !p->error_indicator) // expression? ) { D(fprintf(stderr, "%*c+ _tmp_110[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':' expression?")); @@ -28215,7 +28215,7 @@ _tmp_115_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (z = star_named_expressions_rule(p), 1) // star_named_expressions? + (z = star_named_expressions_rule(p), !p->error_indicator) // star_named_expressions? ) { D(fprintf(stderr, "%*c+ _tmp_115[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions?")); @@ -31766,7 +31766,7 @@ _tmp_176_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_opt_var = arguments_rule(p), 1) // arguments? + (_opt_var = arguments_rule(p), !p->error_indicator) // arguments? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -32775,7 +32775,7 @@ _tmp_197_rule(Parser *p) if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_202_rule(p), 1) // ['as' star_target] + (_opt_var = _tmp_202_rule(p), !p->error_indicator) // ['as' star_target] ) { D(fprintf(stderr, "%*c+ _tmp_197[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); @@ -32815,7 +32815,7 @@ _tmp_198_rule(Parser *p) if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_203_rule(p), 1) // ['as' star_target] + (_opt_var = _tmp_203_rule(p), !p->error_indicator) // ['as' star_target] ) { D(fprintf(stderr, "%*c+ _tmp_198[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); @@ -32855,7 +32855,7 @@ _tmp_199_rule(Parser *p) if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_204_rule(p), 1) // ['as' star_target] + (_opt_var = _tmp_204_rule(p), !p->error_indicator) // ['as' star_target] ) { D(fprintf(stderr, "%*c+ _tmp_199[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); @@ -32895,7 +32895,7 @@ _tmp_200_rule(Parser *p) if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_205_rule(p), 1) // ['as' star_target] + (_opt_var = _tmp_205_rule(p), !p->error_indicator) // ['as' star_target] ) { D(fprintf(stderr, "%*c+ _tmp_200[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); diff --git a/Parser/pegen.c b/Parser/pegen.c index 1bb975d684..66e4b19297 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -371,10 +371,18 @@ tokenizer_error(Parser *p) errtype = PyExc_IndentationError; msg = "too many levels of indentation"; break; - case E_LINECONT: - col_offset = strlen(strtok(p->tok->buf, "\n")) - 1; + case E_LINECONT: { + char* loc = strrchr(p->tok->buf, '\n'); + const char* last_char = p->tok->cur - 1; + if (loc != NULL && loc != last_char) { + col_offset = p->tok->cur - loc - 1; + p->tok->buf = loc; + } else { + col_offset = last_char - p->tok->buf - 1; + } msg = "unexpected character after line continuation character"; break; + } default: msg = "unknown parsing error"; } diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index 7941978252..3079edf54b 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -88,7 +88,7 @@ class FunctionCall: if self.arguments: parts.append(f"({', '.join(map(str, self.arguments))})") if self.force_true: - parts.append(", 1") + parts.append(", !p->error_indicator") if self.assigned_variable: if self.assigned_variable_type: parts = ["(", self.assigned_variable, " = ", '(', self.assigned_variable_type, ')', *parts, ")"] -- cgit v1.2.1 From c15ba304f35362470e29ea5626fed28366bc9571 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 19 Oct 2021 15:07:13 -0700 Subject: bpo-39679: Add tests for classmethod/staticmethod singledispatchmethods (GH-29034) (GH-29072) In Python 3.8 and 3.9, stacking `@functools.singledispatchmethod` on top of `@classmethod` or `@staticmethod` caused an exception to be raised if the method was registered using type-annotations rather than `@method.register(int)`. This was not caught by unit tests, however, as the tests only tested the `@method.register(int)` way of registering additional implementations. The bug is no longer present in Python 3.10+, but `test_functools.py` is still lacking regression tests for these cases. This commit adds these test cases. (cherry picked from commit ad6d162e518963711d24c80f1b7d6079bd437584) Co-authored-by: Alex Waygood --- Lib/test/test_functools.py | 42 ++++++++++++++++++++++ .../Tests/2021-10-18-16-18-41.bpo-39679.F18qcE.rst | 2 ++ 2 files changed, 44 insertions(+) create mode 100644 Misc/NEWS.d/next/Tests/2021-10-18-16-18-41.bpo-39679.F18qcE.rst diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index fece8256a3..bdb4ddcc60 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -2437,6 +2437,48 @@ class TestSingleDispatch(unittest.TestCase): self.assertEqual(a.t(''), "str") self.assertEqual(a.t(0.0), "base") + def test_staticmethod_type_ann_register(self): + class A: + @functools.singledispatchmethod + @staticmethod + def t(arg): + return arg + @t.register + @staticmethod + def _(arg: int): + return isinstance(arg, int) + @t.register + @staticmethod + def _(arg: str): + return isinstance(arg, str) + a = A() + + self.assertTrue(A.t(0)) + self.assertTrue(A.t('')) + self.assertEqual(A.t(0.0), 0.0) + + def test_classmethod_type_ann_register(self): + class A: + def __init__(self, arg): + self.arg = arg + + @functools.singledispatchmethod + @classmethod + def t(cls, arg): + return cls("base") + @t.register + @classmethod + def _(cls, arg: int): + return cls("int") + @t.register + @classmethod + def _(cls, arg: str): + return cls("str") + + self.assertEqual(A.t(0).arg, "int") + self.assertEqual(A.t('').arg, "str") + self.assertEqual(A.t(0.0).arg, "base") + def test_invalid_registrations(self): msg_prefix = "Invalid first argument to `register()`: " msg_suffix = ( diff --git a/Misc/NEWS.d/next/Tests/2021-10-18-16-18-41.bpo-39679.F18qcE.rst b/Misc/NEWS.d/next/Tests/2021-10-18-16-18-41.bpo-39679.F18qcE.rst new file mode 100644 index 0000000000..b0d1b68639 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2021-10-18-16-18-41.bpo-39679.F18qcE.rst @@ -0,0 +1,2 @@ +Add more test cases for `@functools.singledispatchmethod` when combined with +`@classmethod` or `@staticmethod`. -- cgit v1.2.1 From b8dbb3a7f96718dddb8bed31130f421316213dc5 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 Oct 2021 07:30:57 -0700 Subject: bpo-45436: Fix tkinter tests with Tcl/Tk 8.6.11+ (GH-29077) (GH-29080) Since v8.6.11, a few configuration options seem to accept an empty value where they did not previously; particularly the `type` of a `Menu` widget, and the `compound` of any ttk widget with a label. Providing an explicit expected error message to `checkEnumParam` bypasses the check of an empty value, which no longer raises `TclError`. (cherry picked from commit 4fe454c6f54b0948af67b53af6c2f35af6377e69) Co-authored-by: Zachary Ware --- Lib/tkinter/test/test_tkinter/test_widgets.py | 7 +++++-- Lib/tkinter/test/test_ttk/test_widgets.py | 9 ++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Lib/tkinter/test/test_tkinter/test_widgets.py b/Lib/tkinter/test/test_tkinter/test_widgets.py index 39334de8cf..cc227e5796 100644 --- a/Lib/tkinter/test/test_tkinter/test_widgets.py +++ b/Lib/tkinter/test/test_tkinter/test_widgets.py @@ -1241,8 +1241,11 @@ class MenuTest(AbstractWidgetTest, unittest.TestCase): def test_configure_type(self): widget = self.create() - self.checkEnumParam(widget, 'type', - 'normal', 'tearoff', 'menubar') + self.checkEnumParam( + widget, 'type', + 'normal', 'tearoff', 'menubar', + errmsg='bad type "{}": must be normal, tearoff, or menubar', + ) def test_entryconfigure(self): m1 = self.create() diff --git a/Lib/tkinter/test/test_ttk/test_widgets.py b/Lib/tkinter/test/test_ttk/test_widgets.py index a2a4de2e5c..904aed0ac2 100644 --- a/Lib/tkinter/test/test_ttk/test_widgets.py +++ b/Lib/tkinter/test/test_ttk/test_widgets.py @@ -169,10 +169,13 @@ class AbstractLabelTest(AbstractWidgetTest): errmsg='image "spam" doesn\'t exist') def test_configure_compound(self): + options = 'none text image center top bottom left right'.split() + errmsg = ( + 'bad compound "{}": must be' + f' {", ".join(options[:-1])}, or {options[-1]}' + ) widget = self.create() - self.checkEnumParam(widget, 'compound', - 'none', 'text', 'image', 'center', - 'top', 'bottom', 'left', 'right') + self.checkEnumParam(widget, 'compound', *options, errmsg=errmsg) def test_configure_state(self): widget = self.create() -- cgit v1.2.1 From 0d0312e1ac4a92d8cfb71c4c66c66eaf7270262f Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 Oct 2021 08:36:34 -0700 Subject: Add PEPs 593 & 647 to list of PEPs at top of typing docs (GH-29097) (cherry picked from commit d9e1dae35ac20acfeb2509b0dea4c3943693e79d) Co-authored-by: Alex Waygood --- Doc/library/typing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 13760c1921..c402c73b48 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -19,7 +19,7 @@ This module provides runtime support for type hints as specified by :pep:`484`, :pep:`526`, :pep:`544`, :pep:`586`, :pep:`589`, :pep:`591`, -:pep:`612` and :pep:`613`. +:pep:`593`, :pep:`612`, :pep:`613` and :pep:`647`. The most fundamental support consists of the types :data:`Any`, :data:`Union`, :data:`Tuple`, :data:`Callable`, :class:`TypeVar`, and :class:`Generic`. For full specification please see :pep:`484`. For -- cgit v1.2.1 From 5537b9f10510735447bea81079ac586f46decf20 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 Oct 2021 08:46:32 -0700 Subject: bpo-45536: Check OpenSSL APIs in configure (GH-29088) (cherry picked from commit 81520fe677d15cc7f9af5140bc5f9eca8409ad90) Co-authored-by: Christian Heimes --- .../Build/2021-10-20-12-42-39.bpo-45536.oQNYHB.rst | 3 ++ configure | 60 ++++++++++++++++++++++ configure.ac | 42 +++++++++++++++ 3 files changed, 105 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2021-10-20-12-42-39.bpo-45536.oQNYHB.rst diff --git a/Misc/NEWS.d/next/Build/2021-10-20-12-42-39.bpo-45536.oQNYHB.rst b/Misc/NEWS.d/next/Build/2021-10-20-12-42-39.bpo-45536.oQNYHB.rst new file mode 100644 index 0000000000..e560b71ede --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-10-20-12-42-39.bpo-45536.oQNYHB.rst @@ -0,0 +1,3 @@ +The ``configure`` script now checks whether OpenSSL headers and libraries +provide required APIs. Most common APIs are verified. The check detects +outdated or missing OpenSSL. Failures do not stop configure. diff --git a/configure b/configure index a6e0f823b4..e79425d5b9 100755 --- a/configure +++ b/configure @@ -17778,6 +17778,66 @@ esac $as_echo "$OPENSSL_RPATH" >&6; } +# check if OpenSSL libraries work as expected +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether OpenSSL provides required APIs" >&5 +$as_echo_n "checking whether OpenSSL provides required APIs... " >&6; } +save_LIBS="$LIBS" +save_CFLAGS="$CFLAGS" +save_LDFLAGS="$LDFLAGS" +LIBS="$LIBS $OPENSSL_LIBS" +CFLAGS="$CFLAGS_NODIST $OPENSSL_INCLUDES" +LDFLAGS="$LDFLAGS $OPENSSL_LDFLAGS" + +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#include +#include +#include + +#if OPENSSL_VERSION_NUMBER < 0x10101000L +#error "OpenSSL >= 1.1.1 is required" +#endif + +static void keylog_cb(const SSL *ssl, const char *line) {} + +int +main () +{ + +/* SSL APIs */ +SSL_CTX *ctx = SSL_CTX_new(TLS_client_method()); +SSL_CTX_set_keylog_callback(ctx, keylog_cb); +SSL *ssl = SSL_new(ctx); +X509_VERIFY_PARAM *param = SSL_get0_param(ssl); +X509_VERIFY_PARAM_set1_host(param, "python.org", 0); +SSL_free(ssl); +SSL_CTX_free(ctx); + +/* hashlib APIs */ +OBJ_nid2sn(NID_md5); +OBJ_nid2sn(NID_sha1); +OBJ_nid2sn(NID_sha3_512); +OBJ_nid2sn(NID_blake2b512); +EVP_PBE_scrypt(NULL, 0, NULL, 0, 2, 8, 1, 0, NULL, 0); + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS="$save_LIBS" +CFLAGS="$save_CFLAGS" +LDFLAGS="$save_LDFLAGS" + # ssl module default cipher suite string diff --git a/configure.ac b/configure.ac index c4abac6701..4c2ac5915a 100644 --- a/configure.ac +++ b/configure.ac @@ -5831,6 +5831,48 @@ AS_CASE($with_openssl_rpath, AC_MSG_RESULT($OPENSSL_RPATH) AC_SUBST([OPENSSL_RPATH]) +# check if OpenSSL libraries work as expected +AC_MSG_CHECKING(whether OpenSSL provides required APIs) +save_LIBS="$LIBS" +save_CFLAGS="$CFLAGS" +save_LDFLAGS="$LDFLAGS" +LIBS="$LIBS $OPENSSL_LIBS" +CFLAGS="$CFLAGS_NODIST $OPENSSL_INCLUDES" +LDFLAGS="$LDFLAGS $OPENSSL_LDFLAGS" + +AC_LINK_IFELSE([AC_LANG_PROGRAM([[ +#include +#include +#include + +#if OPENSSL_VERSION_NUMBER < 0x10101000L +#error "OpenSSL >= 1.1.1 is required" +#endif + +static void keylog_cb(const SSL *ssl, const char *line) {} +]], [[ +/* SSL APIs */ +SSL_CTX *ctx = SSL_CTX_new(TLS_client_method()); +SSL_CTX_set_keylog_callback(ctx, keylog_cb); +SSL *ssl = SSL_new(ctx); +X509_VERIFY_PARAM *param = SSL_get0_param(ssl); +X509_VERIFY_PARAM_set1_host(param, "python.org", 0); +SSL_free(ssl); +SSL_CTX_free(ctx); + +/* hashlib APIs */ +OBJ_nid2sn(NID_md5); +OBJ_nid2sn(NID_sha1); +OBJ_nid2sn(NID_sha3_512); +OBJ_nid2sn(NID_blake2b512); +EVP_PBE_scrypt(NULL, 0, NULL, 0, 2, 8, 1, 0, NULL, 0); +]])], + [AC_MSG_RESULT(yes)], + [AC_MSG_RESULT(no)]) +LIBS="$save_LIBS" +CFLAGS="$save_CFLAGS" +LDFLAGS="$save_LDFLAGS" + # ssl module default cipher suite string AH_TEMPLATE(PY_SSL_DEFAULT_CIPHERS, [Default cipher suites list for ssl module. -- cgit v1.2.1 From d6afe3be0106818454e1aac07bebe6d5e75bb38d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 Oct 2021 09:42:39 -0700 Subject: Cleanup a couple of comments left on PR 28775 post-merge. (GH-29079) (cherry picked from commit 1dfac27dffbe771f9d88bd1726f7362ce0341437) Co-authored-by: Gregory P. Smith --- Lib/asyncio/unix_events.py | 2 +- Lib/tkinter/test/test_ttk/test_widgets.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 4cef914b9f..c88b818de6 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -1379,7 +1379,7 @@ class ThreadedChildWatcher(AbstractChildWatcher): def remove_child_handler(self, pid): # asyncio never calls remove_child_handler() !!! # The method is no-op but is implemented because - # abstract base classes requires it + # abstract base classes require it. return True def attach_loop(self, loop): diff --git a/Lib/tkinter/test/test_ttk/test_widgets.py b/Lib/tkinter/test/test_ttk/test_widgets.py index 904aed0ac2..935be3d7f1 100644 --- a/Lib/tkinter/test/test_ttk/test_widgets.py +++ b/Lib/tkinter/test/test_ttk/test_widgets.py @@ -971,7 +971,7 @@ class NotebookTest(AbstractWidgetTest, unittest.TestCase): tabs = self.nb.tabs() curr = self.nb.index('current') - # verify that the tab gets read at its previous position + # verify that the tab gets re-added at its previous position child2_index = self.nb.index(self.child2) self.nb.hide(self.child2) self.nb.add(self.child2) -- cgit v1.2.1 From 65de808811f93793599209f74bb1bab3ad399b17 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 Oct 2021 09:52:22 -0700 Subject: bpo-45229: Make doctest tests discoverable (GH-28986) (GH-29095) (cherry picked from commit 8d6740f489fca67a44de165d29d9e0ad86285779) Co-authored-by: Serhiy Storchaka --- Lib/test/test_doctest.py | 20 ++++++-------------- Lib/test/test_doctest2.py | 21 ++++++++++++--------- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 828a0ff567..7f8ccd3896 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -3110,20 +3110,11 @@ def test_no_trailing_whitespace_stripping(): patches that contain trailing whitespace. More info on Issue 24746. """ -###################################################################### -## Main -###################################################################### - -def test_main(): - # Check the doctest cases in doctest itself: - ret = support.run_doctest(doctest, verbosity=True) - # Check the doctest cases defined here: - from test import test_doctest - support.run_doctest(test_doctest, verbosity=True) - - # Run unittests - support.run_unittest(__name__) +def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite(doctest)) + tests.addTest(doctest.DocTestSuite()) + return tests def test_coverage(coverdir): @@ -3136,8 +3127,9 @@ def test_coverage(coverdir): r.write_results(show_missing=True, summary=True, coverdir=coverdir) + if __name__ == '__main__': if '-c' in sys.argv: test_coverage('/tmp/doctest.cover') else: - test_main() + unittest.main() diff --git a/Lib/test/test_doctest2.py b/Lib/test/test_doctest2.py index 347a143641..ab8a069673 100644 --- a/Lib/test/test_doctest2.py +++ b/Lib/test/test_doctest2.py @@ -13,7 +13,6 @@ the example. It should be ignored: import sys import unittest -from test import support if sys.flags.optimize >= 2: raise unittest.SkipTest("Cannot test docstrings with -O2") @@ -107,17 +106,21 @@ class C(object): """ return val -def test_main(): - from test import test_doctest2 - EXPECTED = 19 - f, t = support.run_doctest(test_doctest2) - if t != EXPECTED: - raise support.TestFailed("expected %d tests to run, not %d" % - (EXPECTED, t)) + +class Test(unittest.TestCase): + def test_testmod(self): + import doctest, sys + EXPECTED = 19 + f, t = doctest.testmod(sys.modules[__name__]) + if f: + self.fail("%d of %d doctests failed" % (f, t)) + if t != EXPECTED: + self.fail("expected %d tests to run, not %d" % (EXPECTED, t)) + # Pollute the namespace with a bunch of imported functions and classes, # to make sure they don't get tested. from doctest import * if __name__ == '__main__': - test_main() + unittest.main() -- cgit v1.2.1 From 696a89fef81f8229ebff9c32dfd36921c04f0890 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 Oct 2021 10:41:35 -0700 Subject: bpo-45532: Replace 'default' with 'main' as default in sys.version (GH-29100) (cherry picked from commit d2cd5eef0c3fc0431bfe3fc24b4c020ebfcf8aad) Co-authored-by: Jeong YunWon <69878+youknowone@users.noreply.github.com> --- Misc/NEWS.d/next/Build/2021-10-20-16-07-39.bpo-45532.kyhvis.rst | 2 ++ Modules/getbuildinfo.c | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2021-10-20-16-07-39.bpo-45532.kyhvis.rst diff --git a/Misc/NEWS.d/next/Build/2021-10-20-16-07-39.bpo-45532.kyhvis.rst b/Misc/NEWS.d/next/Build/2021-10-20-16-07-39.bpo-45532.kyhvis.rst new file mode 100644 index 0000000000..575e2fb9ae --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-10-20-16-07-39.bpo-45532.kyhvis.rst @@ -0,0 +1,2 @@ +Update :data:`sys.version` to use ``main`` as fallback information. +Patch by Jeong YunWon. diff --git a/Modules/getbuildinfo.c b/Modules/getbuildinfo.c index 5f941a26e1..7cb7397a22 100644 --- a/Modules/getbuildinfo.c +++ b/Modules/getbuildinfo.c @@ -40,8 +40,9 @@ Py_GetBuildInfo(void) const char *revision = _Py_gitversion(); const char *sep = *revision ? ":" : ""; const char *gitid = _Py_gitidentifier(); - if (!(*gitid)) - gitid = "default"; + if (!(*gitid)) { + gitid = "main"; + } PyOS_snprintf(buildinfo, sizeof(buildinfo), "%s%s%s, %.20s, %.9s", gitid, sep, revision, DATE, TIME); -- cgit v1.2.1 From b2a989995e6b725c5c957927127832fd3fcecfa2 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 Oct 2021 11:50:09 -0700 Subject: bpo-45464: [doc] Explain that subclassing multiple exceptions is fragile (GH-29094) (GH-29104) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Pablo Galindo Salgado (cherry picked from commit dff0b713436e286bb1afdd7c6f3093c8e8db16dd) Co-authored-by: Łukasz Langa --- Doc/library/exceptions.rst | 23 ++++++++++++++++++++++ .../2021-10-20-16-26-53.bpo-45464.mOISBs.rst | 4 ++++ 2 files changed, 27 insertions(+) create mode 100644 Misc/NEWS.d/next/Documentation/2021-10-20-16-26-53.bpo-45464.mOISBs.rst diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 1a883ba19a..f8a692e812 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -34,6 +34,10 @@ class or one of its subclasses, and not from :exc:`BaseException`. More information on defining exceptions is available in the Python Tutorial under :ref:`tut-userexceptions`. + +Exception context +----------------- + When raising (or re-raising) an exception in an :keyword:`except` or :keyword:`finally` clause :attr:`__context__` is automatically set to the last exception caught; if the @@ -67,6 +71,25 @@ exceptions so that the final line of the traceback always shows the last exception that was raised. +Inheriting from built-in exceptions +----------------------------------- + +User code can create subclasses that inherit from an exception type. +It's recommended to only subclass one exception type at a time to avoid +any possible conflicts between how the bases handle the ``args`` +attribute, as well as due to possible memory layout incompatibilities. + +.. impl-detail:: + + Most built-in exceptions are implemented in C for efficiency, see: + :source:`Objects/exceptions.c`. Some have custom memory layouts + which makes it impossible to create a subclass that inherits from + multiple exception types. The memory layout of a type is an implementation + detail and might change between Python versions, leading to new + conflicts in the future. Therefore, it's recommended to avoid + subclassing multiple exception types altogether. + + Base classes ------------ diff --git a/Misc/NEWS.d/next/Documentation/2021-10-20-16-26-53.bpo-45464.mOISBs.rst b/Misc/NEWS.d/next/Documentation/2021-10-20-16-26-53.bpo-45464.mOISBs.rst new file mode 100644 index 0000000000..1721aa2c2d --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2021-10-20-16-26-53.bpo-45464.mOISBs.rst @@ -0,0 +1,4 @@ +Mention in the documentation of :ref:`Built-in Exceptions +` that inheriting from multiple exception types in a +single subclass is not recommended due to possible memory layout +incompatibility. -- cgit v1.2.1 From 64e83c711eb371d60fce64cae074c4d3311f6ece Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 Oct 2021 14:27:30 -0700 Subject: bpo-45192: Fix a bug that infers the type of an os.PathLike[bytes] object as str (GH-28323) (GH-29111) An object implementing the os.PathLike protocol can represent a file system path as a str or bytes object. Therefore, _infer_return_type function should infer os.PathLike[str] object as str type and os.PathLike[bytes] object as bytes type. (cherry picked from commit 6270d3eeaf17b50abc4f8f4d97790d66179638e4) Co-authored-by: Kyungmin Lee --- Lib/tempfile.py | 4 ++++ Lib/test/test_tempfile.py | 19 +++++++++++++++++++ .../Library/2021-09-14-15-52-47.bpo-45192.DjA-BI.rst | 5 +++++ 3 files changed, 28 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2021-09-14-15-52-47.bpo-45192.DjA-BI.rst diff --git a/Lib/tempfile.py b/Lib/tempfile.py index efcf7a7fb3..7b6821240f 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -88,6 +88,10 @@ def _infer_return_type(*args): for arg in args: if arg is None: continue + + if isinstance(arg, _os.PathLike): + arg = _os.fspath(arg) + if isinstance(arg, bytes): if return_type is str: raise TypeError("Can't mix bytes and non-bytes in " diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 96946a281a..2b0ec46a10 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -62,6 +62,25 @@ class TestLowLevelInternals(unittest.TestCase): def test_infer_return_type_pathlib(self): self.assertIs(str, tempfile._infer_return_type(pathlib.Path('/'))) + def test_infer_return_type_pathlike(self): + class Path: + def __init__(self, path): + self.path = path + + def __fspath__(self): + return self.path + + self.assertIs(str, tempfile._infer_return_type(Path('/'))) + self.assertIs(bytes, tempfile._infer_return_type(Path(b'/'))) + self.assertIs(str, tempfile._infer_return_type('', Path(''))) + self.assertIs(bytes, tempfile._infer_return_type(b'', Path(b''))) + self.assertIs(bytes, tempfile._infer_return_type(None, Path(b''))) + self.assertIs(str, tempfile._infer_return_type(None, Path(''))) + + with self.assertRaises(TypeError): + tempfile._infer_return_type('', Path(b'')) + with self.assertRaises(TypeError): + tempfile._infer_return_type(b'', Path('')) # Common functionality. diff --git a/Misc/NEWS.d/next/Library/2021-09-14-15-52-47.bpo-45192.DjA-BI.rst b/Misc/NEWS.d/next/Library/2021-09-14-15-52-47.bpo-45192.DjA-BI.rst new file mode 100644 index 0000000000..7dd9795aaa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-09-14-15-52-47.bpo-45192.DjA-BI.rst @@ -0,0 +1,5 @@ +Fix the ``tempfile._infer_return_type`` function so that the ``dir`` +argument of the :mod:`tempfile` functions accepts an object implementing the +``os.PathLike`` protocol. + +Patch by Kyungmin Lee. -- cgit v1.2.1 From 2a9ab75af32b1ee9f210ae2a0718990687d0f79d Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Wed, 20 Oct 2021 19:48:37 -0700 Subject: bpo-44559: [Enum] restore fixes lost in 3.9 reversion (GH-29114) - fix exception leaks - re-add deprecation warnings --- Doc/library/enum.rst | 14 ++++- Lib/enum.py | 60 ++++++++++++------- Lib/test/test_enum.py | 162 ++++++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 194 insertions(+), 42 deletions(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index e8e49425ab..b354a111a3 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -1125,9 +1125,9 @@ and raise an error if the two do not match:: _Private__names """"""""""""""" -Private names will be normal attributes in Python 3.10 instead of either an error +Private names will be normal attributes in Python 3.11 instead of either an error or a member (depending on if the name ends with an underscore). Using these names -in 3.9 will issue a :exc:`DeprecationWarning`. +in 3.10 will issue a :exc:`DeprecationWarning`. ``Enum`` member type @@ -1150,6 +1150,10 @@ all-uppercase names for members):: >>> FieldTypes.size.value 2 +.. note:: + + This behavior is deprecated and will be removed in 3.12. + .. versionchanged:: 3.5 @@ -1200,3 +1204,9 @@ all named flags and all named combinations of flags that are in the value:: >>> Color(7) # not named combination +.. note:: + + In 3.11 unnamed combinations of flags will only produce the canonical flag + members (aka single-value flags). So ``Color(7)`` will produce something + like ````. + diff --git a/Lib/enum.py b/Lib/enum.py index db79e66903..f5657a6eba 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -44,10 +44,11 @@ def _is_sunder(name): def _is_private(cls_name, name): # do not use `re` as `re` imports `enum` pattern = '_%s__' % (cls_name, ) + pat_len = len(pattern) if ( - len(name) >= 5 + len(name) > pat_len and name.startswith(pattern) - and name[len(pattern)] != '_' + and name[pat_len:pat_len+1] != ['_'] and (name[-1] != '_' or name[-2] != '_') ): return True @@ -392,12 +393,19 @@ class EnumMeta(type): start=start, ) - def __contains__(cls, member): - if not isinstance(member, Enum): + def __contains__(cls, obj): + if not isinstance(obj, Enum): + import warnings + warnings.warn( + "in 3.12 __contains__ will no longer raise TypeError, but will return True if\n" + "obj is a member or a member's value", + DeprecationWarning, + stacklevel=2, + ) raise TypeError( "unsupported operand type(s) for 'in': '%s' and '%s'" % ( - type(member).__qualname__, cls.__class__.__qualname__)) - return isinstance(member, cls) and member._name_ in cls._member_map_ + type(obj).__qualname__, cls.__class__.__qualname__)) + return isinstance(obj, cls) and obj._name_ in cls._member_map_ def __delattr__(cls, attr): # nicer error message when someone tries to delete an attribute @@ -580,7 +588,7 @@ class EnumMeta(type): return object, Enum def _find_data_type(bases): - data_types = [] + data_types = set() for chain in bases: candidate = None for base in chain.__mro__: @@ -588,19 +596,19 @@ class EnumMeta(type): continue elif issubclass(base, Enum): if base._member_type_ is not object: - data_types.append(base._member_type_) + data_types.add(base._member_type_) break elif '__new__' in base.__dict__: if issubclass(base, Enum): continue - data_types.append(candidate or base) + data_types.add(candidate or base) break else: candidate = candidate or base if len(data_types) > 1: raise TypeError('%r: too many data types: %r' % (class_name, data_types)) elif data_types: - return data_types[0] + return data_types.pop() else: return None @@ -693,19 +701,25 @@ class Enum(metaclass=EnumMeta): except Exception as e: exc = e result = None - if isinstance(result, cls): - return result - else: - ve_exc = ValueError("%r is not a valid %s" % (value, cls.__qualname__)) - if result is None and exc is None: - raise ve_exc - elif exc is None: - exc = TypeError( - 'error in %s._missing_: returned %r instead of None or a valid member' - % (cls.__name__, result) - ) - exc.__context__ = ve_exc - raise exc + try: + if isinstance(result, cls): + return result + else: + ve_exc = ValueError("%r is not a valid %s" % (value, cls.__qualname__)) + if result is None and exc is None: + raise ve_exc + elif exc is None: + exc = TypeError( + 'error in %s._missing_: returned %r instead of None or a valid member' + % (cls.__name__, result) + ) + if not isinstance(exc, ValueError): + exc.__context__ = ve_exc + raise exc + finally: + # ensure all variables that could hold an exception are destroyed + exc = None + ve_exc = None def _generate_next_value_(name, start, count, last_values): """ diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index eb1266b960..03cf3533fc 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -11,6 +11,7 @@ from pickle import dumps, loads, PicklingError, HIGHEST_PROTOCOL from test.support import ALWAYS_EQ, check__all__, threading_helper from datetime import timedelta +python_version = sys.version_info[:2] # for pickle tests try: @@ -347,17 +348,38 @@ class TestEnum(unittest.TestCase): self.assertTrue(IntLogic.true) self.assertFalse(IntLogic.false) - def test_contains(self): + @unittest.skipIf( + python_version >= (3, 12), + '__contains__ now returns True/False for all inputs', + ) + def test_contains_er(self): Season = self.Season self.assertIn(Season.AUTUMN, Season) with self.assertRaises(TypeError): - 3 in Season + with self.assertWarns(DeprecationWarning): + 3 in Season with self.assertRaises(TypeError): - 'AUTUMN' in Season - + with self.assertWarns(DeprecationWarning): + 'AUTUMN' in Season val = Season(3) self.assertIn(val, Season) + # + class OtherEnum(Enum): + one = 1; two = 2 + self.assertNotIn(OtherEnum.two, Season) + @unittest.skipIf( + python_version < (3, 12), + '__contains__ only works with enum memmbers before 3.12', + ) + def test_contains_tf(self): + Season = self.Season + self.assertIn(Season.AUTUMN, Season) + self.assertTrue(3 in Season) + self.assertFalse('AUTUMN' in Season) + val = Season(3) + self.assertIn(val, Season) + # class OtherEnum(Enum): one = 1; two = 2 self.assertNotIn(OtherEnum.two, Season) @@ -1932,6 +1954,38 @@ class TestEnum(unittest.TestCase): else: raise Exception('Exception not raised.') + def test_missing_exceptions_reset(self): + import weakref + # + class TestEnum(enum.Enum): + VAL1 = 'val1' + VAL2 = 'val2' + # + class Class1: + def __init__(self): + # Gracefully handle an exception of our own making + try: + raise ValueError() + except ValueError: + pass + # + class Class2: + def __init__(self): + # Gracefully handle an exception of Enum's making + try: + TestEnum('invalid_value') + except ValueError: + pass + # No strong refs here so these are free to die. + class_1_ref = weakref.ref(Class1()) + class_2_ref = weakref.ref(Class2()) + # + # The exception raised by Enum creates a reference loop and thus + # Class2 instances will stick around until the next gargage collection + # cycle, unlike Class1. + self.assertIs(class_1_ref(), None) + self.assertIs(class_2_ref(), None) + def test_multiple_mixin(self): class MaxMixin: @classproperty @@ -2085,7 +2139,7 @@ class TestEnum(unittest.TestCase): exec(code, global_ns, local_ls) @unittest.skipUnless( - sys.version_info[:2] == (3, 9), + python_version == (3, 9), 'private variables are now normal attributes', ) def test_warning_for_private_variables(self): @@ -2390,19 +2444,42 @@ class TestFlag(unittest.TestCase): test_pickle_dump_load(self.assertIs, FlagStooges.CURLY|FlagStooges.MOE) test_pickle_dump_load(self.assertIs, FlagStooges) - def test_contains(self): + @unittest.skipIf( + python_version >= (3, 12), + '__contains__ now returns True/False for all inputs', + ) + def test_contains_er(self): Open = self.Open Color = self.Color self.assertFalse(Color.BLACK in Open) self.assertFalse(Open.RO in Color) with self.assertRaises(TypeError): - 'BLACK' in Color + with self.assertWarns(DeprecationWarning): + 'BLACK' in Color with self.assertRaises(TypeError): - 'RO' in Open + with self.assertWarns(DeprecationWarning): + 'RO' in Open with self.assertRaises(TypeError): - 1 in Color + with self.assertWarns(DeprecationWarning): + 1 in Color with self.assertRaises(TypeError): - 1 in Open + with self.assertWarns(DeprecationWarning): + 1 in Open + + @unittest.skipIf( + python_version < (3, 12), + '__contains__ only works with enum memmbers before 3.12', + ) + def test_contains_tf(self): + Open = self.Open + Color = self.Color + self.assertFalse(Color.BLACK in Open) + self.assertFalse(Open.RO in Color) + self.assertFalse('BLACK' in Color) + self.assertFalse('RO' in Open) + self.assertTrue(1 in Color) + self.assertTrue(1 in Open) + def test_member_contains(self): Perm = self.Perm @@ -2883,7 +2960,11 @@ class TestIntFlag(unittest.TestCase): self.assertEqual(len(lst), len(Thing)) self.assertEqual(len(Thing), 0, Thing) - def test_contains(self): + @unittest.skipIf( + python_version >= (3, 12), + '__contains__ now returns True/False for all inputs', + ) + def test_contains_er(self): Open = self.Open Color = self.Color self.assertTrue(Color.GREEN in Color) @@ -2891,13 +2972,33 @@ class TestIntFlag(unittest.TestCase): self.assertFalse(Color.GREEN in Open) self.assertFalse(Open.RW in Color) with self.assertRaises(TypeError): - 'GREEN' in Color + with self.assertWarns(DeprecationWarning): + 'GREEN' in Color with self.assertRaises(TypeError): - 'RW' in Open + with self.assertWarns(DeprecationWarning): + 'RW' in Open with self.assertRaises(TypeError): - 2 in Color + with self.assertWarns(DeprecationWarning): + 2 in Color with self.assertRaises(TypeError): - 2 in Open + with self.assertWarns(DeprecationWarning): + 2 in Open + + @unittest.skipIf( + python_version < (3, 12), + '__contains__ only works with enum memmbers before 3.12', + ) + def test_contains_tf(self): + Open = self.Open + Color = self.Color + self.assertTrue(Color.GREEN in Color) + self.assertTrue(Open.RW in Open) + self.assertTrue(Color.GREEN in Open) + self.assertTrue(Open.RW in Color) + self.assertFalse('GREEN' in Color) + self.assertFalse('RW' in Open) + self.assertTrue(2 in Color) + self.assertTrue(2 in Open) def test_member_contains(self): Perm = self.Perm @@ -3267,7 +3368,7 @@ class TestIntEnumConvert(unittest.TestCase): if name[0:2] not in ('CO', '__')], [], msg='Names other than CONVERT_TEST_* found.') - @unittest.skipUnless(sys.version_info[:2] == (3, 8), + @unittest.skipUnless(python_version == (3, 8), '_convert was deprecated in 3.8') def test_convert_warn(self): with self.assertWarns(DeprecationWarning): @@ -3276,7 +3377,7 @@ class TestIntEnumConvert(unittest.TestCase): ('test.test_enum', '__main__')[__name__=='__main__'], filter=lambda x: x.startswith('CONVERT_TEST_')) - @unittest.skipUnless(sys.version_info >= (3, 9), + @unittest.skipUnless(python_version >= (3, 9), '_convert was removed in 3.9') def test_convert_raise(self): with self.assertRaises(AttributeError): @@ -3285,6 +3386,33 @@ class TestIntEnumConvert(unittest.TestCase): ('test.test_enum', '__main__')[__name__=='__main__'], filter=lambda x: x.startswith('CONVERT_TEST_')) +class TestHelpers(unittest.TestCase): + + sunder_names = '_bad_', '_good_', '_what_ho_' + dunder_names = '__mal__', '__bien__', '__que_que__' + private_names = '_MyEnum__private', '_MyEnum__still_private' + private_and_sunder_names = '_MyEnum__private_', '_MyEnum__also_private_' + random_names = 'okay', '_semi_private', '_weird__', '_MyEnum__' + + def test_sunder(self): + for name in self.sunder_names + self.private_and_sunder_names: + self.assertTrue(enum._is_sunder(name), '%r is a not sunder name?' % name) + for name in self.dunder_names + self.private_names + self.random_names: + self.assertFalse(enum._is_sunder(name), '%r is a sunder name?' % name) + + def test_dunder(self): + for name in self.dunder_names: + self.assertTrue(enum._is_dunder(name), '%r is a not dunder name?' % name) + for name in self.sunder_names + self.private_names + self.private_and_sunder_names + self.random_names: + self.assertFalse(enum._is_dunder(name), '%r is a dunder name?' % name) + + def test_is_private(self): + for name in self.private_names + self.private_and_sunder_names: + self.assertTrue(enum._is_private('MyEnum', name), '%r is a not private name?') + for name in self.sunder_names + self.dunder_names + self.random_names: + self.assertFalse(enum._is_private('MyEnum', name), '%r is a private name?') + if __name__ == '__main__': unittest.main() + -- cgit v1.2.1