Changes from v1: * Dropped some changes that were independently fixed[1] * No longer separate the f strings to their own patch * Use r strings when the value is a regular expression * Updated verification script
In retrospect a script to find the instances and apply fixes isn't that useful for review, so the attached script this time just looks for differences in the AST. Apply the series and run the script, with the two references to compare as arguments.
There are some intentional changes to the AST now though, as the r strings turn '\t' from a single character tab into a backslash and 't' character pair (similar for '\n'). This does not affect the correctness of the regular expression though.
v1: https://lore.kernel.org/all/20230814060704.79655-1-bgray@linux.ibm.com/
[1]: https://lore.kernel.org/all/20230816122133.1231599-1-vishalc@linux.ibm.com/
--- #!/usr/bin/env python3
""" Verify Python syntax trees are equivalent between two references """
import argparse import ast from pathlib import Path import subprocess as sp
def read_file(path: Path, ref: str) -> str: return sp.run(f"git show {ref}:{path}", stdout=sp.PIPE, shell=True, encoding="utf-8", check=True).stdout
parser = argparse.ArgumentParser("Compare Python ASTs between revisions") parser.add_argument("ref1", type=str, help="First revision to use") parser.add_argument("ref2", type=str, help="Second revision to use") args = parser.parse_args()
for pyfile in Path(".").glob("**/*.py"): try: ref1_content = read_file(pyfile, args.ref1) ref2_content = read_file(pyfile, args.ref2) except Exception as e: print(f"ERROR:{pyfile}: Failed to read ({e})") continue
try: ref1_syntax = ast.parse(ref1_content, filename=pyfile) ref2_syntax = ast.parse(ref2_content, filename=pyfile) except SyntaxError as e: print(f"ERROR:{pyfile}: Failed to parse, is it Python3? ({e})") continue
if ast.dump(ref1_syntax) != ast.dump(ref2_syntax): print(f"ERROR:{pyfile}: Revisions have different AST") cmd = f"diff <(git show {args.ref1}:{pyfile} | python -m ast) <(git show {args.ref2}:{pyfile} | python -m ast)" print(cmd) sp.run(cmd, shell=True) continue
Benjamin Gray (7): ia64: fix Python string escapes Documentation/sphinx: fix Python string escapes drivers/comedi: fix Python string escapes scripts: fix Python string escapes tools/perf: fix Python string escapes tools/power: fix Python string escapes selftests/bpf: fix Python string escapes
Documentation/sphinx/cdomain.py | 2 +- Documentation/sphinx/kernel_abi.py | 2 +- Documentation/sphinx/kernel_feat.py | 2 +- Documentation/sphinx/kerneldoc.py | 2 +- Documentation/sphinx/maintainers_include.py | 8 +++--- arch/ia64/scripts/unwcheck.py | 2 +- .../ni_routing/tools/convert_csv_to_c.py | 2 +- scripts/clang-tools/gen_compile_commands.py | 2 +- scripts/gdb/linux/symbols.py | 2 +- tools/perf/pmu-events/jevents.py | 2 +- .../scripts/python/arm-cs-trace-disasm.py | 4 +-- tools/perf/scripts/python/compaction-times.py | 2 +- .../scripts/python/exported-sql-viewer.py | 4 +-- tools/power/pm-graph/bootgraph.py | 12 ++++----- .../selftests/bpf/test_bpftool_synctypes.py | 26 +++++++++---------- tools/testing/selftests/bpf/test_offload.py | 2 +- 16 files changed, 38 insertions(+), 38 deletions(-)
-- 2.41.0
Python 3.6 introduced a DeprecationWarning for invalid escape sequences. This is upgraded to a SyntaxWarning in Python 3.12, and will eventually be a syntax error.
Fix these now to get ahead of it before it's an error.
Signed-off-by: Benjamin Gray bgray@linux.ibm.com --- arch/ia64/scripts/unwcheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/ia64/scripts/unwcheck.py b/arch/ia64/scripts/unwcheck.py index 9581742f0db2..adc24152d3b9 100644 --- a/arch/ia64/scripts/unwcheck.py +++ b/arch/ia64/scripts/unwcheck.py @@ -21,7 +21,7 @@ if len(sys.argv) != 2:
readelf = os.getenv("READELF", "readelf")
-start_pattern = re.compile("<([^>]*)>: [0x([0-9a-f]+)-0x([0-9a-f]+)]") +start_pattern = re.compile(r"<([^>]*)>: [0x([0-9a-f]+)-0x([0-9a-f]+)]") rlen_pattern = re.compile(".*rlen=([0-9]+)")
def check_func (func, slots, rlen_sum):
On Mon, Sep 11, 2023 at 11:08 PM Benjamin Gray bgray@linux.ibm.com wrote:
Python 3.6 introduced a DeprecationWarning for invalid escape sequences. This is upgraded to a SyntaxWarning in Python 3.12, and will eventually be a syntax error.
Fix these now to get ahead of it before it's an error.
Signed-off-by: Benjamin Gray bgray@linux.ibm.com
arch/ia64/scripts/unwcheck.py | 2 +-
Ard is proposing removing this script, along with the rest of the architecture: https://lore.kernel.org/linux-arch/20230911163129.3777603-2-ardb@google.com/
So this change can be dropped.
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/ia64/scripts/unwcheck.py b/arch/ia64/scripts/unwcheck.py index 9581742f0db2..adc24152d3b9 100644 --- a/arch/ia64/scripts/unwcheck.py +++ b/arch/ia64/scripts/unwcheck.py @@ -21,7 +21,7 @@ if len(sys.argv) != 2:
readelf = os.getenv("READELF", "readelf")
-start_pattern = re.compile("<([^>]*)>: [0x([0-9a-f]+)-0x([0-9a-f]+)]") +start_pattern = re.compile(r"<([^>]*)>: [0x([0-9a-f]+)-0x([0-9a-f]+)]") rlen_pattern = re.compile(".*rlen=([0-9]+)")
def check_func (func, slots, rlen_sum):
2.41.0
Python 3.6 introduced a DeprecationWarning for invalid escape sequences. This is upgraded to a SyntaxWarning in Python 3.12, and will eventually be a syntax error.
Fix these now to get ahead of it before it's an error.
Signed-off-by: Benjamin Gray bgray@linux.ibm.com --- Documentation/sphinx/cdomain.py | 2 +- Documentation/sphinx/kernel_abi.py | 2 +- Documentation/sphinx/kernel_feat.py | 2 +- Documentation/sphinx/kerneldoc.py | 2 +- Documentation/sphinx/maintainers_include.py | 8 ++++---- 5 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/Documentation/sphinx/cdomain.py b/Documentation/sphinx/cdomain.py index a99716bf44b5..de5d132d94c5 100644 --- a/Documentation/sphinx/cdomain.py +++ b/Documentation/sphinx/cdomain.py @@ -93,7 +93,7 @@ def markup_ctype_refs(match): # RE_expr = re.compile(r':c:(expr|texpr):`([^`]+)`') def markup_c_expr(match): - return '\ ``' + match.group(2) + '``\ ' + return '\ ``' + match.group(2) + '``\ '
# # Parse Sphinx 3.x C markups, replacing them by backward-compatible ones diff --git a/Documentation/sphinx/kernel_abi.py b/Documentation/sphinx/kernel_abi.py index b5feb5b1d905..49797c55479c 100644 --- a/Documentation/sphinx/kernel_abi.py +++ b/Documentation/sphinx/kernel_abi.py @@ -138,7 +138,7 @@ class KernelCmd(Directive): code_block += "\n " + l lines = code_block + "\n\n"
- line_regex = re.compile("^.. LINENO (\S+)#([0-9]+)$") + line_regex = re.compile(r"^.. LINENO (\S+)#([0-9]+)$") ln = 0 n = 0 f = fname diff --git a/Documentation/sphinx/kernel_feat.py b/Documentation/sphinx/kernel_feat.py index 27b701ed3681..b5fa2f0542a5 100644 --- a/Documentation/sphinx/kernel_feat.py +++ b/Documentation/sphinx/kernel_feat.py @@ -104,7 +104,7 @@ class KernelFeat(Directive):
lines = self.runCmd(cmd, shell=True, cwd=cwd, env=shell_env)
- line_regex = re.compile("^.. FILE (\S+)$") + line_regex = re.compile(r"^.. FILE (\S+)$")
out_lines = ""
diff --git a/Documentation/sphinx/kerneldoc.py b/Documentation/sphinx/kerneldoc.py index 9395892c7ba3..8dc134904b90 100644 --- a/Documentation/sphinx/kerneldoc.py +++ b/Documentation/sphinx/kerneldoc.py @@ -130,7 +130,7 @@ class KernelDocDirective(Directive): result = ViewList()
lineoffset = 0; - line_regex = re.compile("^.. LINENO ([0-9]+)$") + line_regex = re.compile(r"^.. LINENO ([0-9]+)$") for line in lines: match = line_regex.search(line) if match: diff --git a/Documentation/sphinx/maintainers_include.py b/Documentation/sphinx/maintainers_include.py index 328b3631a585..dcad0fff4723 100755 --- a/Documentation/sphinx/maintainers_include.py +++ b/Documentation/sphinx/maintainers_include.py @@ -77,7 +77,7 @@ class MaintainersInclude(Include): line = line.rstrip()
# Linkify all non-wildcard refs to ReST files in Documentation/. - pat = '(Documentation/([^\s?*]*).rst)' + pat = r'(Documentation/([^\s?*]*).rst)' m = re.search(pat, line) if m: # maintainers.rst is in a subdirectory, so include "../". @@ -90,11 +90,11 @@ class MaintainersInclude(Include): output = "| %s" % (line.replace("\", "\\")) # Look for and record field letter to field name mappings: # R: Designated *reviewer*: FullName address@domain - m = re.search("\s(\S):\s", line) + m = re.search(r"\s(\S):\s", line) if m: field_letter = m.group(1) if field_letter and not field_letter in fields: - m = re.search("*([^*]+)*", line) + m = re.search(r"*([^*]+)*", line) if m: fields[field_letter] = m.group(1) elif subsystems: @@ -112,7 +112,7 @@ class MaintainersInclude(Include): field_content = ""
# Collapse whitespace in subsystem name. - heading = re.sub("\s+", " ", line) + heading = re.sub(r"\s+", " ", line) output = output + "%s\n%s" % (heading, "~" * len(heading)) field_prev = "" else:
Benjamin Gray bgray@linux.ibm.com writes:
Python 3.6 introduced a DeprecationWarning for invalid escape sequences. This is upgraded to a SyntaxWarning in Python 3.12, and will eventually be a syntax error.
Fix these now to get ahead of it before it's an error.
Signed-off-by: Benjamin Gray bgray@linux.ibm.com
Documentation/sphinx/cdomain.py | 2 +- Documentation/sphinx/kernel_abi.py | 2 +- Documentation/sphinx/kernel_feat.py | 2 +- Documentation/sphinx/kerneldoc.py | 2 +- Documentation/sphinx/maintainers_include.py | 8 ++++---- 5 files changed, 8 insertions(+), 8 deletions(-)
I've applied this one to the docs tree, thanks.
jon
Python 3.6 introduced a DeprecationWarning for invalid escape sequences. This is upgraded to a SyntaxWarning in Python 3.12, and will eventually be a syntax error.
Fix these now to get ahead of it before it's an error.
Signed-off-by: Benjamin Gray bgray@linux.ibm.com --- drivers/comedi/drivers/ni_routing/tools/convert_csv_to_c.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/comedi/drivers/ni_routing/tools/convert_csv_to_c.py b/drivers/comedi/drivers/ni_routing/tools/convert_csv_to_c.py index 90378fb50580..d19101fc2a94 100755 --- a/drivers/comedi/drivers/ni_routing/tools/convert_csv_to_c.py +++ b/drivers/comedi/drivers/ni_routing/tools/convert_csv_to_c.py @@ -44,7 +44,7 @@ def routedict_to_structinit_single(name, D, return_name=False):
lines.append('\t\t[B({})] = {{'.format(D0_sig)) for D1_sig, value in D1: - if not re.match('[VIU]([^)]*)', value): + if not re.match(r'[VIU]([^)]*)', value): sys.stderr.write('Invalid register format: {}\n'.format(repr(value))) sys.stderr.write( 'Register values should be formatted with V(),I(),or U()\n')
On 12/09/2023 07:07, Benjamin Gray wrote:
Python 3.6 introduced a DeprecationWarning for invalid escape sequences. This is upgraded to a SyntaxWarning in Python 3.12, and will eventually be a syntax error.
Fix these now to get ahead of it before it's an error.
Signed-off-by: Benjamin Gray bgray@linux.ibm.com
drivers/comedi/drivers/ni_routing/tools/convert_csv_to_c.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/comedi/drivers/ni_routing/tools/convert_csv_to_c.py b/drivers/comedi/drivers/ni_routing/tools/convert_csv_to_c.py index 90378fb50580..d19101fc2a94 100755 --- a/drivers/comedi/drivers/ni_routing/tools/convert_csv_to_c.py +++ b/drivers/comedi/drivers/ni_routing/tools/convert_csv_to_c.py @@ -44,7 +44,7 @@ def routedict_to_structinit_single(name, D, return_name=False): lines.append('\t\t[B({})] = {{'.format(D0_sig)) for D1_sig, value in D1:
if not re.match('[VIU]\([^)]*\)', value):
if not re.match(r'[VIU]\([^)]*\)', value): sys.stderr.write('Invalid register format: {}\n'.format(repr(value))) sys.stderr.write( 'Register values should be formatted with V(),I(),or U()\n')
Looks good thanks! I ran the modified script using 'make everything' in the directory (after setting up a python venv to install ctypesgen) and it didn't break anything. (There were some harmless errors output by ctypesgen due to failing to parse some GCC extensions, but those are nothing to do with this patch and the generated C files are OK.)
Reviewed-by: Ian Abbott abbotti@mev.co.uk
Python 3.6 introduced a DeprecationWarning for invalid escape sequences. This is upgraded to a SyntaxWarning in Python 3.12, and will eventually be a syntax error.
Fix these now to get ahead of it before it's an error.
Signed-off-by: Benjamin Gray bgray@linux.ibm.com --- scripts/clang-tools/gen_compile_commands.py | 2 +- scripts/gdb/linux/symbols.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/scripts/clang-tools/gen_compile_commands.py b/scripts/clang-tools/gen_compile_commands.py index a84cc5737c2c..649a80005c83 100755 --- a/scripts/clang-tools/gen_compile_commands.py +++ b/scripts/clang-tools/gen_compile_commands.py @@ -170,7 +170,7 @@ def process_line(root_directory, command_prefix, file_path): # escape the pound sign '#', either as '#' or '$(pound)' (depending on the # kernel version). The compile_commands.json file is not interepreted # by Make, so this code replaces the escaped version with '#'. - prefix = command_prefix.replace('#', '#').replace('$(pound)', '#') + prefix = command_prefix.replace('\#', '#').replace('$(pound)', '#')
# Use os.path.abspath() to normalize the path resolving '.' and '..' . abs_path = os.path.abspath(os.path.join(root_directory, file_path)) diff --git a/scripts/gdb/linux/symbols.py b/scripts/gdb/linux/symbols.py index 5179edd1b627..2a55c4b83306 100644 --- a/scripts/gdb/linux/symbols.py +++ b/scripts/gdb/linux/symbols.py @@ -82,7 +82,7 @@ lx-symbols command.""" self.module_files_updated = True
def _get_module_file(self, module_name): - module_pattern = ".*/{0}.ko(?:.debug)?$".format( + module_pattern = r".*/{0}.ko(?:.debug)?$".format( module_name.replace("_", r"[_-]")) for name in self.module_files: if re.match(module_pattern, name) and os.path.exists(name):
Python 3.6 introduced a DeprecationWarning for invalid escape sequences. This is upgraded to a SyntaxWarning in Python 3.12, and will eventually be a syntax error.
Fix these now to get ahead of it before it's an error.
Signed-off-by: Benjamin Gray bgray@linux.ibm.com --- tools/perf/pmu-events/jevents.py | 2 +- tools/perf/scripts/python/arm-cs-trace-disasm.py | 4 ++-- tools/perf/scripts/python/compaction-times.py | 2 +- tools/perf/scripts/python/exported-sql-viewer.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py index a7e88332276d..980f080a5a2c 100755 --- a/tools/perf/pmu-events/jevents.py +++ b/tools/perf/pmu-events/jevents.py @@ -83,7 +83,7 @@ def c_len(s: str) -> int: """Return the length of s a C string
This doesn't handle all escape characters properly. It first assumes - all \ are for escaping, it then adjusts as it will have over counted + all \ are for escaping, it then adjusts as it will have over counted \. The code uses \000 rather than \0 as a terminator as an adjacent number would be folded into a string of \0 (ie. "\0" + "5" doesn't equal a terminator followed by the number 5 but the escape of diff --git a/tools/perf/scripts/python/arm-cs-trace-disasm.py b/tools/perf/scripts/python/arm-cs-trace-disasm.py index d59ff53f1d94..de58991c78bb 100755 --- a/tools/perf/scripts/python/arm-cs-trace-disasm.py +++ b/tools/perf/scripts/python/arm-cs-trace-disasm.py @@ -45,8 +45,8 @@ parser = OptionParser(option_list=option_list) # Initialize global dicts and regular expression disasm_cache = dict() cpu_data = dict() -disasm_re = re.compile("^\s*([0-9a-fA-F]+):") -disasm_func_re = re.compile("^\s*([0-9a-fA-F]+)\s.*:") +disasm_re = re.compile(r"^\s*([0-9a-fA-F]+):") +disasm_func_re = re.compile(r"^\s*([0-9a-fA-F]+)\s.*:") cache_size = 64*1024
glb_source_file_name = None diff --git a/tools/perf/scripts/python/compaction-times.py b/tools/perf/scripts/python/compaction-times.py index 2560a042dc6f..9401f7c14747 100644 --- a/tools/perf/scripts/python/compaction-times.py +++ b/tools/perf/scripts/python/compaction-times.py @@ -260,7 +260,7 @@ def pr_help():
comm_re = None pid_re = None -pid_regex = "^(\d*)-(\d*)$|^(\d*)$" +pid_regex = r"^(\d*)-(\d*)$|^(\d*)$"
opt_proc = popt.DISP_DFL opt_disp = topt.DISP_ALL diff --git a/tools/perf/scripts/python/exported-sql-viewer.py b/tools/perf/scripts/python/exported-sql-viewer.py index 13f2d8a81610..121cf61ba1b3 100755 --- a/tools/perf/scripts/python/exported-sql-viewer.py +++ b/tools/perf/scripts/python/exported-sql-viewer.py @@ -677,8 +677,8 @@ class CallGraphModelBase(TreeModel): # sqlite supports GLOB (text only) which uses * and ? and is case sensitive if not self.glb.dbref.is_sqlite3: # Escape % and _ - s = value.replace("%", "%") - s = s.replace("_", "_") + s = value.replace("%", "\%") + s = s.replace("_", "\_") # Translate * and ? into SQL LIKE pattern characters % and _ trans = string.maketrans("*?", "%_") match = " LIKE '" + str(s).translate(trans) + "'"
On 12/09/23 09:07, Benjamin Gray wrote:
Python 3.6 introduced a DeprecationWarning for invalid escape sequences. This is upgraded to a SyntaxWarning in Python 3.12, and will eventually be a syntax error.
Fix these now to get ahead of it before it's an error.
Signed-off-by: Benjamin Gray bgray@linux.ibm.com
tools/perf/pmu-events/jevents.py | 2 +- tools/perf/scripts/python/arm-cs-trace-disasm.py | 4 ++-- tools/perf/scripts/python/compaction-times.py | 2 +- tools/perf/scripts/python/exported-sql-viewer.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py index a7e88332276d..980f080a5a2c 100755 --- a/tools/perf/pmu-events/jevents.py +++ b/tools/perf/pmu-events/jevents.py @@ -83,7 +83,7 @@ def c_len(s: str) -> int: """Return the length of s a C string This doesn't handle all escape characters properly. It first assumes
- all \ are for escaping, it then adjusts as it will have over counted
- all \ are for escaping, it then adjusts as it will have over counted
It looks like the whole string should be a raw string
\. The code uses \000 rather than \0 as a terminator as an adjacent number would be folded into a string of \0 (ie. "\0" + "5" doesn't equal a terminator followed by the number 5 but the escape of diff --git a/tools/perf/scripts/python/arm-cs-trace-disasm.py b/tools/perf/scripts/python/arm-cs-trace-disasm.py index d59ff53f1d94..de58991c78bb 100755 --- a/tools/perf/scripts/python/arm-cs-trace-disasm.py +++ b/tools/perf/scripts/python/arm-cs-trace-disasm.py @@ -45,8 +45,8 @@ parser = OptionParser(option_list=option_list) # Initialize global dicts and regular expression disasm_cache = dict() cpu_data = dict() -disasm_re = re.compile("^\s*([0-9a-fA-F]+):") -disasm_func_re = re.compile("^\s*([0-9a-fA-F]+)\s.*:") +disasm_re = re.compile(r"^\s*([0-9a-fA-F]+):") +disasm_func_re = re.compile(r"^\s*([0-9a-fA-F]+)\s.*:") cache_size = 64*1024 glb_source_file_name = None diff --git a/tools/perf/scripts/python/compaction-times.py b/tools/perf/scripts/python/compaction-times.py index 2560a042dc6f..9401f7c14747 100644 --- a/tools/perf/scripts/python/compaction-times.py +++ b/tools/perf/scripts/python/compaction-times.py @@ -260,7 +260,7 @@ def pr_help(): comm_re = None pid_re = None -pid_regex = "^(\d*)-(\d*)$|^(\d*)$" +pid_regex = r"^(\d*)-(\d*)$|^(\d*)$" opt_proc = popt.DISP_DFL opt_disp = topt.DISP_ALL diff --git a/tools/perf/scripts/python/exported-sql-viewer.py b/tools/perf/scripts/python/exported-sql-viewer.py index 13f2d8a81610..121cf61ba1b3 100755 --- a/tools/perf/scripts/python/exported-sql-viewer.py +++ b/tools/perf/scripts/python/exported-sql-viewer.py @@ -677,8 +677,8 @@ class CallGraphModelBase(TreeModel): # sqlite supports GLOB (text only) which uses * and ? and is case sensitive if not self.glb.dbref.is_sqlite3: # Escape % and _
s = value.replace("%", "\%")s = s.replace("_", "\_")
s = value.replace("%", "\\%")s = s.replace("_", "\\_")
Raw strings seem more readable, so could be used here too
# Translate * and ? into SQL LIKE pattern characters % and _ trans = string.maketrans("*?", "%_") match = " LIKE '" + str(s).translate(trans) + "'"
On 12/9/23 8:56 pm, Adrian Hunter wrote:
On 12/09/23 09:07, Benjamin Gray wrote:
diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py index a7e88332276d..980f080a5a2c 100755 --- a/tools/perf/pmu-events/jevents.py +++ b/tools/perf/pmu-events/jevents.py @@ -83,7 +83,7 @@ def c_len(s: str) -> int: """Return the length of s a C string This doesn't handle all escape characters properly. It first assumes
- all \ are for escaping, it then adjusts as it will have over counted
- all \ are for escaping, it then adjusts as it will have over counted
It looks like the whole string should be a raw string
...
s = value.replace("%", "\%")s = s.replace("_", "\_")
s = value.replace("%", "\\%")s = s.replace("_", "\\_")Raw strings seem more readable, so could be used here too
Yeah, sounds good. I normally use r strings only for regex, but there shouldn't be any ambiguity here (it might have been misleading if the search argument to replace looked like a regex).
Having the docstring be an r string is a good catch. There's probably a few like that in the kernel, but finding them is a little more complicated because they might be 'valid' syntax (e.g., the '\000' just becomes a null byte. This series is focused on the syntax errors though, so I'll just leave it be.
How is the following? --- Subject: [PATCH] tools/perf: fix Python string escapes
Python 3.6 introduced a DeprecationWarning for invalid escape sequences. This is upgraded to a SyntaxWarning in Python 3.12, and will eventually be a syntax error.
Fix these now to get ahead of it before it's an error.
Signed-off-by: Benjamin Gray bgray@linux.ibm.com --- tools/perf/pmu-events/jevents.py | 2 +- tools/perf/scripts/python/arm-cs-trace-disasm.py | 4 ++-- tools/perf/scripts/python/compaction-times.py | 2 +- tools/perf/scripts/python/exported-sql-viewer.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py index a7e88332276d..1b4519333a28 100755 --- a/tools/perf/pmu-events/jevents.py +++ b/tools/perf/pmu-events/jevents.py @@ -80,7 +80,7 @@ def file_name_to_table_name(prefix: str, parents: Sequence[str],
def c_len(s: str) -> int: - """Return the length of s a C string + r"""Return the length of s a C string
This doesn't handle all escape characters properly. It first assumes all \ are for escaping, it then adjusts as it will have over counted diff --git a/tools/perf/scripts/python/arm-cs-trace-disasm.py b/tools/perf/scripts/python/arm-cs-trace-disasm.py index d59ff53f1d94..de58991c78bb 100755 --- a/tools/perf/scripts/python/arm-cs-trace-disasm.py +++ b/tools/perf/scripts/python/arm-cs-trace-disasm.py @@ -45,8 +45,8 @@ parser = OptionParser(option_list=option_list) # Initialize global dicts and regular expression disasm_cache = dict() cpu_data = dict() -disasm_re = re.compile("^\s*([0-9a-fA-F]+):") -disasm_func_re = re.compile("^\s*([0-9a-fA-F]+)\s.*:") +disasm_re = re.compile(r"^\s*([0-9a-fA-F]+):") +disasm_func_re = re.compile(r"^\s*([0-9a-fA-F]+)\s.*:") cache_size = 64*1024
glb_source_file_name = None diff --git a/tools/perf/scripts/python/compaction-times.py b/tools/perf/scripts/python/compaction-times.py index 2560a042dc6f..9401f7c14747 100644 --- a/tools/perf/scripts/python/compaction-times.py +++ b/tools/perf/scripts/python/compaction-times.py @@ -260,7 +260,7 @@ def pr_help():
comm_re = None pid_re = None -pid_regex = "^(\d*)-(\d*)$|^(\d*)$" +pid_regex = r"^(\d*)-(\d*)$|^(\d*)$"
opt_proc = popt.DISP_DFL opt_disp = topt.DISP_ALL diff --git a/tools/perf/scripts/python/exported-sql-viewer.py b/tools/perf/scripts/python/exported-sql-viewer.py index 13f2d8a81610..78763531fe5a 100755 --- a/tools/perf/scripts/python/exported-sql-viewer.py +++ b/tools/perf/scripts/python/exported-sql-viewer.py @@ -677,8 +677,8 @@ class CallGraphModelBase(TreeModel): # sqlite supports GLOB (text only) which uses * and ? and is case sensitive if not self.glb.dbref.is_sqlite3: # Escape % and _ - s = value.replace("%", "%") - s = s.replace("_", "_") + s = value.replace("%", r"%") + s = s.replace("_", r"_") # Translate * and ? into SQL LIKE pattern characters % and _ trans = string.maketrans("*?", "%_") match = " LIKE '" + str(s).translate(trans) + "'"
On 13/09/23 03:26, Benjamin Gray wrote:
On 12/9/23 8:56 pm, Adrian Hunter wrote:
On 12/09/23 09:07, Benjamin Gray wrote:
diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py index a7e88332276d..980f080a5a2c 100755 --- a/tools/perf/pmu-events/jevents.py +++ b/tools/perf/pmu-events/jevents.py @@ -83,7 +83,7 @@ def c_len(s: str) -> int: """Return the length of s a C string This doesn't handle all escape characters properly. It first assumes - all \ are for escaping, it then adjusts as it will have over counted + all \ are for escaping, it then adjusts as it will have over counted
It looks like the whole string should be a raw string
...
- s = value.replace("%", "%") - s = s.replace("_", "_") + s = value.replace("%", "\%") + s = s.replace("_", "\_")
Raw strings seem more readable, so could be used here too
Yeah, sounds good. I normally use r strings only for regex, but there shouldn't be any ambiguity here (it might have been misleading if the search argument to replace looked like a regex).
Having the docstring be an r string is a good catch. There's probably a few like that in the kernel, but finding them is a little more complicated because they might be 'valid' syntax (e.g., the '\000' just becomes a null byte. This series is focused on the syntax errors though, so I'll just leave it be.
How is the following?
Subject: [PATCH] tools/perf: fix Python string escapes
Python 3.6 introduced a DeprecationWarning for invalid escape sequences. This is upgraded to a SyntaxWarning in Python 3.12, and will eventually be a syntax error.
Fix these now to get ahead of it before it's an error.
Signed-off-by: Benjamin Gray bgray@linux.ibm.com
Acked-by: Adrian Hunter adrian.hunter@intel.com
tools/perf/pmu-events/jevents.py | 2 +- tools/perf/scripts/python/arm-cs-trace-disasm.py | 4 ++-- tools/perf/scripts/python/compaction-times.py | 2 +- tools/perf/scripts/python/exported-sql-viewer.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py index a7e88332276d..1b4519333a28 100755 --- a/tools/perf/pmu-events/jevents.py +++ b/tools/perf/pmu-events/jevents.py @@ -80,7 +80,7 @@ def file_name_to_table_name(prefix: str, parents: Sequence[str],
def c_len(s: str) -> int: - """Return the length of s a C string + r"""Return the length of s a C string
This doesn't handle all escape characters properly. It first assumes all \ are for escaping, it then adjusts as it will have over counted diff --git a/tools/perf/scripts/python/arm-cs-trace-disasm.py b/tools/perf/scripts/python/arm-cs-trace-disasm.py index d59ff53f1d94..de58991c78bb 100755 --- a/tools/perf/scripts/python/arm-cs-trace-disasm.py +++ b/tools/perf/scripts/python/arm-cs-trace-disasm.py @@ -45,8 +45,8 @@ parser = OptionParser(option_list=option_list) # Initialize global dicts and regular expression disasm_cache = dict() cpu_data = dict() -disasm_re = re.compile("^\s*([0-9a-fA-F]+):") -disasm_func_re = re.compile("^\s*([0-9a-fA-F]+)\s.*:") +disasm_re = re.compile(r"^\s*([0-9a-fA-F]+):") +disasm_func_re = re.compile(r"^\s*([0-9a-fA-F]+)\s.*:") cache_size = 64*1024
glb_source_file_name = None diff --git a/tools/perf/scripts/python/compaction-times.py b/tools/perf/scripts/python/compaction-times.py index 2560a042dc6f..9401f7c14747 100644 --- a/tools/perf/scripts/python/compaction-times.py +++ b/tools/perf/scripts/python/compaction-times.py @@ -260,7 +260,7 @@ def pr_help():
comm_re = None pid_re = None -pid_regex = "^(\d*)-(\d*)$|^(\d*)$" +pid_regex = r"^(\d*)-(\d*)$|^(\d*)$"
opt_proc = popt.DISP_DFL opt_disp = topt.DISP_ALL diff --git a/tools/perf/scripts/python/exported-sql-viewer.py b/tools/perf/scripts/python/exported-sql-viewer.py index 13f2d8a81610..78763531fe5a 100755 --- a/tools/perf/scripts/python/exported-sql-viewer.py +++ b/tools/perf/scripts/python/exported-sql-viewer.py @@ -677,8 +677,8 @@ class CallGraphModelBase(TreeModel): # sqlite supports GLOB (text only) which uses * and ? and is case sensitive if not self.glb.dbref.is_sqlite3: # Escape % and _ - s = value.replace("%", "%") - s = s.replace("_", "_") + s = value.replace("%", r"%") + s = s.replace("_", r"_") # Translate * and ? into SQL LIKE pattern characters % and _ trans = string.maketrans("*?", "%_") match = " LIKE '" + str(s).translate(trans) + "'"
Em Wed, Sep 13, 2023 at 08:53:26AM +0300, Adrian Hunter escreveu:
On 13/09/23 03:26, Benjamin Gray wrote:
On 12/9/23 8:56 pm, Adrian Hunter wrote:
On 12/09/23 09:07, Benjamin Gray wrote:
diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py index a7e88332276d..980f080a5a2c 100755 --- a/tools/perf/pmu-events/jevents.py +++ b/tools/perf/pmu-events/jevents.py @@ -83,7 +83,7 @@ def c_len(s: str) -> int: """Return the length of s a C string This doesn't handle all escape characters properly. It first assumes - all \ are for escaping, it then adjusts as it will have over counted + all \ are for escaping, it then adjusts as it will have over counted
It looks like the whole string should be a raw string
...
- s = value.replace("%", "%") - s = s.replace("_", "_") + s = value.replace("%", "\%") + s = s.replace("_", "\_")
Raw strings seem more readable, so could be used here too
Yeah, sounds good. I normally use r strings only for regex, but there shouldn't be any ambiguity here (it might have been misleading if the search argument to replace looked like a regex).
Having the docstring be an r string is a good catch. There's probably a few like that in the kernel, but finding them is a little more complicated because they might be 'valid' syntax (e.g., the '\000' just becomes a null byte. This series is focused on the syntax errors though, so I'll just leave it be.
How is the following?
Subject: [PATCH] tools/perf: fix Python string escapes
Python 3.6 introduced a DeprecationWarning for invalid escape sequences. This is upgraded to a SyntaxWarning in Python 3.12, and will eventually be a syntax error.
Fix these now to get ahead of it before it's an error.
Signed-off-by: Benjamin Gray bgray@linux.ibm.com
Acked-by: Adrian Hunter adrian.hunter@intel.com
applied the tools/perf one.
- Arnaldo
tools/perf/pmu-events/jevents.py | 2 +- tools/perf/scripts/python/arm-cs-trace-disasm.py | 4 ++-- tools/perf/scripts/python/compaction-times.py | 2 +- tools/perf/scripts/python/exported-sql-viewer.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py index a7e88332276d..1b4519333a28 100755 --- a/tools/perf/pmu-events/jevents.py +++ b/tools/perf/pmu-events/jevents.py @@ -80,7 +80,7 @@ def file_name_to_table_name(prefix: str, parents: Sequence[str],
def c_len(s: str) -> int: - """Return the length of s a C string + r"""Return the length of s a C string
This doesn't handle all escape characters properly. It first assumes all \ are for escaping, it then adjusts as it will have over counted diff --git a/tools/perf/scripts/python/arm-cs-trace-disasm.py b/tools/perf/scripts/python/arm-cs-trace-disasm.py index d59ff53f1d94..de58991c78bb 100755 --- a/tools/perf/scripts/python/arm-cs-trace-disasm.py +++ b/tools/perf/scripts/python/arm-cs-trace-disasm.py @@ -45,8 +45,8 @@ parser = OptionParser(option_list=option_list) # Initialize global dicts and regular expression disasm_cache = dict() cpu_data = dict() -disasm_re = re.compile("^\s*([0-9a-fA-F]+):") -disasm_func_re = re.compile("^\s*([0-9a-fA-F]+)\s.*:") +disasm_re = re.compile(r"^\s*([0-9a-fA-F]+):") +disasm_func_re = re.compile(r"^\s*([0-9a-fA-F]+)\s.*:") cache_size = 64*1024
glb_source_file_name = None diff --git a/tools/perf/scripts/python/compaction-times.py b/tools/perf/scripts/python/compaction-times.py index 2560a042dc6f..9401f7c14747 100644 --- a/tools/perf/scripts/python/compaction-times.py +++ b/tools/perf/scripts/python/compaction-times.py @@ -260,7 +260,7 @@ def pr_help():
comm_re = None pid_re = None -pid_regex = "^(\d*)-(\d*)$|^(\d*)$" +pid_regex = r"^(\d*)-(\d*)$|^(\d*)$"
opt_proc = popt.DISP_DFL opt_disp = topt.DISP_ALL diff --git a/tools/perf/scripts/python/exported-sql-viewer.py b/tools/perf/scripts/python/exported-sql-viewer.py index 13f2d8a81610..78763531fe5a 100755 --- a/tools/perf/scripts/python/exported-sql-viewer.py +++ b/tools/perf/scripts/python/exported-sql-viewer.py @@ -677,8 +677,8 @@ class CallGraphModelBase(TreeModel): # sqlite supports GLOB (text only) which uses * and ? and is case sensitive if not self.glb.dbref.is_sqlite3: # Escape % and _ - s = value.replace("%", "%") - s = s.replace("_", "_") + s = value.replace("%", r"%") + s = s.replace("_", r"_") # Translate * and ? into SQL LIKE pattern characters % and _ trans = string.maketrans("*?", "%_") match = " LIKE '" + str(s).translate(trans) + "'"
Python 3.6 introduced a DeprecationWarning for invalid escape sequences. This is upgraded to a SyntaxWarning in Python 3.12, and will eventually be a syntax error.
Fix these now to get ahead of it before it's an error.
Signed-off-by: Benjamin Gray bgray@linux.ibm.com --- tools/power/pm-graph/bootgraph.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/tools/power/pm-graph/bootgraph.py b/tools/power/pm-graph/bootgraph.py index f96f50e0c336..c53cfdba0365 100755 --- a/tools/power/pm-graph/bootgraph.py +++ b/tools/power/pm-graph/bootgraph.py @@ -77,12 +77,12 @@ class SystemValues(aslib.SystemValues): fp.close() self.testdir = datetime.now().strftime('boot-%y%m%d-%H%M%S') def kernelVersion(self, msg): - m = re.match('^[Ll]inux *[Vv]ersion *(?P<v>\S*) .*', msg) + m = re.match(r'^[Ll]inux *[Vv]ersion *(?P<v>\S*) .*', msg) if m: return m.group('v') return 'unknown' def checkFtraceKernelVersion(self): - m = re.match('^(?P<x>[0-9]*).(?P<y>[0-9]*).(?P<z>[0-9]*).*', self.kernel) + m = re.match(r'^(?P<x>[0-9]*).(?P<y>[0-9]*).(?P<z>[0-9]*).*', self.kernel) if m: val = tuple(map(int, m.groups())) if val >= (4, 10, 0): @@ -324,7 +324,7 @@ def parseKernelLog(): idx = line.find('[') if idx > 1: line = line[idx:] - m = re.match('[ \t]*([ *)(?P<ktime>[0-9.]*)(]) (?P<msg>.*)', line) + m = re.match(r'[ \t]*([ *)(?P<ktime>[0-9.]*)(]) (?P<msg>.*)', line) if(not m): continue ktime = float(m.group('ktime')) @@ -336,20 +336,20 @@ def parseKernelLog(): if(not sysvals.stamp['kernel']): sysvals.stamp['kernel'] = sysvals.kernelVersion(msg) continue - m = re.match('.* setting system clock to (?P<d>[0-9-]*)[ A-Z](?P<t>[0-9:]*) UTC.*', msg) + m = re.match(r'.* setting system clock to (?P<d>[0-9-]*)[ A-Z](?P<t>[0-9:]*) UTC.*', msg) if(m): bt = datetime.strptime(m.group('d')+' '+m.group('t'), '%Y-%m-%d %H:%M:%S') bt = bt - timedelta(seconds=int(ktime)) data.boottime = bt.strftime('%Y-%m-%d_%H:%M:%S') sysvals.stamp['time'] = bt.strftime('%B %d %Y, %I:%M:%S %p') continue - m = re.match('^calling *(?P<f>.*)+.* @ (?P<p>[0-9]*)', msg) + m = re.match(r'^calling *(?P<f>.*)+.* @ (?P<p>[0-9]*)', msg) if(m): func = m.group('f') pid = int(m.group('p')) devtemp[func] = (ktime, pid) continue - m = re.match('^initcall *(?P<f>.*)+.* returned (?P<r>.*) after (?P<t>.*) usecs', msg) + m = re.match(r'^initcall *(?P<f>.*)+.* returned (?P<r>.*) after (?P<t>.*) usecs', msg) if(m): data.valid = True data.end = ktime
Python 3.6 introduced a DeprecationWarning for invalid escape sequences. This is upgraded to a SyntaxWarning in Python 3.12, and will eventually be a syntax error.
Fix these now to get ahead of it before it's an error.
Signed-off-by: Benjamin Gray bgray@linux.ibm.com --- .../selftests/bpf/test_bpftool_synctypes.py | 26 +++++++++---------- tools/testing/selftests/bpf/test_offload.py | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/tools/testing/selftests/bpf/test_bpftool_synctypes.py b/tools/testing/selftests/bpf/test_bpftool_synctypes.py index 0cfece7ff4f8..46862af44f86 100755 --- a/tools/testing/selftests/bpf/test_bpftool_synctypes.py +++ b/tools/testing/selftests/bpf/test_bpftool_synctypes.py @@ -66,7 +66,7 @@ class ArrayParser(BlockParser):
def __init__(self, reader, array_name): self.array_name = array_name - self.start_marker = re.compile(f'(static )?const bool {self.array_name}[.*] = {{\n') + self.start_marker = re.compile(rf'(static )?const bool {self.array_name}[.*] = {{\n') super().__init__(reader)
def search_block(self): @@ -80,7 +80,7 @@ class ArrayParser(BlockParser): Parse a block and return data as a dictionary. Items to extract must be on separate lines in the file. """ - pattern = re.compile('[(BPF_\w*)]\s*= (true|false),?$') + pattern = re.compile(r'[(BPF_\w*)]\s*= (true|false),?$') entries = set() while True: line = self.reader.readline() @@ -178,7 +178,7 @@ class FileExtractor(object): @enum_name: name of the enum to parse """ start_marker = re.compile(f'enum {enum_name} {{\n') - pattern = re.compile('^\s*(BPF_\w+),?(\s+/*.**/)?$') + pattern = re.compile(r'^\s*(BPF_\w+),?(\s+/*.**/)?$') end_marker = re.compile('^};') parser = BlockParser(self.reader) parser.search_block(start_marker) @@ -226,8 +226,8 @@ class FileExtractor(object):
@block_name: name of the blog to parse, 'TYPE' in the example """ - start_marker = re.compile(f'*{block_name}* := {{') - pattern = re.compile('**([\w/-]+)**') + start_marker = re.compile(rf'*{block_name}* := {{') + pattern = re.compile(r'**([\w/-]+)**') end_marker = re.compile('}\n') return self.__get_description_list(start_marker, pattern, end_marker)
@@ -245,8 +245,8 @@ class FileExtractor(object):
@block_name: name of the blog to parse, 'TYPE' in the example """ - start_marker = re.compile(f'"\s*{block_name} := {{') - pattern = re.compile('([\w/]+) [|}]') + start_marker = re.compile(rf'"\s*{block_name} := {{') + pattern = re.compile(r'([\w/]+) [|}]') end_marker = re.compile('}') return self.__get_description_list(start_marker, pattern, end_marker)
@@ -264,8 +264,8 @@ class FileExtractor(object):
@macro: macro starting the block, 'HELP_SPEC_OPTIONS' in the example """ - start_marker = re.compile(f'"\s*{macro}\s*" [|}}]') - pattern = re.compile('([\w-]+) ?(?:||}[ }]])') + start_marker = re.compile(rf'"\s*{macro}\s*" [|}}]') + pattern = re.compile(r'([\w-]+) ?(?:||}[ }]])') end_marker = re.compile('}\\n') return self.__get_description_list(start_marker, pattern, end_marker)
@@ -284,7 +284,7 @@ class FileExtractor(object): @block_name: name of the blog to parse, 'TYPE' in the example """ start_marker = re.compile(f'local {block_name}='') - pattern = re.compile('(?:.*=')?([\w/]+)') + pattern = re.compile(r"(?:.*=')?([\w/]+)") end_marker = re.compile(''$') return self.__get_description_list(start_marker, pattern, end_marker)
@@ -316,7 +316,7 @@ class MainHeaderFileExtractor(SourceFileExtractor): {'-p', '-d', '--pretty', '--debug', '--json', '-j'} """ start_marker = re.compile(f'"OPTIONS :=') - pattern = re.compile('([\w-]+) ?(?:||}[ }]"])') + pattern = re.compile(r'([\w-]+) ?(?:||}[ }]"])') end_marker = re.compile('#define')
parser = InlineListParser(self.reader) @@ -338,8 +338,8 @@ class ManSubstitutionsExtractor(SourceFileExtractor):
{'-p', '-d', '--pretty', '--debug', '--json', '-j'} """ - start_marker = re.compile('|COMMON_OPTIONS| replace:: {') - pattern = re.compile('**([\w/-]+)**') + start_marker = re.compile(r'|COMMON_OPTIONS| replace:: {') + pattern = re.compile(r'**([\w/-]+)**') end_marker = re.compile('}$')
parser = InlineListParser(self.reader) diff --git a/tools/testing/selftests/bpf/test_offload.py b/tools/testing/selftests/bpf/test_offload.py index 40cba8d368d9..fffbc375a7e7 100755 --- a/tools/testing/selftests/bpf/test_offload.py +++ b/tools/testing/selftests/bpf/test_offload.py @@ -429,7 +429,7 @@ class NetdevSim: def __init__(self, nsimdev, port_index, ifname): # In case udev renamed the netdev to according to new schema, # check if the name matches the port_index. - nsimnamere = re.compile("eni\d+np(\d+)") + nsimnamere = re.compile(r"eni\d+np(\d+)") match = nsimnamere.match(ifname) if match and int(match.groups()[0]) != port_index + 1: raise Exception("netdevice name mismatches the expected one")
linux-kselftest-mirror@lists.linaro.org
-
Adrian Hunter -
Arnaldo Carvalho de Melo -
Benjamin Gray -
Ian Abbott -
Jonathan Corbet -
Nick Desaulniers