mirror of
https://git.yoctoproject.org/poky
synced 2026-02-01 14:28:44 +01:00
7b65658ede
A ReDoS issue was discovered in pygments/lexers/smithy.py in pygments through 2.15.0 via SmithyLexer. The CVE issue is fixed by these 3 different commits in different version: 1. Improve the Smithy metadata matcher (These changes are already available as part of current python3-pygments_2.14.0 version):dd52102c38(2.14.0) 2. SQL+Jinja: use a simpler regex in analyse_text:97eb3d5ec7(2.15.0) 3. Improve Java properties lexer (#2404):fdf182a7af(2.15.1) References: https://nvd.nist.gov/vuln/detail/CVE-2022-40896 https://pyup.io/posts/pyup-discovers-redos-vulnerabilities-in-top-python-packages-part-2/ (From OE-Core rev: 5a02307af5e593be864423a9f3ab309703d61dbf) Signed-off-by: Narpat Mali <narpat.mali@windriver.com> Signed-off-by: Steve Sakoman <steve@sakoman.com>
302 lines
9.4 KiB
Diff
302 lines
9.4 KiB
Diff
From 45ff8eabe0363f829c397372aefc3b23aeb135b3 Mon Sep 17 00:00:00 2001
|
|
From: Narpat Mali <narpat.mali@windriver.com>
|
|
Date: Tue, 29 Aug 2023 10:45:34 +0000
|
|
Subject: [PATCH] Improve Java properties lexer (#2404)
|
|
|
|
Use special lexer rules for escapes; fixes catastrophic backtracking,
|
|
and highlights them too.
|
|
|
|
Fixes #2356
|
|
|
|
CVE: CVE-2022-40896
|
|
|
|
Upstream-Status: Backport [https://github.com/pygments/pygments/commit/fdf182a7af85b1deeeb637ca970d31935e7c9d52]
|
|
|
|
Signed-off-by: Narpat Mali <narpat.mali@windriver.com>
|
|
---
|
|
pygments/lexers/configs.py | 50 +++++---
|
|
tests/examplefiles/properties/java.properties | 11 ++
|
|
.../properties/java.properties.output | 110 +++++++++++++++---
|
|
.../test_escaped_space_in_value.txt | 4 +-
|
|
.../properties/test_just_key_with_space.txt | 4 +-
|
|
5 files changed, 143 insertions(+), 36 deletions(-)
|
|
|
|
diff --git a/pygments/lexers/configs.py b/pygments/lexers/configs.py
|
|
index e04c722..b28b56a 100644
|
|
--- a/pygments/lexers/configs.py
|
|
+++ b/pygments/lexers/configs.py
|
|
@@ -129,26 +129,42 @@ class PropertiesLexer(RegexLexer):
|
|
|
|
tokens = {
|
|
'root': [
|
|
- (r'\s+', Whitespace),
|
|
+ # comments
|
|
(r'[!#].*|/{2}.*', Comment.Single),
|
|
- # search for first separator
|
|
- (r'([^\\\n]|\\.)*?(?=[ \f\t=:])', Name.Attribute, "separator"),
|
|
- # empty key
|
|
- (r'.+?$', Name.Attribute),
|
|
+ # ending a comment or whitespace-only line
|
|
+ (r'\n', Whitespace),
|
|
+ # eat whitespace at the beginning of a line
|
|
+ (r'^[^\S\n]+', Whitespace),
|
|
+ # start lexing a key
|
|
+ default('key'),
|
|
],
|
|
- 'separator': [
|
|
- # search for line continuation escape
|
|
- (r'([ \f\t]*)([=:]*)([ \f\t]*)(.*(?<!\\)(?:\\{2})*)(\\)(?!\\)$',
|
|
- bygroups(Whitespace, Operator, Whitespace, String, Text), "value", "#pop"),
|
|
- (r'([ \f\t]*)([=:]*)([ \f\t]*)(.*)',
|
|
- bygroups(Whitespace, Operator, Whitespace, String), "#pop"),
|
|
+ 'key': [
|
|
+ # non-escaped key characters
|
|
+ (r'[^\\:=\s]+', Name.Attribute),
|
|
+ # escapes
|
|
+ include('escapes'),
|
|
+ # separator is the first non-escaped whitespace or colon or '=' on the line;
|
|
+ # if it's whitespace, = and : are gobbled after it
|
|
+ (r'([^\S\n]*)([:=])([^\S\n]*)',
|
|
+ bygroups(Whitespace, Operator, Whitespace),
|
|
+ ('#pop', 'value')),
|
|
+ (r'[^\S\n]+', Whitespace, ('#pop', 'value')),
|
|
+ # maybe we got no value after all
|
|
+ (r'\n', Whitespace, '#pop'),
|
|
],
|
|
- 'value': [ # line continuation
|
|
- (r'\s+', Whitespace),
|
|
- # search for line continuation escape
|
|
- (r'(\s*)(.*(?<!\\)(?:\\{2})*)(\\)(?!\\)([ \t]*)',
|
|
- bygroups(Whitespace, String, Text, Whitespace)),
|
|
- (r'.*$', String, "#pop"),
|
|
+ 'value': [
|
|
+ # non-escaped value characters
|
|
+ (r'[^\\\n]+', String),
|
|
+ # escapes
|
|
+ include('escapes'),
|
|
+ # end the value on an unescaped newline
|
|
+ (r'\n', Whitespace, '#pop'),
|
|
+ ],
|
|
+ 'escapes': [
|
|
+ # line continuations; these gobble whitespace at the beginning of the next line
|
|
+ (r'(\\\n)([^\S\n]*)', bygroups(String.Escape, Whitespace)),
|
|
+ # other escapes
|
|
+ (r'\\(.|\n)', String.Escape),
|
|
],
|
|
}
|
|
|
|
diff --git a/tests/examplefiles/properties/java.properties b/tests/examplefiles/properties/java.properties
|
|
index d5b594e..7fe915c 100644
|
|
--- a/tests/examplefiles/properties/java.properties
|
|
+++ b/tests/examplefiles/properties/java.properties
|
|
@@ -14,6 +14,8 @@ key = \
|
|
and value2\\
|
|
key\ 2 = value
|
|
key\\ 3 = value3
|
|
+key \
|
|
+ = value
|
|
|
|
! empty keys and edge cases
|
|
key1 =
|
|
@@ -22,3 +24,12 @@ key3 the value3
|
|
key4 the:value4
|
|
key5 the=value5
|
|
key6=the value6
|
|
+
|
|
+! escapes in keys
|
|
+key\ with\ spaces = value
|
|
+key\nwith\nnewlines = value\nwith\nnewlines
|
|
+
|
|
+ ! indented comment
|
|
+
|
|
+! line continuations do \
|
|
+not = work for comments
|
|
diff --git a/tests/examplefiles/properties/java.properties.output b/tests/examplefiles/properties/java.properties.output
|
|
index 0c1fdee..4822575 100644
|
|
--- a/tests/examplefiles/properties/java.properties.output
|
|
+++ b/tests/examplefiles/properties/java.properties.output
|
|
@@ -2,13 +2,17 @@
|
|
'\n' Text.Whitespace
|
|
|
|
'# mixing spaces' Comment.Single
|
|
-'\n\t' Text.Whitespace
|
|
+'\n' Text.Whitespace
|
|
+
|
|
+'\t' Text.Whitespace
|
|
'Truth' Name.Attribute
|
|
' ' Text.Whitespace
|
|
'=' Operator
|
|
' ' Text.Whitespace
|
|
'Beauty' Literal.String
|
|
-'\n ' Text.Whitespace
|
|
+'\n' Text.Whitespace
|
|
+
|
|
+' ' Text.Whitespace
|
|
'Truth' Name.Attribute
|
|
':' Operator
|
|
'Beauty' Literal.String
|
|
@@ -23,18 +27,24 @@
|
|
' ' Text.Whitespace
|
|
':' Operator
|
|
'Beauty' Literal.String
|
|
-'\n \n' Text.Whitespace
|
|
+'\n' Text.Whitespace
|
|
+
|
|
+'\n' Text.Whitespace
|
|
|
|
'! line continuations and escapes' Comment.Single
|
|
-'\n ' Text.Whitespace
|
|
+'\n' Text.Whitespace
|
|
+
|
|
+' ' Text.Whitespace
|
|
'fruits' Name.Attribute
|
|
' ' Text.Whitespace
|
|
'apple, banana, pear, ' Literal.String
|
|
-'\\' Text
|
|
-'\n ' Text.Whitespace
|
|
+'\\\n' Literal.String.Escape
|
|
+
|
|
+' ' Text.Whitespace
|
|
'cantaloupe, watermelon, ' Literal.String
|
|
-'\\' Text
|
|
-'\n ' Text.Whitespace
|
|
+'\\\n' Literal.String.Escape
|
|
+
|
|
+' ' Text.Whitespace
|
|
'kiwi, mango' Literal.String
|
|
'\n' Text.Whitespace
|
|
|
|
@@ -42,25 +52,42 @@
|
|
' ' Text.Whitespace
|
|
'=' Operator
|
|
' ' Text.Whitespace
|
|
-'\\' Text
|
|
-'\n ' Text.Whitespace
|
|
-'value1 \\\\' Literal.String
|
|
-'\\' Text
|
|
-'\n ' Text.Whitespace
|
|
-'and value2\\\\' Literal.String
|
|
+'\\\n' Literal.String.Escape
|
|
+
|
|
+' ' Text.Whitespace
|
|
+'value1 ' Literal.String
|
|
+'\\\\' Literal.String.Escape
|
|
+'\\\n' Literal.String.Escape
|
|
+
|
|
+' ' Text.Whitespace
|
|
+'and value2' Literal.String
|
|
+'\\\\' Literal.String.Escape
|
|
'\n' Text.Whitespace
|
|
|
|
-'key\\ 2' Name.Attribute
|
|
+'key' Name.Attribute
|
|
+'\\ ' Literal.String.Escape
|
|
+'2' Name.Attribute
|
|
' ' Text.Whitespace
|
|
'=' Operator
|
|
' ' Text.Whitespace
|
|
'value' Literal.String
|
|
'\n' Text.Whitespace
|
|
|
|
-'key\\\\' Name.Attribute
|
|
+'key' Name.Attribute
|
|
+'\\\\' Literal.String.Escape
|
|
' ' Text.Whitespace
|
|
'3 = value3' Literal.String
|
|
-'\n\n' Text.Whitespace
|
|
+'\n' Text.Whitespace
|
|
+
|
|
+'key' Name.Attribute
|
|
+' ' Text.Whitespace
|
|
+'\\\n' Literal.String.Escape
|
|
+
|
|
+' ' Text.Whitespace
|
|
+'= value' Literal.String
|
|
+'\n' Text.Whitespace
|
|
+
|
|
+'\n' Text.Whitespace
|
|
|
|
'! empty keys and edge cases' Comment.Single
|
|
'\n' Text.Whitespace
|
|
@@ -92,3 +119,52 @@
|
|
'=' Operator
|
|
'the value6' Literal.String
|
|
'\n' Text.Whitespace
|
|
+
|
|
+'\n' Text.Whitespace
|
|
+
|
|
+'! escapes in keys' Comment.Single
|
|
+'\n' Text.Whitespace
|
|
+
|
|
+'key' Name.Attribute
|
|
+'\\ ' Literal.String.Escape
|
|
+'with' Name.Attribute
|
|
+'\\ ' Literal.String.Escape
|
|
+'spaces' Name.Attribute
|
|
+' ' Text.Whitespace
|
|
+'=' Operator
|
|
+' ' Text.Whitespace
|
|
+'value' Literal.String
|
|
+'\n' Text.Whitespace
|
|
+
|
|
+'key' Name.Attribute
|
|
+'\\n' Literal.String.Escape
|
|
+'with' Name.Attribute
|
|
+'\\n' Literal.String.Escape
|
|
+'newlines' Name.Attribute
|
|
+' ' Text.Whitespace
|
|
+'=' Operator
|
|
+' ' Text.Whitespace
|
|
+'value' Literal.String
|
|
+'\\n' Literal.String.Escape
|
|
+'with' Literal.String
|
|
+'\\n' Literal.String.Escape
|
|
+'newlines' Literal.String
|
|
+'\n' Text.Whitespace
|
|
+
|
|
+'\n' Text.Whitespace
|
|
+
|
|
+' ' Text.Whitespace
|
|
+'! indented comment' Comment.Single
|
|
+'\n' Text.Whitespace
|
|
+
|
|
+'\n' Text.Whitespace
|
|
+
|
|
+'! line continuations do \\' Comment.Single
|
|
+'\n' Text.Whitespace
|
|
+
|
|
+'not' Name.Attribute
|
|
+' ' Text.Whitespace
|
|
+'=' Operator
|
|
+' ' Text.Whitespace
|
|
+'work for comments' Literal.String
|
|
+'\n' Text.Whitespace
|
|
diff --git a/tests/snippets/properties/test_escaped_space_in_value.txt b/tests/snippets/properties/test_escaped_space_in_value.txt
|
|
index f76507f..44772d8 100644
|
|
--- a/tests/snippets/properties/test_escaped_space_in_value.txt
|
|
+++ b/tests/snippets/properties/test_escaped_space_in_value.txt
|
|
@@ -6,5 +6,7 @@ key = doubleword\ value
|
|
' ' Text.Whitespace
|
|
'=' Operator
|
|
' ' Text.Whitespace
|
|
-'doubleword\\ value' Literal.String
|
|
+'doubleword' Literal.String
|
|
+'\\ ' Literal.String.Escape
|
|
+'value' Literal.String
|
|
'\n' Text.Whitespace
|
|
diff --git a/tests/snippets/properties/test_just_key_with_space.txt b/tests/snippets/properties/test_just_key_with_space.txt
|
|
index 660c37c..833fe40 100644
|
|
--- a/tests/snippets/properties/test_just_key_with_space.txt
|
|
+++ b/tests/snippets/properties/test_just_key_with_space.txt
|
|
@@ -2,5 +2,7 @@
|
|
just\ key
|
|
|
|
---tokens---
|
|
-'just\\ key' Name.Attribute
|
|
+'just' Name.Attribute
|
|
+'\\ ' Literal.String.Escape
|
|
+'key' Name.Attribute
|
|
'\n' Text.Whitespace
|
|
--
|
|
2.40.0
|