From 95c0c262cfef384039bccf8fa02548c1dc59e335 Mon Sep 17 00:00:00 2001 From: upstream Date: Wed, 2 Sep 2026 11:37:41 +0200 Subject: [PATCH] Upstream andialbrecht/sqlparse state of the touched files before 9e625ff289 --- sqlparse/engine/grouping.py | 525 ++++++++++++++++++++++++++++++++++++ tests/test_regressions.py | 518 +++++++++++++++++++++++++++++++++++ 2 files changed, 1043 insertions(+) create mode 100644 sqlparse/engine/grouping.py create mode 100644 tests/test_regressions.py diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py new file mode 100644 index 0000000..d8cfa9e --- /dev/null +++ b/sqlparse/engine/grouping.py @@ -0,0 +1,525 @@ +# +# Copyright (C) 2009-2020 the sqlparse authors and contributors +# +# +# This module is part of python-sqlparse and is released under +# the BSD License: https://opensource.org/licenses/BSD-3-Clause + +from sqlparse import sql +from sqlparse import tokens as T +from sqlparse.exceptions import SQLParseError +from sqlparse.utils import imt, recurse + +# Maximum recursion depth for grouping operations to prevent DoS attacks +# Set to None to disable limit (not recommended for untrusted input) +MAX_GROUPING_DEPTH = 100 + +# Maximum number of tokens to process in one grouping operation to prevent +# DoS attacks. +# Set to None to disable limit (not recommended for untrusted input) +MAX_GROUPING_TOKENS = 10000 + +T_NUMERICAL = (T.Number, T.Number.Integer, T.Number.Float) +T_STRING = (T.String, T.String.Single, T.String.Symbol) +T_NAME = (T.Name, T.Name.Placeholder) + + +def _group_matching(tlist, cls, depth=0): + """Groups Tokens that have beginning and end.""" + if MAX_GROUPING_DEPTH is not None and depth > MAX_GROUPING_DEPTH: + raise SQLParseError( + f"Maximum grouping depth exceeded ({MAX_GROUPING_DEPTH})." + ) + + # Limit the number of tokens to prevent DoS attacks + if MAX_GROUPING_TOKENS is not None \ + and len(tlist.tokens) > MAX_GROUPING_TOKENS: + raise SQLParseError( + f"Maximum number of tokens exceeded ({MAX_GROUPING_TOKENS})." + ) + + opens = [] + tidx_offset = 0 + token_list = list(tlist) + + for idx, token in enumerate(token_list): + tidx = idx - tidx_offset + + if token.is_whitespace: + # ~50% of tokens will be whitespace. Will checking early + # for them avoid 3 comparisons, but then add 1 more comparison + # for the other ~50% of tokens... + continue + + if token.is_group and not isinstance(token, cls): + # Check inside previously grouped (i.e. parenthesis) if group + # of different type is inside (i.e., case). though ideally should + # should check for all open/close tokens at once to avoid recursion + _group_matching(token, cls, depth + 1) + continue + + if token.match(*cls.M_OPEN): + opens.append(tidx) + + elif token.match(*cls.M_CLOSE): + try: + open_idx = opens.pop() + except IndexError: + # this indicates invalid sql and unbalanced tokens. + # instead of break, continue in case other "valid" groups exist + continue + close_idx = tidx + tlist.group_tokens(cls, open_idx, close_idx) + tidx_offset += close_idx - open_idx + + +def group_brackets(tlist): + _group_matching(tlist, sql.SquareBrackets) + + +def group_parenthesis(tlist): + _group_matching(tlist, sql.Parenthesis) + + +def group_case(tlist): + _group_matching(tlist, sql.Case) + + +def group_if(tlist): + _group_matching(tlist, sql.If) + + +def group_for(tlist): + _group_matching(tlist, sql.For) + + +def group_begin(tlist): + _group_matching(tlist, sql.Begin) + + +def group_typecasts(tlist): + def match(token): + return token.match(T.Punctuation, '::') + + def valid(token): + return token is not None + + def post(tlist, pidx, tidx, nidx): + return pidx, nidx + + valid_prev = valid_next = valid + _group(tlist, sql.Identifier, match, valid_prev, valid_next, post) + + +def group_tzcasts(tlist): + def match(token): + return token.ttype == T.Keyword.TZCast + + def valid_prev(token): + return token is not None + + def valid_next(token): + return token is not None and ( + token.is_whitespace + or token.match(T.Keyword, 'AS') + or token.match(*sql.TypedLiteral.M_CLOSE) + ) + + def post(tlist, pidx, tidx, nidx): + return pidx, nidx + + _group(tlist, sql.Identifier, match, valid_prev, valid_next, post) + + +def group_typed_literal(tlist): + # definitely not complete, see e.g.: + # https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/interval-literal-syntax + # https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/interval-literals + # https://www.postgresql.org/docs/9.1/datatype-datetime.html + # https://www.postgresql.org/docs/9.1/functions-datetime.html + def match(token): + return imt(token, m=sql.TypedLiteral.M_OPEN) + + def match_to_extend(token): + return isinstance(token, sql.TypedLiteral) + + def valid_prev(token): + return token is not None + + def valid_next(token): + return token is not None and token.match(*sql.TypedLiteral.M_CLOSE) + + def valid_final(token): + return token is not None and token.match(*sql.TypedLiteral.M_EXTEND) + + def post(tlist, pidx, tidx, nidx): + return tidx, nidx + + _group(tlist, sql.TypedLiteral, match, valid_prev, valid_next, + post, extend=False) + _group(tlist, sql.TypedLiteral, match_to_extend, valid_prev, valid_final, + post, extend=True) + + +def group_period(tlist): + def match(token): + for ttype, value in ((T.Punctuation, '.'), + (T.Operator, '->'), + (T.Operator, '->>')): + if token.match(ttype, value): + return True + return False + + def valid_prev(token): + sqlcls = sql.SquareBrackets, sql.Identifier + ttypes = T.Name, T.String.Symbol + return imt(token, i=sqlcls, t=ttypes) + + def valid_next(token): + # issue261, allow invalid next token + return True + + def post(tlist, pidx, tidx, nidx): + # next_ validation is being performed here. issue261 + sqlcls = sql.SquareBrackets, sql.Function + ttypes = T.Name, T.String.Symbol, T.Wildcard, T.String.Single + next_ = tlist[nidx] if nidx is not None else None + valid_next = imt(next_, i=sqlcls, t=ttypes) + + return (pidx, nidx) if valid_next else (pidx, tidx) + + _group(tlist, sql.Identifier, match, valid_prev, valid_next, post) + + +def group_as(tlist): + def match(token): + return token.is_keyword and token.normalized == 'AS' + + def valid_prev(token): + return token.normalized == 'NULL' or not token.is_keyword + + def valid_next(token): + ttypes = T.DML, T.DDL, T.CTE + return not imt(token, t=ttypes) and token is not None + + def post(tlist, pidx, tidx, nidx): + return pidx, nidx + + _group(tlist, sql.Identifier, match, valid_prev, valid_next, post) + + +def group_assignment(tlist): + def match(token): + return token.match(T.Assignment, ':=') + + def valid(token): + return token is not None and token.ttype not in (T.Keyword,) + + def post(tlist, pidx, tidx, nidx): + m_semicolon = T.Punctuation, ';' + snidx, _ = tlist.token_next_by(m=m_semicolon, idx=nidx) + nidx = snidx or nidx + return pidx, nidx + + valid_prev = valid_next = valid + _group(tlist, sql.Assignment, match, valid_prev, valid_next, post) + + +def group_comparison(tlist): + sqlcls = (sql.Parenthesis, sql.Function, sql.Identifier, + sql.Operation, sql.TypedLiteral) + ttypes = T_NUMERICAL + T_STRING + T_NAME + + def match(token): + return token.ttype == T.Operator.Comparison + + def valid(token): + return bool(imt(token, t=ttypes, i=sqlcls) or (token and token.is_keyword and token.normalized == 'NULL')) + + def post(tlist, pidx, tidx, nidx): + return pidx, nidx + + valid_prev = valid_next = valid + _group(tlist, sql.Comparison, match, + valid_prev, valid_next, post, extend=False) + + +@recurse(sql.Identifier) +def group_identifier(tlist): + ttypes = (T.String.Symbol, T.Name) + + tidx, token = tlist.token_next_by(t=ttypes) + while token: + tlist.group_tokens(sql.Identifier, tidx, tidx) + tidx, token = tlist.token_next_by(t=ttypes, idx=tidx) + + +@recurse(sql.Over) +def group_over(tlist): + tidx, token = tlist.token_next_by(m=sql.Over.M_OPEN) + while token: + nidx, next_ = tlist.token_next(tidx) + if imt(next_, i=sql.Parenthesis, t=T.Name): + tlist.group_tokens(sql.Over, tidx, nidx) + tidx, token = tlist.token_next_by(m=sql.Over.M_OPEN, idx=tidx) + + +def group_arrays(tlist): + sqlcls = sql.SquareBrackets, sql.Identifier, sql.Function + ttypes = T.Name, T.String.Symbol + + def match(token): + return isinstance(token, sql.SquareBrackets) + + def valid_prev(token): + return imt(token, i=sqlcls, t=ttypes) + + def valid_next(token): + return True + + def post(tlist, pidx, tidx, nidx): + return pidx, tidx + + _group(tlist, sql.Identifier, match, + valid_prev, valid_next, post, extend=True, recurse=False) + + +def group_operator(tlist): + ttypes = T_NUMERICAL + T_STRING + T_NAME + sqlcls = (sql.SquareBrackets, sql.Parenthesis, sql.Function, + sql.Identifier, sql.Operation, sql.TypedLiteral) + + def match(token): + return imt(token, t=(T.Operator, T.Wildcard)) + + def valid(token): + return imt(token, i=sqlcls, t=ttypes) \ + or (token and token.match( + T.Keyword, + ('CURRENT_DATE', 'CURRENT_TIME', 'CURRENT_TIMESTAMP'))) + + def post(tlist, pidx, tidx, nidx): + tlist[tidx].ttype = T.Operator + return pidx, nidx + + valid_prev = valid_next = valid + _group(tlist, sql.Operation, match, + valid_prev, valid_next, post, extend=False) + + +def group_identifier_list(tlist): + m_role = T.Keyword, ('null', 'role') + sqlcls = (sql.Function, sql.Case, sql.Identifier, sql.Comparison, + sql.IdentifierList, sql.Operation) + ttypes = (T_NUMERICAL + T_STRING + T_NAME + + (T.Keyword, T.Comment, T.Wildcard)) + + def match(token): + return token.match(T.Punctuation, ',') + + def valid(token): + return imt(token, i=sqlcls, m=m_role, t=ttypes) + + def post(tlist, pidx, tidx, nidx): + return pidx, nidx + + valid_prev = valid_next = valid + _group(tlist, sql.IdentifierList, match, + valid_prev, valid_next, post, extend=True) + + +@recurse(sql.Comment) +def group_comments(tlist): + tidx, token = tlist.token_next_by(t=T.Comment) + while token: + eidx, end = tlist.token_not_matching( + lambda tk: imt(tk, t=T.Comment) or tk.is_newline, idx=tidx) + if end is None: + # From tidx onward everything is comment/newline: there is no + # terminator to group against, and every later start would hit + # the same dead end. Stop instead of re-scanning the tail once + # per remaining comment token (which is O(n**2)). + break + eidx, end = tlist.token_prev(eidx, skip_ws=False) + tlist.group_tokens(sql.Comment, tidx, eidx) + + tidx, token = tlist.token_next_by(t=T.Comment, idx=tidx) + + +@recurse(sql.Where) +def group_where(tlist): + tidx, token = tlist.token_next_by(m=sql.Where.M_OPEN) + while token: + eidx, end = tlist.token_next_by(m=sql.Where.M_CLOSE, idx=tidx) + + if end is None: + end = tlist._groupable_tokens[-1] + else: + end = tlist.tokens[eidx - 1] + # TODO: convert this to eidx instead of end token. + # i think above values are len(tlist) and eidx-1 + eidx = tlist.token_index(end) + tlist.group_tokens(sql.Where, tidx, eidx) + tidx, token = tlist.token_next_by(m=sql.Where.M_OPEN, idx=tidx) + + +@recurse() +def group_aliased(tlist): + I_ALIAS = (sql.Parenthesis, sql.Function, sql.Case, sql.Identifier, + sql.Operation, sql.Comparison) + + tidx, token = tlist.token_next_by(i=I_ALIAS, t=T.Number) + while token: + nidx, next_ = tlist.token_next(tidx) + if isinstance(next_, sql.Identifier): + tlist.group_tokens(sql.Identifier, tidx, nidx, extend=True) + tidx, token = tlist.token_next_by(i=I_ALIAS, t=T.Number, idx=tidx) + + +@recurse(sql.Function) +def group_functions(tlist): + has_create = False + has_table = False + has_as = False + for tmp_token in tlist.tokens: + if tmp_token.value.upper() == 'CREATE': + has_create = True + if tmp_token.value.upper() == 'TABLE': + has_table = True + if tmp_token.value.upper() == 'AS': + has_as = True + if has_create and has_table and not has_as: + return + + tidx, token = tlist.token_next_by(t=T.Name) + while token: + nidx, next_ = tlist.token_next(tidx) + if isinstance(next_, sql.Parenthesis): + over_idx, over = tlist.token_next(nidx) + if over and isinstance(over, sql.Over): + eidx = over_idx + else: + eidx = nidx + tlist.group_tokens(sql.Function, tidx, eidx) + tidx, token = tlist.token_next_by(t=T.Name, idx=tidx) + + +@recurse(sql.Identifier) +def group_order(tlist): + """Group together Identifier and Asc/Desc token""" + tidx, token = tlist.token_next_by(t=T.Keyword.Order) + while token: + pidx, prev_ = tlist.token_prev(tidx) + if imt(prev_, i=sql.Identifier, t=T.Number): + tlist.group_tokens(sql.Identifier, pidx, tidx) + tidx = pidx + tidx, token = tlist.token_next_by(t=T.Keyword.Order, idx=tidx) + + +@recurse() +def align_comments(tlist): + tidx, token = tlist.token_next_by(i=sql.Comment) + while token: + pidx, prev_ = tlist.token_prev(tidx) + if isinstance(prev_, sql.TokenList): + tlist.group_tokens(sql.TokenList, pidx, tidx, extend=True) + tidx = pidx + tidx, token = tlist.token_next_by(i=sql.Comment, idx=tidx) + + +def group_values(tlist): + tidx, token = tlist.token_next_by(m=(T.Keyword, 'VALUES')) + start_idx = tidx + end_idx = -1 + while token: + if isinstance(token, sql.Parenthesis): + end_idx = tidx + tidx, token = tlist.token_next(tidx) + if end_idx != -1: + tlist.group_tokens(sql.Values, start_idx, end_idx, extend=True) + + +def group(stmt): + for func in [ + group_comments, + + # _group_matching + group_brackets, + group_parenthesis, + group_case, + group_if, + group_for, + group_begin, + + group_over, + group_functions, + group_where, + group_period, + group_arrays, + group_identifier, + group_order, + group_typecasts, + group_tzcasts, + group_typed_literal, + group_operator, + group_comparison, + group_as, + group_aliased, + group_assignment, + + align_comments, + group_identifier_list, + group_values, + ]: + func(stmt) + return stmt + + +def _group(tlist, cls, match, + valid_prev=lambda t: True, + valid_next=lambda t: True, + post=None, + extend=True, + recurse=True, + depth=0 + ): + """Groups together tokens that are joined by a middle token. i.e. x < y""" + if MAX_GROUPING_DEPTH is not None and depth > MAX_GROUPING_DEPTH: + raise SQLParseError( + f"Maximum grouping depth exceeded ({MAX_GROUPING_DEPTH})." + ) + + # Limit the number of tokens to prevent DoS attacks + if MAX_GROUPING_TOKENS is not None \ + and len(tlist.tokens) > MAX_GROUPING_TOKENS: + raise SQLParseError( + f"Maximum number of tokens exceeded ({MAX_GROUPING_TOKENS})." + ) + + tidx_offset = 0 + pidx, prev_ = None, None + token_list = list(tlist) + + for idx, token in enumerate(token_list): + tidx = idx - tidx_offset + if tidx < 0: # tidx shouldn't get negative + continue + + if token.is_whitespace: + continue + + if recurse and token.is_group and not isinstance(token, cls): + _group(token, cls, match, valid_prev, valid_next, + post, extend, True, depth + 1) + + if match(token): + nidx, next_ = tlist.token_next(tidx) + if prev_ and valid_prev(prev_) and valid_next(next_): + from_idx, to_idx = post(tlist, pidx, tidx, nidx) + grp = tlist.group_tokens(cls, from_idx, to_idx, extend=extend) + + tidx_offset += to_idx - from_idx + pidx, prev_ = from_idx, grp + continue + + pidx, prev_ = tidx, token diff --git a/tests/test_regressions.py b/tests/test_regressions.py new file mode 100644 index 0000000..aca7f7b --- /dev/null +++ b/tests/test_regressions.py @@ -0,0 +1,518 @@ +import copy +import sys + +import pytest + +import sqlparse +from sqlparse import sql, tokens as T +from sqlparse.exceptions import SQLParseError + + +def test_issue9(): + # make sure where doesn't consume parenthesis + p = sqlparse.parse('(where 1)')[0] + assert isinstance(p, sql.Statement) + assert len(p.tokens) == 1 + assert isinstance(p.tokens[0], sql.Parenthesis) + prt = p.tokens[0] + assert len(prt.tokens) == 3 + assert prt.tokens[0].ttype == T.Punctuation + assert prt.tokens[-1].ttype == T.Punctuation + + +def test_issue13(): + parsed = sqlparse.parse("select 'one';\n" + "select 'two\\'';\n" + "select 'three';") + assert len(parsed) == 3 + assert str(parsed[1]).strip() == "select 'two\\'';" + + +@pytest.mark.parametrize('s', ['--hello', '-- hello', '--hello\n', + '--', '--\n']) +def test_issue26(s): + # parse stand-alone comments + p = sqlparse.parse(s)[0] + assert len(p.tokens) == 1 + assert p.tokens[0].ttype is T.Comment.Single + + +@pytest.mark.parametrize('value', ['create', 'CREATE']) +def test_issue34(value): + t = sqlparse.parse("create")[0].token_first() + assert t.match(T.Keyword.DDL, value) is True + + +def test_issue35(): + # missing space before LIMIT. Updated for #321 + sql = sqlparse.format("select * from foo where bar = 1 limit 1", + reindent=True) + assert sql == "\n".join([ + "select *", + "from foo", + "where bar = 1", + "limit 1"]) + + +def test_issue38(): + sql = sqlparse.format("SELECT foo; -- comment", strip_comments=True) + assert sql == "SELECT foo;" + sql = sqlparse.format("/* foo */", strip_comments=True) + assert sql == "" + + +def test_issue39(): + p = sqlparse.parse('select user.id from user')[0] + assert len(p.tokens) == 7 + idt = p.tokens[2] + assert idt.__class__ == sql.Identifier + assert len(idt.tokens) == 3 + assert idt.tokens[0].match(T.Name, 'user') is True + assert idt.tokens[1].match(T.Punctuation, '.') is True + assert idt.tokens[2].match(T.Name, 'id') is True + + +def test_issue40(): + # make sure identifier lists in subselects are grouped + p = sqlparse.parse('SELECT id, name FROM ' + '(SELECT id, name FROM bar) as foo')[0] + assert len(p.tokens) == 7 + assert p.tokens[2].__class__ == sql.IdentifierList + assert p.tokens[-1].__class__ == sql.Identifier + assert p.tokens[-1].get_name() == 'foo' + sp = p.tokens[-1].tokens[0] + assert sp.tokens[3].__class__ == sql.IdentifierList + # make sure that formatting works as expected + s = sqlparse.format('SELECT id == name FROM ' + '(SELECT id, name FROM bar)', reindent=True) + assert s == '\n'.join([ + 'SELECT id == name', + 'FROM', + ' (SELECT id,', + ' name', + ' FROM bar)']) + + s = sqlparse.format('SELECT id == name FROM ' + '(SELECT id, name FROM bar) as foo', reindent=True) + assert s == '\n'.join([ + 'SELECT id == name', + 'FROM', + ' (SELECT id,', + ' name', + ' FROM bar) as foo']) + + +@pytest.mark.parametrize('s', ['select x.y::text as z from foo', + 'select x.y::text as "z" from foo', + 'select x."y"::text as z from foo', + 'select x."y"::text as "z" from foo', + 'select "x".y::text as z from foo', + 'select "x".y::text as "z" from foo', + 'select "x"."y"::text as z from foo', + 'select "x"."y"::text as "z" from foo']) +@pytest.mark.parametrize('func_name, result', [('get_name', 'z'), + ('get_real_name', 'y'), + ('get_parent_name', 'x'), + ('get_alias', 'z'), + ('get_typecast', 'text')]) +def test_issue78(s, func_name, result): + # the bug author provided this nice examples, let's use them! + p = sqlparse.parse(s)[0] + i = p.tokens[2] + assert isinstance(i, sql.Identifier) + + func = getattr(i, func_name) + assert func() == result + + +def test_issue83(): + sql = """ CREATE OR REPLACE FUNCTION func_a(text) + RETURNS boolean LANGUAGE plpgsql STRICT IMMUTABLE AS + $_$ + BEGIN + ... + END; + $_$; + + CREATE OR REPLACE FUNCTION func_b(text) + RETURNS boolean LANGUAGE plpgsql STRICT IMMUTABLE AS + $_$ + BEGIN + ... + END; + $_$; + + ALTER TABLE..... ;""" + t = sqlparse.split(sql) + assert len(t) == 3 + + +def test_comment_encoding_when_reindent(): + # There was an UnicodeEncodeError in the reindent filter that + # casted every comment followed by a keyword to str. + sql = 'select foo -- Comment containing Ümläuts\nfrom bar' + formatted = sqlparse.format(sql, reindent=True) + assert formatted == sql + + +def test_parse_sql_with_binary(): + # See https://github.com/andialbrecht/sqlparse/pull/88 + # digest = '‚|ËêŠplL4¡h‘øN{' + digest = '\x82|\xcb\x0e\xea\x8aplL4\xa1h\x91\xf8N{' + sql = f"select * from foo where bar = '{digest}'" + formatted = sqlparse.format(sql, reindent=True) + tformatted = f"select *\nfrom foo\nwhere bar = '{digest}'" + assert formatted == tformatted + + +def test_dont_alias_keywords(): + # The _group_left_right function had a bug where the check for the + # left side wasn't handled correctly. In one case this resulted in + # a keyword turning into an identifier. + p = sqlparse.parse('FROM AS foo')[0] + assert len(p.tokens) == 5 + assert p.tokens[0].ttype is T.Keyword + assert p.tokens[2].ttype is T.Keyword + + +def test_format_accepts_encoding(load_file): + # issue20 + sql = load_file('test_cp1251.sql', 'cp1251') + formatted = sqlparse.format(sql, reindent=True, encoding='cp1251') + tformatted = 'insert into foo\nvalues (1); -- Песня про надежду' + + assert formatted == tformatted + + +def test_stream(get_stream): + with get_stream("stream.sql") as stream: + p = sqlparse.parse(stream)[0] + assert p.get_type() == 'INSERT' + + +def test_issue90(): + sql = ('UPDATE "gallery_photo" SET "owner_id" = 4018, "deleted_at" = NULL,' + ' "width" = NULL, "height" = NULL, "rating_votes" = 0,' + ' "rating_score" = 0, "thumbnail_width" = NULL,' + ' "thumbnail_height" = NULL, "price" = 1, "description" = NULL') + formatted = sqlparse.format(sql, reindent=True) + tformatted = '\n'.join([ + 'UPDATE "gallery_photo"', + 'SET "owner_id" = 4018,', + ' "deleted_at" = NULL,', + ' "width" = NULL,', + ' "height" = NULL,', + ' "rating_votes" = 0,', + ' "rating_score" = 0,', + ' "thumbnail_width" = NULL,', + ' "thumbnail_height" = NULL,', + ' "price" = 1,', + ' "description" = NULL']) + assert formatted == tformatted + + +def test_except_formatting(): + sql = 'SELECT 1 FROM foo WHERE 2 = 3 EXCEPT SELECT 2 FROM bar WHERE 1 = 2' + formatted = sqlparse.format(sql, reindent=True) + tformatted = '\n'.join([ + 'SELECT 1', + 'FROM foo', + 'WHERE 2 = 3', + 'EXCEPT', + 'SELECT 2', + 'FROM bar', + 'WHERE 1 = 2']) + assert formatted == tformatted + + +def test_null_with_as(): + sql = 'SELECT NULL AS c1, NULL AS c2 FROM t1' + formatted = sqlparse.format(sql, reindent=True) + tformatted = '\n'.join([ + 'SELECT NULL AS c1,', + ' NULL AS c2', + 'FROM t1']) + assert formatted == tformatted + + +def test_issue190_open_file(filepath): + path = filepath('stream.sql') + with open(path) as stream: + p = sqlparse.parse(stream)[0] + assert p.get_type() == 'INSERT' + + +def test_issue193_splitting_function(): + sql = """ CREATE FUNCTION a(x VARCHAR(20)) RETURNS VARCHAR(20) + BEGIN + DECLARE y VARCHAR(20); + RETURN x; + END; + SELECT * FROM a.b;""" + statements = sqlparse.split(sql) + assert len(statements) == 2 + + +def test_issue194_splitting_function(): + sql = """ CREATE FUNCTION a(x VARCHAR(20)) RETURNS VARCHAR(20) + BEGIN + DECLARE y VARCHAR(20); + IF (1 = 1) THEN + SET x = y; + END IF; + RETURN x; + END; + SELECT * FROM a.b;""" + statements = sqlparse.split(sql) + assert len(statements) == 2 + + +def test_issue186_get_type(): + sql = "-- comment\ninsert into foo" + p = sqlparse.parse(sql)[0] + assert p.get_type() == 'INSERT' + + +def test_issue212_py2unicode(): + t1 = sql.Token(T.String, 'schöner ') + t2 = sql.Token(T.String, 'bug') + token_list = sql.TokenList([t1, t2]) + assert str(token_list) == 'schöner bug' + + +def test_issue213_leadingws(): + sql = " select * from foo" + assert sqlparse.format(sql, strip_whitespace=True) == "select * from foo" + + +def test_issue227_gettype_cte(): + select_stmt = sqlparse.parse('SELECT 1, 2, 3 FROM foo;') + assert select_stmt[0].get_type() == 'SELECT' + with_stmt = sqlparse.parse('WITH foo AS (SELECT 1, 2, 3)' + 'SELECT * FROM foo;') + assert with_stmt[0].get_type() == 'SELECT' + with2_stmt = sqlparse.parse(""" + WITH foo AS (SELECT 1 AS abc, 2 AS def), + bar AS (SELECT * FROM something WHERE x > 1) + INSERT INTO elsewhere SELECT * FROM foo JOIN bar;""") + assert with2_stmt[0].get_type() == 'INSERT' + + +def test_issue207_runaway_format(): + sql = 'select 1 from (select 1 as one, 2 as two, 3 from dual) t0' + p = sqlparse.format(sql, reindent=True) + assert p == '\n'.join([ + "select 1", + "from", + " (select 1 as one,", + " 2 as two,", + " 3", + " from dual) t0"]) + + +def test_token_next_doesnt_ignore_skip_cm(): + sql = '--comment\nselect 1' + tok = sqlparse.parse(sql)[0].token_next(-1, skip_cm=True)[1] + assert tok.value == 'select' + + +@pytest.mark.parametrize('s', [ + 'SELECT x AS', + 'AS' +]) +def test_issue284_as_grouping(s): + p = sqlparse.parse(s)[0] + assert s == str(p) + + +def test_issue315_utf8_by_default(): + # Make sure the lexer can handle utf-8 string by default correctly + # digest = '齐天大圣.カラフルな雲.사랑해요' + # The digest contains Chinese, Japanese and Korean characters + # All in 'utf-8' encoding. + digest = ( + '\xe9\xbd\x90\xe5\xa4\xa9\xe5\xa4\xa7\xe5\x9c\xa3.' + '\xe3\x82\xab\xe3\x83\xa9\xe3\x83\x95\xe3\x83\xab\xe3\x81\xaa\xe9' + '\x9b\xb2.' + '\xec\x82\xac\xeb\x9e\x91\xed\x95\xb4\xec\x9a\x94' + ) + sql = f"select * from foo where bar = '{digest}'" + formatted = sqlparse.format(sql, reindent=True) + tformatted = f"select *\nfrom foo\nwhere bar = '{digest}'" + assert formatted == tformatted + + +def test_issue322_concurrently_is_keyword(): + s = 'CREATE INDEX CONCURRENTLY myindex ON mytable(col1);' + p = sqlparse.parse(s)[0] + + assert len(p.tokens) == 12 + assert p.tokens[0].ttype is T.Keyword.DDL # CREATE + assert p.tokens[2].ttype is T.Keyword # INDEX + assert p.tokens[4].ttype is T.Keyword # CONCURRENTLY + assert p.tokens[4].value == 'CONCURRENTLY' + assert isinstance(p.tokens[6], sql.Identifier) + assert p.tokens[6].value == 'myindex' + + +@pytest.mark.parametrize('s', [ + 'SELECT @min_price:=MIN(price), @max_price:=MAX(price) FROM shop;', + 'SELECT @min_price:=MIN(price), @max_price:=MAX(price) FROM shop', + +]) +def test_issue359_index_error_assignments(s): + sqlparse.parse(s) + sqlparse.format(s, strip_comments=True) + + +def test_issue469_copy_as_psql_command(): + formatted = sqlparse.format( + '\\copy select * from foo', + keyword_case='upper', identifier_case='capitalize') + assert formatted == '\\copy SELECT * FROM Foo' + + +@pytest.mark.xfail(reason='Needs to be fixed') +def test_issue484_comments_and_newlines(): + formatted = sqlparse.format('\n'.join([ + 'Create table myTable', + '(', + ' myId TINYINT NOT NULL, --my special comment', + ' myName VARCHAR2(100) NOT NULL', + ')']), + strip_comments=True) + assert formatted == ('\n'.join([ + 'Create table myTable', + '(', + ' myId TINYINT NOT NULL,', + ' myName VARCHAR2(100) NOT NULL', + ')'])) + + +def test_issue485_split_multi(): + p_sql = '''CREATE OR REPLACE RULE ruled_tab_2rules AS ON INSERT +TO public.ruled_tab +DO instead ( +select 1; +select 2; +);''' + assert len(sqlparse.split(p_sql)) == 1 + + +def test_issue489_tzcasts(): + p = sqlparse.parse('select bar at time zone \'UTC\' as foo')[0] + assert p.tokens[-1].has_alias() is True + assert p.tokens[-1].get_alias() == 'foo' + + +def test_issue562_tzcasts(): + # Test that whitespace between 'from' and 'bar' is retained + formatted = sqlparse.format( + 'SELECT f(HOUR from bar AT TIME ZONE \'UTC\') from foo', reindent=True + ) + assert formatted == \ + 'SELECT f(HOUR\n from bar AT TIME ZONE \'UTC\')\nfrom foo' + + +def test_as_in_parentheses_indents(): + # did raise NoneType has no attribute is_group in _process_parentheses + formatted = sqlparse.format('(as foo)', reindent=True) + assert formatted == '(as foo)' + + +def test_format_invalid_where_clause(): + # did raise ValueError + formatted = sqlparse.format('where, foo', reindent=True) + assert formatted == 'where, foo' + + +def test_splitting_at_and_backticks_issue588(): + splitted = sqlparse.split( + 'grant foo to user1@`myhost`; grant bar to user1@`myhost`;') + assert len(splitted) == 2 + assert splitted[-1] == 'grant bar to user1@`myhost`;' + + +def test_comment_between_cte_clauses_issue632(): + p, = sqlparse.parse(""" + WITH foo AS (), + -- A comment before baz subquery + baz AS () + SELECT * FROM baz;""") + assert p.get_type() == "SELECT" + + +def test_copy_issue672(): + p = sqlparse.parse('select * from foo')[0] + copied = copy.deepcopy(p) + assert str(p) == str(copied) + + +def test_primary_key_issue740(): + p = sqlparse.parse('PRIMARY KEY')[0] + assert len(p.tokens) == 1 + assert p.tokens[0].ttype == T.Keyword + + +def test_alter_table_row_format_issue773(): + # ROW_FORMAT is a keyword (like ENGINE), so it must not be absorbed into + # the preceding table name as if it were an alias. + p = sqlparse.parse('ALTER TABLE mytable ROW_FORMAT=Dynamic')[0] + row_format = p.token_next_by(m=(T.Keyword, 'ROW_FORMAT'))[1] + assert row_format is not None + # The table name should be parsed as a standalone identifier. + assert isinstance(p.tokens[4], sql.Identifier) + assert p.tokens[4].get_real_name() == 'mytable' + assert p.tokens[4].get_alias() is None + # Sanity check: ENGINE (already a keyword) behaves the same way. + e = sqlparse.parse('ALTER TABLE mytable ENGINE=InnoDB')[0] + assert e.tokens[4].get_real_name() == 'mytable' + + +def test_materialized_view_issue752(): + p = sqlparse.parse('CREATE MATERIALIZED VIEW v AS SELECT 1')[0] + assert p.tokens[2].ttype == T.Keyword + formatted = sqlparse.format('create materialized view v as select 1', + keyword_case='upper') + assert formatted == 'CREATE MATERIALIZED VIEW v AS SELECT 1' + + +@pytest.mark.parametrize('sql', [ + 'a BETWEEN .03 AND .06', + 'a between .03 and .06', +]) +def test_between_leading_dot_float_issue601(sql): + # A keyword followed by a float literal written without a leading zero + # (e.g. ``.03``) must not be reclassified as an identifier because of the + # following period. BETWEEN has to stay a keyword and the literals have to + # remain standalone number tokens. + p = sqlparse.parse(sql)[0] + kw = p.token_next_by(m=(T.Keyword, 'BETWEEN'))[1] + assert kw is not None + floats = list(p.flatten()) + floats = [t for t in floats if t.ttype is T.Number.Float] + assert [t.value for t in floats] == ['.03', '.06'] + # 'and' must remain a keyword too, not be absorbed into an identifier. + assert p.token_next_by(m=(T.Keyword, 'AND'))[1] is not None + + +def test_keyword_before_qualified_name_still_grouped(): + # Regression guard for issue #601 fix: member access such as + # ``schema.name`` (with or without surrounding spaces) must keep working. + for stmt in ('select foo.bar', 'select foo . bar', 'select a.b.c'): + p = sqlparse.parse(stmt)[0] + ident = p.token_next_by(i=sql.Identifier)[1] + assert ident is not None + + +@pytest.fixture +def limit_recursion(): + curr_limit = sys.getrecursionlimit() + sys.setrecursionlimit(100) + yield + sys.setrecursionlimit(curr_limit) + + +def test_max_recursion(limit_recursion): + with pytest.raises(SQLParseError): + sqlparse.parse('[' * 1000 + ']' * 1000)