Commit 5e207b8c authored by Grant Paton-Simpson's avatar Grant Paton-Simpson

Lint advisor fixes; extra testing; snippet line nos

* Test specific snippets against linter
* Enable handling of groups of linter message types
* Custom messaging now always inserted before consolidated standard linter messages
* Add line numbers to overall snippet
* Bump version
parent c695388b
Pipeline #575 failed with stages
# https://git.nzoss.org.nz/pyGrant/superhelp
version number: 0.9.13
version number: 0.9.14
author: Grant Paton-Simpson
## Overview
......
......@@ -2,7 +2,7 @@ from setuptools import setup, find_packages # @UnresolvedImport
from codecs import open
from os import path
__version__ = '0.9.13'
__version__ = '0.9.14'
here = path.abspath(path.dirname(__file__))
......
......@@ -20,8 +20,14 @@ MISC_ISSUES_TITLE = layout("""\
""")
def _store_snippet(snippet):
"""
At least one test (E501 line too long) only triggered if a trailing newline.
Note - if more than one newline we trigger W391 (blank line at end of file)
so an rstrip('\n') needed.
Having done this need to deactivate W292 (blank line at end of file) LOL
"""
tmp_fh, fpath = make_open_tmp_file(conf.SNIPPET_FNAME, mode='w')
tmp_fh.write(snippet)
tmp_fh.write(snippet.rstrip('\n') + '\n')
tmp_fh.close()
return fpath
......@@ -60,7 +66,7 @@ def _get_flake8_results(fpath):
args = [flake8_fpath, str(fpath)]
if lint_conf.IGNORED_LINT_RULES:
ignored = ','.join(lint_conf.IGNORED_LINT_RULES)
args.append(f"--extend-ignore={ignored}")
args.append(f"--ignore={ignored}")
res = run(args=args, stdout=PIPE)
return res
......@@ -101,30 +107,14 @@ def _get_msg_type_and_dets(lint_regex_dicts):
Basically a list of pulled apart lint messages.
:return: dict of message types as keys (possibly consolidated e.g.
E123-9 -> line continuation message type) and MsgDets tuples as values.
Note - messages being replaced are consolidated into a placeholder.
:rtype: dict
"""
msg_type_and_dets = defaultdict(list)
for lint_regex_dict in lint_regex_dicts:
line_no = int(lint_regex_dict[conf.LINT_LINE_NO])
raw_msg_type = lint_regex_dict[conf.LINT_MSG_TYPE]
msg_type = lint_conf.CONSOLIDATE_MSG_TYPE.get(
raw_msg_type, raw_msg_type)
msg_type = lint_conf.consolidated_msg_type(raw_msg_type)
msg = layout(lint_regex_dict[conf.LINT_MSG])
try:
msg_dets = lint_conf.CUSTOM_LINT_MSGS[msg_type]
except KeyError:
pass
else:
if msg_dets.replacement:
## Can't consolidate on the actual replacement message because
## we need different versions for different message levels.
## So we store a placeholder which enables us to consolidate AND
## replace for brief and then main. Will be using .format() so
## want to do something like:
## "{E123_msg} (lines 1 and 2)".format(E123_msg=e123_brief_msg)
msg_placeholder = _msg_type_to_placeholder(msg_type)
msg = msg_placeholder
line_no = int(lint_regex_dict[conf.LINT_LINE_NO])
msg_dets = MsgDets(msg, line_no)
msg_type_and_dets[msg_type].append(msg_dets)
return msg_type_and_dets
......@@ -162,17 +152,17 @@ def _get_unfinished_messages(msg_type_and_dets):
generic_msg_type = msg_type not in lint_conf.CUSTOM_LINT_MSGS
if generic_msg_type:
unfinished_msg = '* ' + unfinished_msg.lstrip('\n') ## bullet points for generic messages (custom messages have full layout treatment e.g. code highlighting)
unfinished_msgs.append(unfinished_msg)
## add supplementary line?
supplement_configured = msg_type in lint_conf.CUSTOM_LINT_MSGS
if msg_type not in already_supplemented and supplement_configured:
msg_part_dets = lint_conf.CUSTOM_LINT_MSGS[msg_type]
supplement_needed = not msg_part_dets.replacement
if supplement_needed:
msg_placeholder = _msg_type_to_placeholder(msg_type)
last_msg = unfinished_msgs[-1]
last_msg = last_msg + '\n\n' + msg_placeholder ## will be replaced by appropriate level message in brief and main
already_supplemented.add(msg_type)
msg_placeholder = _msg_type_to_placeholder(msg_type)
unfinished_msg = (
msg_placeholder
+ '\n\nDetails: '
+ unfinished_msg.lstrip('\n')
) ## will be replaced by appropriate level message in brief and main
already_supplemented.add(msg_type)
unfinished_msgs.append(unfinished_msg)
return unfinished_msgs
def _get_extra_msg(msg_type_and_dets):
......
......@@ -26,59 +26,13 @@ else:
## When testing user-supplied snippets watch out for the BOM MS inserts via Notepad. AST chokes on it.
## All snippets here should be raw strings (see https://stackoverflow.com/questions/53636723/python-parsing-code-with-new-line-character-in-them-using-ast)
TEST_SNIPPET = r"""
def function_with_really_long_name(parameter_1, parameter_2,
parameter_3):
pass
def function_with_really_long_name2(
parameter_1, parameter_2, parameter_3):
pass
def function_with_really_long_name3(
parameter_1,
parameter_2,
parameter_3):
pass
def function_with_really_long_name4(
parameter_1,
parameter_2,
parameter_3):
pass
def function_with_really_long_name5(
parameter_1,
parameter_2,
parameter_3):
pass
my_list = [
'a',
]
my_list2 = [
'a',
]
my_list3 = [
'a',
]
my_list4 = [
'a',
]
a = 'vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvaaaavvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv'
b = 'vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvbbvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv'
c = 'vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvcvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv'
func1("abc"
"def")
func2("abc"
"def")
func3("abc"
"def")
def get_overall_snippet_messages_dets(snippet, blocks_dets):
'''
Returns messages which apply to snippet as a whole, not just specific
blocks. E.g. looking at every block to look for opportunities to unpack. Or
reporting on linting results.
'''
messages_dets = []
"""
PY3_6 = '3.6'
......
from . import cli_displayer, html_displayer
\ No newline at end of file
from . import cli_displayer, html_displayer
......@@ -2,7 +2,7 @@ import logging
from textwrap import dedent
from .cli_extras import md2cli
from ..utils import layout_comment as layout
from ..utils import get_line_numbered_snippet, layout_comment as layout
"""
Note - displays properly in the terminal but not necessarily in other output
......@@ -64,10 +64,11 @@ def display(snippet, messages_dets, *,
display_snippet = _need_snippet_displayed(
overall_messages_dets, block_messages_dets, multi_block=multi_block)
if display_snippet:
line_numbered_snippet = get_line_numbered_snippet(snippet)
text.append(md2cli.main(dedent(
"## Overall Snippet"
f"\n{MDV_CODE_BOUNDARY}\n"
+ snippet
+ line_numbered_snippet
+ f"\n{MDV_CODE_BOUNDARY}")))
for message_dets in overall_messages_dets:
message = get_message(message_dets, message_level)
......
......@@ -64,6 +64,7 @@ def code(text, from_fenced_block=None, **kw):
"""
if not from_fenced_block:
text = ('\n' + text).replace('\n ', '\n')[1:]
n_text_lines = len(text.split('\n'))
# funny: ":-" confuses the tokenizer. replace/backreplace:
raw_code = text.replace(':-', '\x01--')
raw_code = raw_code.replace('## >>>', '## >>>')
......@@ -75,7 +76,15 @@ def code(text, from_fenced_block=None, **kw):
firstl = text.split('\n')[0]
n_spaces2delete = len(firstl) - len(firstl.lstrip())
spaces2delete = ' ' * n_spaces2delete
text = '\n' + (f"\n{text}").replace(f"\n{spaces2delete}", '\n')[1:]
width = len(str(n_text_lines)) ## e.g. 3 so we can cope with line number 100 onwards
width_of_line_num_1 = 1
missing_indents = width - width_of_line_num_1 ## so if 3 wide to handle 100+ we need 2
indent_for_first_line_num = missing_indents * ' '
text = (
'\n'
+ indent_for_first_line_num
+ (f"\n{text}").replace(f"\n{spaces2delete}", '\n')[1:]
)
# we want an indent of one and low vis prefix. this does it:
code_lines = text.splitlines()
code_prefix = cli_colour.colourise_low_vis(cli_conf.CODE_PREFIX)
......@@ -83,6 +92,6 @@ def code(text, from_fenced_block=None, **kw):
prefix = f"\n{indent}{code_prefix} {empty}"
if code_lines[-1] == '\x1b[0m':
code_lines.pop()
code = prefix.join(code_lines)
code = code.replace('\x01--', ':-')
return code + '\n' + cli_conf.DEFAULT_ANSI_COLOUR_BYTE_STR
code_str = prefix.join(code_lines)
code_str = code_str.replace('\x01--', ':-')
return code_str + '\n' + cli_conf.DEFAULT_ANSI_COLOUR_BYTE_STR
from pathlib import Path #@UnresovedImport
from textwrap import dedent, indent
import webbrowser
from .. import conf
from ..utils import make_open_tmp_file
from ..utils import get_line_numbered_snippet, make_open_tmp_file
from markdown import markdown ## https://coderbook.com/@marcus/how-to-render-markdown-syntax-as-html-using-python/ @UnresolvedImport
......@@ -510,7 +509,7 @@ def get_separate_code_message_parts(message):
open_code_block = False
return message_parts
def get_html_strs(message, message_type, *, warning=False):
def get_html_strs(message, message_type, *, warning=False): # @UnusedVariable
if not message:
return []
message_type_class = MESSAGE_LEVEL2CLASS[message_type]
......@@ -563,8 +562,9 @@ def get_message_html_strs(message_dets):
def repeat_overall_snippet(snippet):
html_strs = []
html_strs.append("<h2>Overall Snippet</h2>")
line_numbered_snippet = get_line_numbered_snippet(snippet)
overall_code_str = indent(
f"{conf.MD_PYTHON_CODE_START}\n{snippet}",
f"{conf.MD_PYTHON_CODE_START}\n{line_numbered_snippet}",
' '*4)
overall_code_str_highlighted = markdown(
overall_code_str, extensions=['codehilite'])
......
......@@ -18,53 +18,97 @@ LINT_PATTERN = fr"""^.+?: ## starts with misc u
IGNORED_LINT_RULES = [
'E266', 'E262', ## I like ## before comments and # before commented out code (idea copied off Tom Eastman - thanks Tom!)
'E305', ## for classes I agree with 2 spaces but not functions
'W292', ## no newline at end of file inappropriate for snippets as opposed to modules
]
def title2msg_type(title):
return title.upper().replace(' ', '_') + '_MSG_TYPE'
## Ensure brief AND main are the same so titles don't shift when
## changing message level
line_continuation_title = "Line continuation"
line_length_title = "Line length"
unused_imports = "Unused imports"
line_indentation_title = "Line indentation issues"
line_length_title = "Excessive line length"
unused_imports_title = "Unused imports"
whitespace_title = "White space issues"
blank_line_title = "Blank line issues"
## nice to keep placeholder names etc aligned with actual titles but nothing breaks if we don't
LINE_CONTINUATION_MSG_TYPE = title2msg_type(line_continuation_title)
LINE_INDENTATION_MSG_TYPE = title2msg_type(line_indentation_title)
LINE_LENGTH_MSG_TYPE = title2msg_type(line_length_title)
UNUSED_IMPORT_MSG_TYPE = title2msg_type(unused_imports)
CONSOLIDATE_MSG_TYPE = {
'E123': LINE_CONTINUATION_MSG_TYPE,
'E124': LINE_CONTINUATION_MSG_TYPE,
'E125': LINE_CONTINUATION_MSG_TYPE,
'E126': LINE_CONTINUATION_MSG_TYPE,
'E127': LINE_CONTINUATION_MSG_TYPE,
'E128': LINE_CONTINUATION_MSG_TYPE,
'E129': LINE_CONTINUATION_MSG_TYPE,
'E131': LINE_CONTINUATION_MSG_TYPE,
'E501': LINE_LENGTH_MSG_TYPE,
'F401': UNUSED_IMPORT_MSG_TYPE,
}
LintMsgs = namedtuple('LintMsgs', 'brief, main, extra, replacement')
UNUSED_IMPORT_MSG_TYPE = title2msg_type(unused_imports_title)
WHITESPACE_MSG_TYPE = title2msg_type(whitespace_title)
BLANK_LINES_MSG_TYPE = title2msg_type(blank_line_title)
def consolidated_msg_type(msg_type):
if msg_type == 'E501':
msg_type = LINE_LENGTH_MSG_TYPE
elif msg_type == 'F401':
msg_type = UNUSED_IMPORT_MSG_TYPE
elif msg_type.startswith('E1'):
msg_type = LINE_INDENTATION_MSG_TYPE
elif msg_type.startswith('E2'):
msg_type = WHITESPACE_MSG_TYPE
elif msg_type.startswith('E3'):
msg_type = BLANK_LINES_MSG_TYPE
return msg_type
LevelMsgs = namedtuple('LintMsgsByLevel', 'brief, main, extra')
CUSTOM_LINT_MSGS = {
LINE_CONTINUATION_MSG_TYPE: LintMsgs(
BLANK_LINES_MSG_TYPE: LevelMsgs(
layout(f"""
#### {line_continuation_title}
#### {blank_line_title}
The linter has raised questions about blank lines.
"""),
layout(f"""
#### {blank_line_title}
The linter has raised questions about blank lines. Class definitions
should have two blank lines before. On function definitions there is
more flexibility. It should either be one or two.
"""),
'',
),
WHITESPACE_MSG_TYPE: LevelMsgs(
layout(f"""
#### {whitespace_title}
The linter has raised questions about "whitespace" (tabs, spaces).
The linter has raised questions about line continuation. There are
at least two styles of line continuation. Whichever you follow be
consistent.
"""),
layout(f"""
#### {whitespace_title}
The linter has raised questions about "whitespace" (tabs, spaces).
Even when whitespace doesn't seem to matter it is best to follow
Python whitespace conventions when writing Python. Conventions may
differ in other languages.
"""),
'',
),
LINE_INDENTATION_MSG_TYPE: LevelMsgs(
layout(f"""
#### {line_indentation_title}
The linter has raised questions about indentation. There are at
least two styles of indentation. Whichever you follow be consistent.
"""),
(
layout(f"""\
#### {line_continuation_title}
#### {line_indentation_title}
The linter has raised questions about line continuation. There
are at least two styles of line continuation:
The linter has raised questions about indentation. There are at
least two styles of indentation:
1) Visual line continuation e.g. note how parameter_3 lines up
with the opening parenthesis
1) Visual line continuation e.g. note how `parameter_3` lines up
with the opening parenthesis:
""")
+
......@@ -165,8 +209,8 @@ CUSTOM_LINT_MSGS = {
for some thought-provoking ideas on code style and much more.
"""),
True),
LINE_LENGTH_MSG_TYPE: LintMsgs(
),
LINE_LENGTH_MSG_TYPE: LevelMsgs(
layout(f"""\
#### {line_length_title}
......@@ -191,26 +235,24 @@ CUSTOM_LINT_MSGS = {
section "A Foolish Consistency is the Hobgoblin of Little Minds".
"""),
'',
True),
UNUSED_IMPORT_MSG_TYPE: LintMsgs(
'',),
UNUSED_IMPORT_MSG_TYPE: LevelMsgs(
layout(f"""\
#### {unused_imports}
#### {unused_imports_title}
One or more imports not used in snippet.
"""),
layout(f"""\
#### {unused_imports}
#### {unused_imports_title}
One or more imports not used in snippet. If the snippet was
extracted from a larger piece of code and the imports are used in
that code then there is no problem.
"""),
'',
False
''
)
}
......@@ -17,6 +17,16 @@ starting_num_space_pattern = r"""(?x)
"""
starting_num_space_prog = re.compile(starting_num_space_pattern)
def get_line_numbered_snippet(snippet):
snippet_lines = snippet.split('\n')
n_lines = len(snippet_lines)
width = len(str(n_lines))
new_snippet_lines = []
for n, line in enumerate(snippet_lines, 1):
new_snippet_lines.append(f"{n:>{width}} {line}".rstrip())
lined_snippet = '\n'.join(new_snippet_lines)
return lined_snippet
def get_os_platform():
platforms = {
'Linux': conf.LINUX, 'Windows': conf.WINDOWS, 'Darwin': conf.MAC}
......
import re
from textwrap import dedent
from nose.tools import assert_equal, assert_not_equal, assert_true, assert_false # @UnusedImport @UnresolvedImport
from tests import check_as_expected
try:
from ..superhelp import lint_conf # @UnresolvedImport @UnusedImport
from ..superhelp import conf, lint_conf # @UnresolvedImport @UnusedImport
except (ImportError, ValueError):
from pathlib import Path
import sys
parent = str(Path.cwd().parent.parent)
sys.path.insert(0, parent)
from superhelp import lint_conf # @Reimport
from superhelp import conf, lint_conf # @Reimport
def test_linter_regex():
tests = [
......@@ -27,4 +30,97 @@ def test_linter_regex():
lint_conf.LINT_PATTERN, lint_str, flags=re.VERBOSE).groupdict() # @UndefinedVariable
assert_equal(actual_dict, expected_dict)
ROOT = 'superhelp.advisors.lint_advisors.'
def test_misc():
test_conf = [
(
"pet = 'cat'",
{
ROOT + 'lint_snippet': 0,
}
),
(
"names = ['Noor', 'Grant', 'Hyeji', 'Vicky', 'Olek', ]",
{
ROOT + 'lint_snippet': 0,
}
),
(
"names = ( 'Noor', 'Grant', 'Hyeji', 'Vicky', 'Olek', )", ## spaces around parentheses
{
ROOT + 'lint_snippet': 1,
}
),
(
"names = ('Noor' , 'Grant', 'Hyeji', 'Vicky', 'Olek')", ## spaces before comma
{
ROOT + 'lint_snippet': 1,
}
),
(
"a = 2+1", ## no spaces around operators
{
ROOT + 'lint_snippet': 1,
}
),
(
"a = 2 + 1", ## OK now it has spaces around operators
{
ROOT + 'lint_snippet': 0,
}
),
(
"a=2",
{
ROOT + 'lint_snippet': 1,
}
),
(
dedent("""\
def TooJammedUp():
pass
class LetMeBreathe:
pass
"""),
{
ROOT + 'lint_snippet': 1,
}
),
(
"a = 'vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv'",
{
ROOT + 'lint_snippet': 1,
}
),
(
"a = 'vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv'",
{
ROOT + 'lint_snippet': 0,
}
),
(
dedent("""
print("abc"
"def")
"""),
{
ROOT + 'lint_snippet': 1,
}
),
(
dedent("""
print("abc"
"def")
"""),
{
ROOT + 'lint_snippet': 0,
}
),
]
conf.INCLUDE_LINTING = True ## or else never gets even run to make or fail to make a message!
check_as_expected(test_conf)
conf.INCLUDE_LINTING = False
# test_misc()
# test_linter_regex()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment