ahamilton/eris
1
0
Fork 0
Code Releases Activity

tools: Not detecting the type of python files. Assuming python3.

Browse source 
- Also now not using any tools that depend on python2.
- Simplifies installation.
- Personally not using python2.
- Eris is not used widely enough to justify supporting python2 files.
- The way of determining whether a source file was python2 or python3 was
  not very good.
This commit is contained in:
Andrew Hamilton 2019-01-21 11:46:51 +10:00
parent 7d412ec8f1
commit 85a5840cff
17 changed files with 53 additions and 140 deletions
Download patch file Download diff file Expand all files Collapse all files

111
eris/tools.py
View file

@ -239,7 +239,7 @@ def metadata(path):
return (Status.normal, fill3.Text(fill3.join("", text))) return (Status.normal, fill3.Text(fill3.join("", text)))
@deps(deps={"pip3/pygments"}, url="python3-pygments") @deps(deps={"pip/pygments"}, url="python3-pygments")
def contents(path): def contents(path):
with open(path) as file_: with open(path) as file_:
try: try:
@ -254,35 +254,17 @@ def contents(path):
return Status.normal, text_widget return Status.normal, text_widget
def _is_python_syntax_correct(path, python_version): @deps(url="https://en.wikipedia.org/wiki/Python_syntax_and_semantics")
if python_version == "python": def python_syntax(path):
stdin, stdout, returncode = _do_command(
["python", "-c",
f"__import__('compiler').parse(open('{path}').read())"])
return returncode == 0
else: # python3
with open(path) as f: with open(path) as f:
source = f.read() source = f.read()
try: try:
ast.parse(source) ast.parse(source)
except: except SyntaxError:
return False is_correct = False
return True else:
is_correct = True
return (Status.ok if is_correct else Status.problem), fill3.Text("")
def _python_version(path): # Need a better hueristic
for version in [PYTHON_EXECUTABLE, "python"]:
if _is_python_syntax_correct(path, version):
return version
return PYTHON_EXECUTABLE
@deps(deps={"python"},
url="https://en.wikipedia.org/wiki/Python_syntax_and_semantics")
def python_syntax(path):
status = (Status.ok if _is_python_syntax_correct(path, "python") or
_is_python_syntax_correct(path, "python3") else Status.problem)
return status, fill3.Text("")
def _has_shebang_line(path): def _has_shebang_line(path):
@ -295,12 +277,11 @@ def _is_python_test_file(path):
return path.endswith("_test.py") or path.startswith("test_") return path.endswith("_test.py") or path.startswith("test_")
@deps(deps={"python", "python3"}, @deps(url="https://docs.python.org/3/library/unittest.html")
url="https://docs.python.org/3/library/unittest.html")
def python_unittests(path): def python_unittests(path):
if _is_python_test_file(path): if _is_python_test_file(path):
command = ([path] if _has_shebang_line(path) command = ([path] if _has_shebang_line(path)
else [_python_version(path), path]) else [PYTHON_EXECUTABLE, path])
stdout, stderr, returncode = _do_command(command, timeout=TIMEOUT) stdout, stderr, returncode = _do_command(command, timeout=TIMEOUT)
status = Status.ok if returncode == 0 else Status.problem status = Status.ok if returncode == 0 else Status.problem
return status, fill3.Text(stdout + "\n" + stderr) return status, fill3.Text(stdout + "\n" + stderr)
@ -308,11 +289,10 @@ def python_unittests(path):
return Status.not_applicable, fill3.Text("No tests.") return Status.not_applicable, fill3.Text("No tests.")
@deps(deps={"python", "python3"}, @deps(url="https://docs.python.org/3/library/pydoc.html")
url="https://docs.python.org/3/library/pydoc.html")
def pydoc(path): def pydoc(path):
stdout, stderr, returncode = _do_command( stdout, stderr, returncode = _do_command(
[_python_version(path), "-m", "pydoc", path], timeout=TIMEOUT) [PYTHON_EXECUTABLE, "-m", "pydoc", path], timeout=TIMEOUT)
status = Status.normal if returncode == 0 else Status.not_applicable status = Status.normal if returncode == 0 else Status.not_applicable
if not stdout.startswith("Help on module"): if not stdout.startswith("Help on module"):
status = Status.not_applicable status = Status.not_applicable
@ -320,7 +300,7 @@ def pydoc(path):
return status, fill3.Text(_fix_input(stdout)) return status, fill3.Text(_fix_input(stdout))
@deps(deps={"pip3/mypy"}, url="http://mypy-lang.org/", executables={"mypy"}) @deps(deps={"pip/mypy"}, url="http://mypy-lang.org/", executables={"mypy"})
def mypy(path): def mypy(path):
stdout, stderr, returncode = _do_command( stdout, stderr, returncode = _do_command(
[PYTHON_EXECUTABLE, "-m", "mypy", path], timeout=TIMEOUT) [PYTHON_EXECUTABLE, "-m", "mypy", path], timeout=TIMEOUT)
@ -335,14 +315,13 @@ def _colorize_coverage_report(text):
for line in text.splitlines(keepends=True)]) for line in text.splitlines(keepends=True)])
@deps(deps={"pip/coverage", "pip3/coverage"}, @deps(deps={"pip/coverage"}, url="https://coverage.readthedocs.io/")
url="https://coverage.readthedocs.io/")
def python_coverage(path): def python_coverage(path):
# FIX: Also use test_*.py files. # FIX: Also use test_*.py files.
test_path = path[:-(len(".py"))] + "_test.py" test_path = path[:-(len(".py"))] + "_test.py"
if os.path.exists(test_path): if os.path.exists(test_path):
with tempfile.TemporaryDirectory() as temp_dir: with tempfile.TemporaryDirectory() as temp_dir:
coverage_cmd = [_python_version(path), "-m", "coverage"] coverage_cmd = [PYTHON_EXECUTABLE, "-m", "coverage"]
coverage_path = os.path.join(temp_dir, "coverage") coverage_path = os.path.join(temp_dir, "coverage")
env = os.environ.copy() env = os.environ.copy()
env["COVERAGE_FILE"] = coverage_path env["COVERAGE_FILE"] = coverage_path
@ -361,28 +340,25 @@ def python_coverage(path):
"No corresponding test file: " + os.path.normpath(test_path)) "No corresponding test file: " + os.path.normpath(test_path))
@deps(deps={"pip/pycodestyle", "pip3/pycodestyle"}, @deps(deps={"pip/pycodestyle"}, url="http://pycodestyle.pycqa.org/en/latest/")
url="http://pycodestyle.pycqa.org/en/latest/")
def pycodestyle(path): def pycodestyle(path):
return _run_command([_python_version(path), "-m", "pycodestyle", path]) return _run_command([PYTHON_EXECUTABLE, "-m", "pycodestyle", path])
@deps(deps={"pip/pydocstyle", "pip3/pydocstyle"}, @deps(deps={"pip/pydocstyle"}, url="http://pycodestyle.pycqa.org/en/latest/")
url="http://pycodestyle.pycqa.org/en/latest/")
def pydocstyle(path): def pydocstyle(path):
return _run_command([_python_version(path), "-m", "pydocstyle", path]) return _run_command([PYTHON_EXECUTABLE, "-m", "pydocstyle", path])
@deps(deps={"pip/pyflakes", "pip3/pyflakes"}, @deps(deps={"pip/pyflakes"}, url="https://pypi.org/project/pyflakes/")
url="https://pypi.org/project/pyflakes/")
def pyflakes(path): def pyflakes(path):
return _run_command([_python_version(path), "-m", "pyflakes", path]) return _run_command([PYTHON_EXECUTABLE, "-m", "pyflakes", path])
@deps(deps={"pip/pylint", "pip3/pylint"}, url="https://www.pylint.org/") @deps(deps={"pip/pylint"}, url="https://www.pylint.org/")
def pylint(path): def pylint(path):
return _run_command([_python_version(path), "-m", "pylint", return _run_command([PYTHON_EXECUTABLE, "-m", "pylint", "--errors-only",
"--errors-only", path]) path])
@deps(url="https://github.com/ahamilton/eris/blob/master/gut.py") @deps(url="https://github.com/ahamilton/eris/blob/master/gut.py")
@ -393,59 +369,52 @@ def python_gut(path):
return Status.normal, source_widget return Status.normal, source_widget
@deps(deps={"python", "python3"}, @deps(url="https://docs.python.org/3/library/modulefinder.html")
url="https://docs.python.org/3/library/modulefinder.html")
def python_modulefinder(path): def python_modulefinder(path):
return _run_command([_python_version(path), "-m", "modulefinder", path], return _run_command([PYTHON_EXECUTABLE, "-m", "modulefinder", path],
Status.normal) Status.normal)
@deps(deps={"python", "python3"}, @deps(url="https://docs.python.org/3/library/dis.html")
url="https://docs.python.org/3/library/dis.html")
def dis(path): def dis(path):
return _run_command([_python_version(path), "-m", "dis", path], return _run_command([PYTHON_EXECUTABLE, "-m", "dis", path], Status.normal)
Status.normal)
def _get_mccabe_line_score(line, python_version): def _get_mccabe_line_score(line):
position, function_name, score = line.split() position, function_name, score = line.split()
return int(score if python_version == PYTHON_EXECUTABLE else score[:-1]) return int(score)
def _colorize_mccabe(text, python_version): def _colorize_mccabe(text):
return fill3.join("", [ return fill3.join("", [
termstr.TermStr(line).fg_color(termstr.Color.yellow) termstr.TermStr(line).fg_color(termstr.Color.yellow)
if _get_mccabe_line_score(line, python_version) > 10 else line if _get_mccabe_line_score(line) > 10 else line
for line in text.splitlines(keepends=True)]) for line in text.splitlines(keepends=True)])
@deps(deps={"pip/mccabe", "pip3/mccabe"}, @deps(deps={"pip/mccabe"}, url="https://pypi.org/project/mccabe/")
url="https://pypi.org/project/mccabe/")
def python_mccabe(path): def python_mccabe(path):
python_version = _python_version(path) stdout, *rest = _do_command([PYTHON_EXECUTABLE, "-m", "mccabe", path])
stdout, *rest = _do_command([python_version, "-m", "mccabe", path])
max_score = 0 max_score = 0
with contextlib.suppress(ValueError): # When there are no lines with contextlib.suppress(ValueError): # When there are no lines
max_score = max(_get_mccabe_line_score(line, python_version) max_score = max(_get_mccabe_line_score(line)
for line in stdout.splitlines()) for line in stdout.splitlines())
status = Status.problem if max_score > 10 else Status.ok status = Status.problem if max_score > 10 else Status.ok
return status, fill3.Text(_colorize_mccabe(stdout, python_version)) return status, fill3.Text(_colorize_mccabe(stdout))
# FIX: Reenable when pydisasm is not causing problems # FIX: Reenable when pydisasm is not causing problems
# @deps(deps={"pip3/xdis"}, executables={"pydisasm"}, # @deps(deps={"pip/xdis"}, executables={"pydisasm"},
# url="https://pypi.python.org/pypi/xdis") # url="https://pypi.python.org/pypi/xdis")
# def pydisasm(path): # def pydisasm(path):
# return _run_command(["pydisasm", path], Status.normal, # return _run_command(["pydisasm", path], Status.normal,
# Status.not_applicable) # Status.not_applicable)
@deps(deps={"pip/bandit", "pip3/bandit"}, @deps(deps={"pip/bandit"}, url="https://pypi.org/project/bandit/")
url="https://pypi.org/project/bandit/")
def bandit(path): def bandit(path):
python_version = _python_version(path)
stdout, stderr, returncode = _do_command( stdout, stderr, returncode = _do_command(
[python_version, "-m", "bandit.cli.main", "-f", "txt", path], [PYTHON_EXECUTABLE, "-m", "bandit.cli.main", "-f", "txt", path],
timeout=TIMEOUT) timeout=TIMEOUT)
status = Status.ok if returncode == 0 else Status.problem status = Status.ok if returncode == 0 else Status.problem
text_without_timestamp = "".join(stdout.splitlines(keepends=True)[2:]) text_without_timestamp = "".join(stdout.splitlines(keepends=True)[2:])
@ -534,7 +503,7 @@ def _resize_image(image, new_width):
PIL.Image.ANTIALIAS) PIL.Image.ANTIALIAS)
@deps(deps={"pip3/pillow"}, url="python3-pil") @deps(deps={"pip/pillow"}, url="python3-pil")
def pil(path): def pil(path):
import PIL.Image import PIL.Image
with open(path, "rb") as image_file: with open(path, "rb") as image_file:

6
eris/tools.toml
View file

@ -136,8 +136,8 @@ tools_for_extensions = [
success_status = "normal" success_status = "normal"
[pdf2txt] [pdf2txt]
dependencies = ["pip/pdfminer"] dependencies = ["pip/pdfminer.six"]
url = "https://euske.github.io/pdfminer/" url = "https://github.com/pdfminer/pdfminer.six"
command = "pdf2txt.py" command = "pdf2txt.py"
success_status = "normal" success_status = "normal"
@ -216,7 +216,7 @@ tools_for_extensions = [
command = "golint -set_exit_status" command = "golint -set_exit_status"
[yamllint] [yamllint]
dependencies = ["pip3/yamllint"] dependencies = ["pip/yamllint"]
url = "https://github.com/adrienverge/yamllint" url = "https://github.com/adrienverge/yamllint"
command = "python3.7 -m yamllint" command = "python3.7 -m yamllint"

11
install-tools
View file

@ -8,21 +8,16 @@ import subprocess
import eris.tools import eris.tools
pip_deps, pip3_deps, dist_deps = set(), set(), set() pip_deps, dist_deps = set(), set()
for dependency in eris.tools.dependencies(): for dependency in eris.tools.dependencies():
if "/" in dependency: if "/" in dependency:
pip_version, pip_dependency = dependency.split("/") pip_version, pip_dependency = dependency.split("/")
(pip_deps if pip_version == "pip" else pip3_deps).add(pip_dependency) pip_deps.add(pip_dependency)
else: else:
dist_deps.add(dependency) dist_deps.add(dependency)
if pip_deps:
dist_deps.add("python-pip")
if dist_deps: if dist_deps:
subprocess.run(["sudo", "apt-get", "-y", "install"] + list(dist_deps), subprocess.run(["sudo", "apt-get", "-y", "install"] + list(dist_deps),
check=True) check=True)
if pip_deps: if pip_deps:
subprocess.run(["python", "-m", "pip", "install"] + list(pip_deps),
check=True)
if pip3_deps:
subprocess.run(["python" + eris.tools.PYTHON_VERSION, "-m", "pip", subprocess.run(["python" + eris.tools.PYTHON_VERSION, "-m", "pip",
"install"] + list(pip3_deps), check=True) "install"] + list(pip_deps), check=True)

4
tests/golden-files/input/hi.py
View file

@ -1,4 +0,0 @@
def hi():
print "hi"

19
tests/golden-files/results/bandit-hi_py
View file

@ -1,19 +0,0 @@
Test results:
No issues identified.
Code scanned:
Total lines of code: 2
Total lines skipped (#nosec): 0
Run metrics:
Total issues (by severity):
Undefined: 0
Low: 0
Medium: 0
High: 0
Total issues (by confidence):
Undefined: 0
Low: 0
Medium: 0
High: 0
Files skipped (0):

1
tests/golden-files/results/mypy-hi_py
View file

@ -1 +0,0 @@
input/hi.py:4: error: Missing parentheses in call to 'print'. Did you mean print("hi")?

0
tests/golden-files/results/pycodestyle-hi_py
View file

12
tests/golden-files/results/pydoc-hi_py
View file

@ -1,12 +0,0 @@
Help on module hi:
NAME
hi
FILE
input/hi.py
FUNCTIONS
hi()

0
tests/golden-files/results/pyflakes-hi_py
View file

1
tests/golden-files/results/pylint-hi_py
View file

@ -1 +0,0 @@
No config file found, using default configuration

4
tests/golden-files/results/python_coverage-hi_py
View file

@ -1,4 +0,0 @@
(B
(B> def hi():(B
(B> print "hi"(B

1
tests/golden-files/results/python_gut-hi_py
View file

@ -1 +0,0 @@
(Bdef(B (Bhi(B():(B

1
tests/golden-files/results/python_mccabe-hi_py
View file

@ -1 +0,0 @@
("3:0: 'hi'", 1)

4
tests/golden-files/results/python_modulefinder-hi_py
View file

@ -1,4 +0,0 @@
Name File
---- ----
m __main__ ./input/hi.py

0
tests/golden-files/results/python_syntax-hi_py
View file

1
tests/golden-files/results/python_unittests-hi_py
View file

@ -1 +0,0 @@
No tests.

11
tests/tools_test.py
View file

@ -81,7 +81,7 @@ class ToolsTestCase(unittest.TestCase):
def test_contents(self): def test_contents(self):
self._test_tool(tools.contents, [("hi3.py", tools.Status.normal)]) self._test_tool(tools.contents, [("hi3.py", tools.Status.normal)])
HI_OK = [("hi3.py", tools.Status.ok), ("hi.py", tools.Status.ok)] HI_OK = [("hi3.py", tools.Status.ok)]
def test_python_syntax(self): def test_python_syntax(self):
self._test_tool(tools.python_syntax, self.HI_OK) self._test_tool(tools.python_syntax, self.HI_OK)
@ -94,8 +94,7 @@ class ToolsTestCase(unittest.TestCase):
# ("hi3_test.py", tools.Status.ok), # ("hi3_test.py", tools.Status.ok),
# ("test_foo.py", tools.Status.ok)]) # ("test_foo.py", tools.Status.ok)])
HI_NORMAL = [("hi3.py", tools.Status.normal), HI_NORMAL = [("hi3.py", tools.Status.normal)]
("hi.py", tools.Status.normal)]
def test_pydoc(self): def test_pydoc(self):
# FIX: This is failing inside AppImages. # FIX: This is failing inside AppImages.
@ -103,8 +102,7 @@ class ToolsTestCase(unittest.TestCase):
self._test_tool(tools.pydoc, self.HI_NORMAL) self._test_tool(tools.pydoc, self.HI_NORMAL)
def test_mypy(self): def test_mypy(self):
self._test_tool(tools.mypy, [("hi3.py", tools.Status.ok), self._test_tool(tools.mypy, self.HI_OK)
("hi.py", tools.Status.problem)])
def test_python_coverage(self): def test_python_coverage(self):
self._test_tool(tools.python_coverage, self.HI_NORMAL) self._test_tool(tools.python_coverage, self.HI_NORMAL)
@ -128,8 +126,7 @@ class ToolsTestCase(unittest.TestCase):
self._test_tool(tools.python_mccabe, self.HI_OK) self._test_tool(tools.python_mccabe, self.HI_OK)
def test_bandit(self): def test_bandit(self):
self._test_tool(tools.bandit, [("hi3.py", tools.Status.ok), self._test_tool(tools.bandit, self.HI_OK)
("hi.py", tools.Status.ok)])
# FIX: Make the golden-file deterministic # FIX: Make the golden-file deterministic
# def test_pydisasm(self): # def test_pydisasm(self):