diff --git a/fill3.py b/fill3.py index 9138a8c..374572e 100644 --- a/fill3.py +++ b/fill3.py @@ -445,9 +445,10 @@ def _urwid_screen(): screen.stop() -async def _update_screen(screen_widget, appearance_changed_event): +@asyncio.coroutine +def _update_screen(screen_widget, appearance_changed_event): while True: - await appearance_changed_event.wait() + yield from appearance_changed_event.wait() appearance_changed_event.clear() patch_screen(screen_widget) @@ -464,8 +465,7 @@ def main(loop, appearance_changed_event, screen_widget, exit_loop=None): loop.add_signal_handler(signal.SIGWINCH, appearance_changed_event.set) loop.add_signal_handler(signal.SIGINT, exit_loop) loop.add_signal_handler(signal.SIGTERM, exit_loop) - asyncio.ensure_future(_update_screen(screen_widget, - appearance_changed_event)) + asyncio.async(_update_screen(screen_widget, appearance_changed_event)) with terminal.hidden_cursor(), terminal.fullscreen(), \ _urwid_screen() as urwid_screen: loop.add_reader(sys.stdin, on_input, urwid_screen, screen_widget) diff --git a/fill3_test.py b/fill3_test.py index 1a02041..7c1ea55 100755 --- a/fill3_test.py +++ b/fill3_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.5 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (C) 2015-2017 Andrew Hamilton. All rights reserved. diff --git a/gut.py b/gut.py index fd68fcc..090666e 100755 --- a/gut.py +++ b/gut.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.5 +#!/usr/bin/env python3 # Copyright (C) 2015-2017 Andrew Hamilton. All rights reserved. # Licensed under the Artistic License 2.0. diff --git a/gut_test.py b/gut_test.py index 1a8b93b..7d87804 100755 --- a/gut_test.py +++ b/gut_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.5 +#!/usr/bin/env python3 # Copyright (C) 2015-2017 Andrew Hamilton. All rights reserved. # Licensed under the Artistic License 2.0. diff --git a/install-dependencies b/install-dependencies index 94e4765..210dbc8 100755 --- a/install-dependencies +++ b/install-dependencies @@ -4,9 +4,19 @@ set -e +DIST_ID=$(cat /etc/os-release | grep "^ID=" | cut -d "=" -f 2) +if [ $DIST_ID == "fedora" ]; then + INSTALL_CMD="dnf -y install" + INOTIFY_NAME="python3-inotify python3-pygments python3-docopt python3-pillow" +elif [ $DIST_ID == "arch" ]; then + INSTALL_CMD="pacman -S --noconfirm" + INOTIFY_NAME="python-pyinotify python-pygments python-docopt python-pillow" +else + INSTALL_CMD="apt --yes install" + INOTIFY_NAME="python3-pyinotify python3-pygments python3-docopt python3-pillow" +fi echo "Install the dependencies of the vigil script..." -sudo apt-get --yes install python3.5 python3-minimal python3-pygments python3-pyinotify \ - python3-docopt util-linux python3-pil +sudo $INSTALL_CMD $INOTIFY_NAME util-linux echo echo "Install all the tools vigil may need..." ./install-tools diff --git a/install-tools b/install-tools index 6d3365b..af85996 100755 --- a/install-tools +++ b/install-tools @@ -1,9 +1,30 @@ -#!/usr/bin/env python3.5 +#!/usr/bin/env python3 +import platform import subprocess import tools -command = ["sudo", "apt-get", "--yes", "install"] + list(tools.dependencies()) -subprocess.check_call(command) +dist_id = platform.linux_distribution()[0].lower() +pip_deps, pip3_deps, dist_deps = set(), set(), set() +for dependency in tools.dependencies(dist_id): + if "/" in dependency: + pip_version, pip_dependency = dependency.split("/") + (pip_deps if pip_version == "pip" else pip3_deps).add(pip_dependency) + else: + dist_deps.add(dependency) +cmd_for_dist = {"ubuntu": ["apt-get", "-y", "install"], + "debian": ["apt-get", "-y", "install"], + "fedora": ["dnf", "-y", "install"], + "arch": ["pacman", "-S", "--noconfirm"]} +if pip_deps: + dist_deps.add("python2-pip" if dist_id == "arch" else "python-pip") +if pip3_deps: + dist_deps.add("python-pip" if dist_id == "arch" else "python3-pip") +if dist_deps: + subprocess.check_call(["sudo"] + cmd_for_dist[dist_id] + list(dist_deps)) +if pip_deps: + subprocess.check_call(["sudo", "pip", "install"] + list(pip_deps)) +if pip3_deps: + subprocess.check_call(["sudo", "pip3", "install"] + list(pip3_deps)) diff --git a/lscolors_test.py b/lscolors_test.py index 8329afd..e806e15 100755 --- a/lscolors_test.py +++ b/lscolors_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.5 +#!/usr/bin/env python3 # Copyright (C) 2011, 2015-2017 Andrew Hamilton. All rights reserved. # Licensed under the Artistic License 2.0. diff --git a/make-readme.py b/make-readme.py index a1081bd..b36be05 100755 --- a/make-readme.py +++ b/make-readme.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.5 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (C) 2017 Andrew Hamilton. All rights reserved. diff --git a/sandbox_fs.py b/sandbox_fs.py index 77878c3..ae87da1 100755 --- a/sandbox_fs.py +++ b/sandbox_fs.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.5 +#!/usr/bin/env python3 # Copyright (C) 2017 Andrew Hamilton. All rights reserved. # Licensed under the Artistic License 2.0. @@ -51,7 +51,7 @@ def _parse_proc_mounts(): def _find_mounts(): all_mounts = set(part[1] for part in _parse_proc_mounts()) mount_points = {"/", "/usr", "/bin", "/etc", "/lib", "/dev", "/home", - "/boot", "/opt", "/run", "/root", "/var"} + "/boot", "/opt", "/run", "/root", "/var", "/vigil"} return all_mounts.intersection(mount_points) diff --git a/sandbox_fs_test.py b/sandbox_fs_test.py index f8f347c..89725af 100755 --- a/sandbox_fs_test.py +++ b/sandbox_fs_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.5 +#!/usr/bin/env python3 # Copyright (C) 2017 Andrew Hamilton. All rights reserved. # Licensed under the Artistic License 2.0. diff --git a/termstr_test.py b/termstr_test.py index ee6b824..e1d7a9b 100755 --- a/termstr_test.py +++ b/termstr_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.5 +#!/usr/bin/env python3 # Copyright (C) 2015-2017 Andrew Hamilton. All rights reserved. # Licensed under the Artistic License 2.0. diff --git a/test-distributions b/test-distributions new file mode 100755 index 0000000..3b00e22 --- /dev/null +++ b/test-distributions @@ -0,0 +1,78 @@ +#!/bin/bash + + +set -e + + +VIGIL_PATH=$(realpath $(dirname $0)) + + +function run_in_container { + CONTAINER=$1 + shift + [ -f $CONTAINER ] && OPTION="--image" || OPTION="--directory" + sudo systemd-nspawn --chdir=/vigil --overlay=$VIGIL_PATH:/vigil \ + $OPTION=$CONTAINER $@ +} + + +function build_ubuntu { + sudo debootstrap --components=main,restricted,universe,multiverse \ + zesty ubuntu.part + run_in_container ubuntu.part \ + ln -sf /lib/systemd/resolv.conf /etc/resolv.conf + run_in_container ubuntu.part apt-get update + mv ubuntu.part ubuntu +} + + +function build_fedora { + IMAGE="Fedora-Cloud-Base-25-1.3.x86_64.raw" + wget --continue "https://dl.fedoraproject.org/pub/fedora/linux/releases/25/CloudImages/x86_64/images/$IMAGE.xz" + unxz $IMAGE.xz + mv $IMAGE fedora +} + + +function build_debian { + sudo debootstrap --components=main,contrib,non-free \ + --include=sudo jessie debian.part + run_in_container debian.part apt-get update + mv debian.part debian +} + + +function build_archlinux { + ARCHIVE="archlinux-bootstrap-2017.05.01-x86_64.tar.gz" + wget --continue "http://mirrors.kernel.org/archlinux/iso/latest/$ARCHIVE" + tar -zxf $ARCHIVE + mv root.x86_64 archlinux.part + run_in_container archlinux.part pacman-key --init + run_in_container archlinux.part pacman-key --populate archlinux + echo "Server = http://mirror.rackspace.com/archlinux/\$repo/os/\$arch" > \ + archlinux.part/etc/pacman.d/mirrorlist + run_in_container archlinux.part pacman -Syyu --noconfirm + run_in_container archlinux.part pacman -S --noconfirm sudo grep + rm $ARCHIVE + mv archlinux.part archlinux +} + + +[ $# -eq 0 ] && WORK_PATH=$(mktemp -d --suffix=-vigil) || WORK_PATH="$1" +sudo apt-get install -y systemd-container debootstrap xz-utils wget +cd $WORK_PATH +for DISTRIBUTION in ubuntu fedora debian archlinux; do + if [ -e $DISTRIBUTION ]; then + echo "$DISTRIBUTION container already exists." + else + echo "Building $DISTRIBUTION container..." + build_$DISTRIBUTION + fi + echo "Installing vigil's dependencies in $DISTRIBUTION..." + run_in_container $DISTRIBUTION ./install-dependencies + echo "Successfully installed vigil's dependencies in $DISTRIBUTION." + echo "Removing $DISTRIBUTION container..." + sudo rm -rf $DISTRIBUTION +done +rmdir $WORK_PATH +echo "Finished." diff --git a/tools.py b/tools.py index 3b8f1e1..db860ae 100644 --- a/tools.py +++ b/tools.py @@ -4,6 +4,7 @@ # Licensed under the Artistic License 2.0. import ast +import asyncio import contextlib import dis import enum @@ -268,6 +269,7 @@ def contents(path): else: return pygments_(path) contents.dependencies = {"python3-pygments"} +contents.arch_dependencies = {"python-pygments"} def _is_python_syntax_correct(path, python_version): @@ -343,6 +345,9 @@ def mypy(path): status = Status.ok if returncode == 0 else Status.normal return status, fill3.Text(stdout) mypy.dependencies = {"mypy"} +mypy.fedora_dependencies = {"python3-mypy"} +mypy.debian_dependencies = {"pip3/mypy"} +mypy.arch_dependencies = {"pip3/mypy"} mypy.url = "mypy" @@ -375,6 +380,7 @@ def python_coverage(path): return Status.not_applicable, fill3.Text( "No corresponding test file: " + os.path.normpath(test_path)) python_coverage.dependencies = {"python-coverage", "python3-coverage"} +python_coverage.arch_dependencies = {"python2-coverage", "python-coverage"} python_coverage.url = "python3-coverage" @@ -387,16 +393,18 @@ python_profile.url = "https://docs.python.org/3/library/profile.html" def pycodestyle(path): - cmd = (["pycodestyle"] if _python_version(path) == "python" - else ["python3", "-m", "pycodestyle"]) - return _run_command(cmd + [path]) -pycodestyle.dependencies = {"pycodestyle", "python3-pycodestyle"} -pycodestyle.url = "pycodestyle" + return _run_command([_python_version(path), "-m", "pycodestyle", path]) +pycodestyle.dependencies = {"python-pycodestyle", "python3-pycodestyle"} +pycodestyle.fedora_dependencies = {"python2-pycodestyle", "python3-pycodestyle"} +pycodestyle.debian_dependencies = {"pip/pycodestyle", "pip3/pycodestyle"} +pycodestyle.arch_dependencies = {"python-pycodestyle", "python2-pycodestyle"} +pycodestyle.url = "python-pycodestyle" def pyflakes(path): return _run_command([_python_version(path), "-m", "pyflakes", path]) pyflakes.dependencies = {"pyflakes"} +pyflakes.arch_dependencies = {"python2-pyflakes", "python-pyflakes"} pyflakes.url = "pyflakes" @@ -404,6 +412,9 @@ def pylint(path): return _run_command([_python_version(path), "-m", "pylint", "--errors-only", path]) pylint.dependencies = {"pylint", "pylint3"} +pylint.fedora_dependencies = {"pylint", "python3-pylint"} +pylint.arch_dependencies = {"python2-pylint", "python-pylint"} +pylint.debian_dependencies = {"pip/pylint", "pip3/pylint"} pylint.url = "pylint3" @@ -445,6 +456,7 @@ def python_mccabe(path): status = Status.problem if max_score > 10 else Status.ok return status, fill3.Text(_colorize_mccabe(stdout, python_version)) python_mccabe.dependencies = {"python-mccabe", "python3-mccabe"} +python_mccabe.arch_dependencies = {"python2-mccabe", "python-mccabe"} python_mccabe.url = "python3-mccabe" @@ -474,6 +486,9 @@ def bandit(path): text_without_timestamp = "".join(text.splitlines(keepends=True)[2:]) return status, fill3.Text(text_without_timestamp) bandit.dependencies = {"python-bandit", "python3-bandit"} +bandit.fedora_dependencies = {"bandit"} +bandit.debian_dependencies = {"pip/bandit", "pip3/bandit"} +bandit.arch_dependencies = {"bandit"} bandit.url = "python3-bandit" @@ -495,6 +510,8 @@ def perldoc(path): return ((Status.normal, fill3.Text(stdout)) if returncode == 0 else (Status.not_applicable, fill3.Text(stderr))) perldoc.dependencies = {"perl-doc"} +perldoc.fedora_dependencies = {"perl-Pod-Perldoc"} +perldoc.arch_dependencies = {"perl-pod-perldoc"} perldoc.url = "http://perldoc.perl.org/" @@ -502,6 +519,7 @@ def perltidy(path): stdout, *rest = _do_command(["perltidy", "-st", path]) return Status.normal, _syntax_highlight_using_path(stdout, path) perltidy.dependencies = {"perltidy"} +perltidy.arch_dependencies = {"perl-test-perltidy"} perltidy.url = "http://perltidy.sourceforge.net/" @@ -583,6 +601,7 @@ nm.url = "https://linux.die.net/man/1/nm" def pdf2txt(path): return _run_command(["pdf2txt", path], Status.normal) pdf2txt.dependencies = {"python-pdfminer"} +pdf2txt.arch_dependencies = set() pdf2txt.url = "python-pdfminer" @@ -605,6 +624,7 @@ tidy.url = "tidy" def html2text(path): return _run_command(["html2text", path], Status.normal) html2text.dependencies = {"html2text"} +html2text.arch_dependencies = {"python-html2text"} html2text.url = "html2text" @@ -625,6 +645,8 @@ def bcpp(path): status = Status.normal if returncode == 0 else Status.problem return status, _syntax_highlight_using_path(stdout, path) bcpp.dependencies = {"bcpp"} +bcpp.fedora_dependencies = set() +bcpp.arch_dependencies = set() def uncrustify(path): @@ -638,12 +660,14 @@ def uncrustify(path): status = Status.normal if returncode == 0 else Status.problem return status, _syntax_highlight_using_path(stdout, path) uncrustify.dependencies = {"uncrustify"} +uncrustify.debian_dependencies = set() uncrustify.url = "uncrustify" def php5_syntax(path): return _run_command(["php", "--syntax-check", path]) php5_syntax.dependencies = {"php"} +php5_syntax.debian_dependencies = {"pip3/php"} php5_syntax.url = "https://en.wikipedia.org/wiki/PHP" @@ -678,6 +702,8 @@ def pil(path): result.append(termstr.TermStr(text, tuple(row_style))) return Status.normal, fill3.Fixed(result) pil.dependencies = {"python3-pil"} +pil.fedora_dependencies = {"python3-pillow"} +pil.arch_dependencies = {"python-pillow"} pil.url = "python3-pil" @@ -697,6 +723,8 @@ def pil_half(path): for index in range(0, image.height, 2)]) return Status.normal, result pil_half.dependencies = {"python3-pil"} +pil_half.fedora_dependencies = {"python3-pillow"} +pil_half.arch_dependencies = {"python-pillow"} pil_half.url = "python3-pil" @@ -788,7 +816,8 @@ class Result: self.status = status self.entry.appearance_cache = None - async def run(self, log, appearance_changed_event, runner): + @asyncio.coroutine + def run(self, log, appearance_changed_event, runner): self.is_placeholder = False tool_name = tool_name_colored(self.tool, self.path) path = path_colored(self.path) @@ -799,7 +828,7 @@ class Result: runner.pause() appearance_changed_event.set() start_time = time.time() - new_status = await runner.run_tool(self.path, self.tool) + new_status = yield from runner.run_tool(self.path, self.tool) Result.result.fget.evict(self) end_time = time.time() self.set_status(new_status) @@ -866,10 +895,32 @@ def tools_all(): return tools_ -def dependencies(): +def tool_dependencies(tool, distribution="ubuntu"): + if distribution == "ubuntu": + return tool.dependencies + elif distribution == "debian": + try: + return tool.debian_dependencies + except AttributeError: + return tool.dependencies + elif distribution == "fedora": + try: + return tool.fedora_dependencies + except AttributeError: + return tool.dependencies + elif distribution == "arch": + try: + return tool.arch_dependencies + except AttributeError: + return tool.dependencies + else: + raise NotImplementedError + + +def dependencies(distribution="ubuntu"): dependencies_all = set() for tool in tools_all(): - dependencies_all.update(tool.dependencies) + dependencies_all.update(tool_dependencies(tool, distribution)) return dependencies_all diff --git a/tools_test.py b/tools_test.py index a59cdcb..3f7eb59 100755 --- a/tools_test.py +++ b/tools_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.5 +#!/usr/bin/env python3 # Copyright (C) 2017 Andrew Hamilton. All rights reserved. # Licensed under the Artistic License 2.0. diff --git a/vigil b/vigil index 10d8bc8..e00151d 100755 --- a/vigil +++ b/vigil @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.5 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (C) 2015-2017 Andrew Hamilton. All rights reserved. @@ -559,8 +559,7 @@ class Screen: future = worker_.job_runner( self._summary, self._log, self._summary._jobs_added_event, self._appearance_changed_event) - worker_.future = asyncio.ensure_future( - future, loop=self._main_loop) + worker_.future = asyncio.async(future, loop=self._main_loop) self.workers = workers def stop_workers(self): diff --git a/vigil_test.py b/vigil_test.py index dbaf270..d71a398 100755 --- a/vigil_test.py +++ b/vigil_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.5 +#!/usr/bin/env python3 # Copyright (C) 2015-2017 Andrew Hamilton. All rights reserved. # Licensed under the Artistic License 2.0. diff --git a/worker.py b/worker.py index 0c33ecb..24d6593 100755 --- a/worker.py +++ b/worker.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.5 +#!/usr/bin/env python3 # Copyright (C) 2015-2017 Andrew Hamilton. All rights reserved. # Licensed under the Artistic License 2.0. @@ -20,7 +20,8 @@ class Worker: self.process = None self.child_pgid = None - async def create_process(self): + @asyncio.coroutine + def create_process(self): if self.is_sandboxed: sandbox_fs_path = os.path.join(os.path.dirname(__file__), "sandbox_fs") @@ -32,22 +33,24 @@ class Worker: *command, stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, preexec_fn=os.setsid) - self.process = await create - pid_line = await self.process.stdout.readline() + self.process = yield from create + pid_line = yield from self.process.stdout.readline() self.child_pgid = int(pid_line.strip()) os.setpriority(os.PRIO_PGRP, self.child_pgid, 19) - async def run_tool(self, path, tool): + @asyncio.coroutine + def run_tool(self, path, tool): self.process.stdin.write(("%s\n%s\n" % (tool.__qualname__, path)).encode("utf-8")) - data = await self.process.stdout.readline() + data = yield from self.process.stdout.readline() return tools.Status(int(data)) - async def job_runner(self, summary, log, jobs_added_event, - appearance_changed_event): - await self.create_process() + @asyncio.coroutine + def job_runner(self, summary, log, jobs_added_event, + appearance_changed_event): + yield from self.create_process() while True: - await jobs_added_event.wait() + yield from jobs_added_event.wait() while True: try: self.result = summary.get_closest_placeholder() @@ -58,8 +61,7 @@ class Worker: if self.is_being_tested: os.kill(os.getpid(), signal.SIGINT) break - await self.result.run(log, appearance_changed_event, - self) + yield from self.result.run(log, appearance_changed_event, self) summary.completed_total += 1 jobs_added_event.clear() diff --git a/worker_test.py b/worker_test.py index a2760a1..4fe5102 100755 --- a/worker_test.py +++ b/worker_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.5 +#!/usr/bin/env python3 # Copyright (C) 2017 Andrew Hamilton. All rights reserved. # Licensed under the Artistic License 2.0.