Make vigil work on other distributions. (fedora, debian, archlinux)

- Now using python3.4 on debian, and python3.5 elsewhere.
 - Added test-distributions script that checks that
   install-dependencies works on different distributions.

fill3.py

@@ -445,9 +445,10 @@ def _urwid_screen():
         screen.stop()
 
 
-async def _update_screen(screen_widget, appearance_changed_event):
+@asyncio.coroutine
+def _update_screen(screen_widget, appearance_changed_event):
     while True:
-        await appearance_changed_event.wait()
+        yield from appearance_changed_event.wait()
         appearance_changed_event.clear()
         patch_screen(screen_widget)
 
@@ -464,8 +465,7 @@ def main(loop, appearance_changed_event, screen_widget, exit_loop=None):
     loop.add_signal_handler(signal.SIGWINCH, appearance_changed_event.set)
     loop.add_signal_handler(signal.SIGINT, exit_loop)
     loop.add_signal_handler(signal.SIGTERM, exit_loop)
-    asyncio.ensure_future(_update_screen(screen_widget,
+    asyncio.async(_update_screen(screen_widget, appearance_changed_event))
-                                         appearance_changed_event))
     with terminal.hidden_cursor(), terminal.fullscreen(), \
             _urwid_screen() as urwid_screen:
         loop.add_reader(sys.stdin, on_input, urwid_screen, screen_widget)

fill3_test.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.5
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2015-2017 Andrew Hamilton. All rights reserved.

gut.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.5
+#!/usr/bin/env python3
 
 # Copyright (C) 2015-2017 Andrew Hamilton. All rights reserved.
 # Licensed under the Artistic License 2.0.

gut_test.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.5
+#!/usr/bin/env python3
 
 # Copyright (C) 2015-2017 Andrew Hamilton. All rights reserved.
 # Licensed under the Artistic License 2.0.

install-dependencies

@@ -4,9 +4,19 @@
 set -e
 
 
+DIST_ID=$(cat /etc/os-release | grep "^ID=" | cut -d "=" -f 2)
+if [ $DIST_ID == "fedora" ]; then
+    INSTALL_CMD="dnf -y install"
+    INOTIFY_NAME="python3-inotify python3-pygments python3-docopt python3-pillow"
+elif [ $DIST_ID == "arch" ]; then
+    INSTALL_CMD="pacman -S --noconfirm"
+    INOTIFY_NAME="python-pyinotify python-pygments python-docopt python-pillow"
+else
+    INSTALL_CMD="apt --yes install"
+    INOTIFY_NAME="python3-pyinotify python3-pygments python3-docopt python3-pillow"
+fi
 echo "Install the dependencies of the vigil script..."
-sudo apt-get --yes install python3.5 python3-minimal python3-pygments python3-pyinotify \
+sudo $INSTALL_CMD $INOTIFY_NAME util-linux
-     python3-docopt util-linux python3-pil
 echo
 echo "Install all the tools vigil may need..."
 ./install-tools

install-tools

@@ -1,9 +1,30 @@
-#!/usr/bin/env python3.5
+#!/usr/bin/env python3
 
 
+import platform
 import subprocess
 import tools
 
 
-command = ["sudo", "apt-get", "--yes", "install"] + list(tools.dependencies())
+dist_id = platform.linux_distribution()[0].lower()
-subprocess.check_call(command)
+pip_deps, pip3_deps, dist_deps = set(), set(), set()
+for dependency in tools.dependencies(dist_id):
+    if "/" in dependency:
+        pip_version, pip_dependency = dependency.split("/")
+        (pip_deps if pip_version == "pip" else pip3_deps).add(pip_dependency)
+    else:
+        dist_deps.add(dependency)
+cmd_for_dist = {"ubuntu": ["apt-get", "-y", "install"],
+                "debian": ["apt-get", "-y", "install"],
+                "fedora": ["dnf", "-y", "install"],
+                "arch": ["pacman", "-S", "--noconfirm"]}
+if pip_deps:
+    dist_deps.add("python2-pip" if dist_id == "arch" else "python-pip")
+if pip3_deps:
+    dist_deps.add("python-pip" if dist_id == "arch" else "python3-pip")
+if dist_deps:
+    subprocess.check_call(["sudo"] + cmd_for_dist[dist_id] + list(dist_deps))
+if pip_deps:
+    subprocess.check_call(["sudo", "pip", "install"] + list(pip_deps))
+if pip3_deps:
+    subprocess.check_call(["sudo", "pip3", "install"] + list(pip3_deps))

lscolors_test.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.5
+#!/usr/bin/env python3
 
 # Copyright (C) 2011, 2015-2017 Andrew Hamilton. All rights reserved.
 # Licensed under the Artistic License 2.0.

make-readme.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.5
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2017 Andrew Hamilton. All rights reserved.

sandbox_fs.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.5
+#!/usr/bin/env python3
 
 # Copyright (C) 2017 Andrew Hamilton. All rights reserved.
 # Licensed under the Artistic License 2.0.
@@ -51,7 +51,7 @@ def _parse_proc_mounts():
 def _find_mounts():
     all_mounts = set(part[1] for part in _parse_proc_mounts())
     mount_points = {"/", "/usr", "/bin", "/etc", "/lib", "/dev", "/home",
-                    "/boot", "/opt", "/run", "/root", "/var"}
+                    "/boot", "/opt", "/run", "/root", "/var", "/vigil"}
     return all_mounts.intersection(mount_points)
 
 

sandbox_fs_test.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.5
+#!/usr/bin/env python3
 
 # Copyright (C) 2017 Andrew Hamilton. All rights reserved.
 # Licensed under the Artistic License 2.0.

termstr_test.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.5
+#!/usr/bin/env python3
 
 # Copyright (C) 2015-2017 Andrew Hamilton. All rights reserved.
 # Licensed under the Artistic License 2.0.

test-distributions

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+
+set -e
+
+
+VIGIL_PATH=$(realpath $(dirname $0))
+
+
+function run_in_container {
+    CONTAINER=$1
+    shift
+    [ -f $CONTAINER ] && OPTION="--image" || OPTION="--directory"
+    sudo systemd-nspawn --chdir=/vigil --overlay=$VIGIL_PATH:/vigil \
+	 $OPTION=$CONTAINER $@
+}
+
+
+function build_ubuntu {
+    sudo debootstrap --components=main,restricted,universe,multiverse \
+	 zesty ubuntu.part
+    run_in_container ubuntu.part \
+		     ln -sf /lib/systemd/resolv.conf /etc/resolv.conf
+    run_in_container ubuntu.part apt-get update
+    mv ubuntu.part ubuntu
+}
+
+
+function build_fedora {
+    IMAGE="Fedora-Cloud-Base-25-1.3.x86_64.raw"
+    wget --continue "https://dl.fedoraproject.org/pub/fedora/linux/releases/25/CloudImages/x86_64/images/$IMAGE.xz"
+    unxz $IMAGE.xz
+    mv $IMAGE fedora
+}
+
+
+function build_debian {
+    sudo debootstrap --components=main,contrib,non-free \
+	 --include=sudo jessie debian.part
+    run_in_container debian.part apt-get update
+    mv debian.part debian
+}
+
+
+function build_archlinux {
+    ARCHIVE="archlinux-bootstrap-2017.05.01-x86_64.tar.gz"
+    wget --continue "http://mirrors.kernel.org/archlinux/iso/latest/$ARCHIVE"
+    tar -zxf $ARCHIVE
+    mv root.x86_64 archlinux.part
+    run_in_container archlinux.part pacman-key --init
+    run_in_container archlinux.part pacman-key --populate archlinux
+    echo "Server = http://mirror.rackspace.com/archlinux/\$repo/os/\$arch" > \
+	 archlinux.part/etc/pacman.d/mirrorlist
+    run_in_container archlinux.part pacman -Syyu --noconfirm
+    run_in_container archlinux.part pacman -S --noconfirm sudo grep
+    rm $ARCHIVE
+    mv archlinux.part archlinux
+}
+
+
+[ $# -eq 0 ] && WORK_PATH=$(mktemp -d --suffix=-vigil) || WORK_PATH="$1"
+sudo apt-get install -y systemd-container debootstrap xz-utils wget
+cd $WORK_PATH
+for DISTRIBUTION in ubuntu fedora debian archlinux; do
+    if [ -e $DISTRIBUTION ]; then
+	echo "$DISTRIBUTION container already exists."
+    else
+	echo "Building $DISTRIBUTION container..."
+	build_$DISTRIBUTION
+    fi
+    echo "Installing vigil's dependencies in $DISTRIBUTION..."
+    run_in_container $DISTRIBUTION ./install-dependencies
+    echo "Successfully installed vigil's dependencies in $DISTRIBUTION."
+    echo "Removing $DISTRIBUTION container..."
+    sudo rm -rf $DISTRIBUTION
+done
+rmdir $WORK_PATH
+echo "Finished."

tools.py

@@ -4,6 +4,7 @@
 # Licensed under the Artistic License 2.0.
 
 import ast
+import asyncio
 import contextlib
 import dis
 import enum
@@ -268,6 +269,7 @@ def contents(path):
     else:
         return pygments_(path)
 contents.dependencies = {"python3-pygments"}
+contents.arch_dependencies = {"python-pygments"}
 
 
 def _is_python_syntax_correct(path, python_version):
@@ -343,6 +345,9 @@ def mypy(path):
     status = Status.ok if returncode == 0 else Status.normal
     return status, fill3.Text(stdout)
 mypy.dependencies = {"mypy"}
+mypy.fedora_dependencies = {"python3-mypy"}
+mypy.debian_dependencies = {"pip3/mypy"}
+mypy.arch_dependencies = {"pip3/mypy"}
 mypy.url = "mypy"
 
 
@@ -375,6 +380,7 @@ def python_coverage(path):
         return Status.not_applicable, fill3.Text(
             "No corresponding test file: " + os.path.normpath(test_path))
 python_coverage.dependencies = {"python-coverage", "python3-coverage"}
+python_coverage.arch_dependencies = {"python2-coverage", "python-coverage"}
 python_coverage.url = "python3-coverage"
 
 
@@ -387,16 +393,18 @@ python_profile.url = "https://docs.python.org/3/library/profile.html"
 
 
 def pycodestyle(path):
-    cmd = (["pycodestyle"] if _python_version(path) == "python"
+    return _run_command([_python_version(path), "-m", "pycodestyle", path])
-           else ["python3", "-m", "pycodestyle"])
+pycodestyle.dependencies = {"python-pycodestyle", "python3-pycodestyle"}
-    return _run_command(cmd + [path])
+pycodestyle.fedora_dependencies = {"python2-pycodestyle", "python3-pycodestyle"}
-pycodestyle.dependencies = {"pycodestyle", "python3-pycodestyle"}
+pycodestyle.debian_dependencies = {"pip/pycodestyle", "pip3/pycodestyle"}
-pycodestyle.url = "pycodestyle"
+pycodestyle.arch_dependencies = {"python-pycodestyle", "python2-pycodestyle"}
+pycodestyle.url = "python-pycodestyle"
 
 
 def pyflakes(path):
     return _run_command([_python_version(path), "-m", "pyflakes", path])
 pyflakes.dependencies = {"pyflakes"}
+pyflakes.arch_dependencies = {"python2-pyflakes", "python-pyflakes"}
 pyflakes.url = "pyflakes"
 
 
@@ -404,6 +412,9 @@ def pylint(path):
     return _run_command([_python_version(path), "-m", "pylint",
                          "--errors-only", path])
 pylint.dependencies = {"pylint", "pylint3"}
+pylint.fedora_dependencies = {"pylint", "python3-pylint"}
+pylint.arch_dependencies = {"python2-pylint", "python-pylint"}
+pylint.debian_dependencies = {"pip/pylint", "pip3/pylint"}
 pylint.url = "pylint3"
 
 
@@ -445,6 +456,7 @@ def python_mccabe(path):
     status = Status.problem if max_score > 10 else Status.ok
     return status, fill3.Text(_colorize_mccabe(stdout, python_version))
 python_mccabe.dependencies = {"python-mccabe", "python3-mccabe"}
+python_mccabe.arch_dependencies = {"python2-mccabe", "python-mccabe"}
 python_mccabe.url = "python3-mccabe"
 
 
@@ -474,6 +486,9 @@ def bandit(path):
     text_without_timestamp = "".join(text.splitlines(keepends=True)[2:])
     return status, fill3.Text(text_without_timestamp)
 bandit.dependencies = {"python-bandit", "python3-bandit"}
+bandit.fedora_dependencies = {"bandit"}
+bandit.debian_dependencies = {"pip/bandit", "pip3/bandit"}
+bandit.arch_dependencies = {"bandit"}
 bandit.url = "python3-bandit"
 
 
@@ -495,6 +510,8 @@ def perldoc(path):
     return ((Status.normal, fill3.Text(stdout)) if returncode == 0
             else (Status.not_applicable, fill3.Text(stderr)))
 perldoc.dependencies = {"perl-doc"}
+perldoc.fedora_dependencies = {"perl-Pod-Perldoc"}
+perldoc.arch_dependencies = {"perl-pod-perldoc"}
 perldoc.url = "http://perldoc.perl.org/"
 
 
@@ -502,6 +519,7 @@ def perltidy(path):
     stdout, *rest = _do_command(["perltidy", "-st", path])
     return Status.normal, _syntax_highlight_using_path(stdout, path)
 perltidy.dependencies = {"perltidy"}
+perltidy.arch_dependencies = {"perl-test-perltidy"}
 perltidy.url = "http://perltidy.sourceforge.net/"
 
 
@@ -583,6 +601,7 @@ nm.url = "https://linux.die.net/man/1/nm"
 def pdf2txt(path):
     return _run_command(["pdf2txt", path], Status.normal)
 pdf2txt.dependencies = {"python-pdfminer"}
+pdf2txt.arch_dependencies = set()
 pdf2txt.url = "python-pdfminer"
 
 
@@ -605,6 +624,7 @@ tidy.url = "tidy"
 def html2text(path):
     return _run_command(["html2text", path], Status.normal)
 html2text.dependencies = {"html2text"}
+html2text.arch_dependencies = {"python-html2text"}
 html2text.url = "html2text"
 
 
@@ -625,6 +645,8 @@ def bcpp(path):
     status = Status.normal if returncode == 0 else Status.problem
     return status, _syntax_highlight_using_path(stdout, path)
 bcpp.dependencies = {"bcpp"}
+bcpp.fedora_dependencies = set()
+bcpp.arch_dependencies = set()
 
 
 def uncrustify(path):
@@ -638,12 +660,14 @@ def uncrustify(path):
     status = Status.normal if returncode == 0 else Status.problem
     return status, _syntax_highlight_using_path(stdout, path)
 uncrustify.dependencies = {"uncrustify"}
+uncrustify.debian_dependencies = set()
 uncrustify.url = "uncrustify"
 
 
 def php5_syntax(path):
     return _run_command(["php", "--syntax-check", path])
 php5_syntax.dependencies = {"php"}
+php5_syntax.debian_dependencies = {"pip3/php"}
 php5_syntax.url = "https://en.wikipedia.org/wiki/PHP"
 
 
@@ -678,6 +702,8 @@ def pil(path):
                 result.append(termstr.TermStr(text, tuple(row_style)))
     return Status.normal, fill3.Fixed(result)
 pil.dependencies = {"python3-pil"}
+pil.fedora_dependencies = {"python3-pillow"}
+pil.arch_dependencies = {"python-pillow"}
 pil.url = "python3-pil"
 
 
@@ -697,6 +723,8 @@ def pil_half(path):
                 for index in range(0, image.height, 2)])
     return Status.normal, result
 pil_half.dependencies = {"python3-pil"}
+pil_half.fedora_dependencies = {"python3-pillow"}
+pil_half.arch_dependencies = {"python-pillow"}
 pil_half.url = "python3-pil"
 
 
@@ -788,7 +816,8 @@ class Result:
         self.status = status
         self.entry.appearance_cache = None
 
-    async def run(self, log, appearance_changed_event, runner):
+    @asyncio.coroutine
+    def run(self, log, appearance_changed_event, runner):
         self.is_placeholder = False
         tool_name = tool_name_colored(self.tool, self.path)
         path = path_colored(self.path)
@@ -799,7 +828,7 @@ class Result:
             runner.pause()
         appearance_changed_event.set()
         start_time = time.time()
-        new_status = await runner.run_tool(self.path, self.tool)
+        new_status = yield from runner.run_tool(self.path, self.tool)
         Result.result.fget.evict(self)
         end_time = time.time()
         self.set_status(new_status)
@@ -866,10 +895,32 @@ def tools_all():
     return tools_
 
 
-def dependencies():
+def tool_dependencies(tool, distribution="ubuntu"):
+    if distribution == "ubuntu":
+        return tool.dependencies
+    elif distribution == "debian":
+        try:
+            return tool.debian_dependencies
+        except AttributeError:
+            return tool.dependencies
+    elif distribution == "fedora":
+        try:
+            return tool.fedora_dependencies
+        except AttributeError:
+            return tool.dependencies
+    elif distribution == "arch":
+        try:
+            return tool.arch_dependencies
+        except AttributeError:
+            return tool.dependencies
+    else:
+        raise NotImplementedError
+
+
+def dependencies(distribution="ubuntu"):
     dependencies_all = set()
     for tool in tools_all():
-        dependencies_all.update(tool.dependencies)
+        dependencies_all.update(tool_dependencies(tool, distribution))
     return dependencies_all
 
 

tools_test.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.5
+#!/usr/bin/env python3
 
 # Copyright (C) 2017 Andrew Hamilton. All rights reserved.
 # Licensed under the Artistic License 2.0.

vigil

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.5
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2015-2017 Andrew Hamilton. All rights reserved.
@@ -559,8 +559,7 @@ class Screen:
             future = worker_.job_runner(
                 self._summary, self._log, self._summary._jobs_added_event,
                 self._appearance_changed_event)
-            worker_.future = asyncio.ensure_future(
+            worker_.future = asyncio.async(future, loop=self._main_loop)
-                future, loop=self._main_loop)
         self.workers = workers
 
     def stop_workers(self):

vigil_test.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.5
+#!/usr/bin/env python3
 
 # Copyright (C) 2015-2017 Andrew Hamilton. All rights reserved.
 # Licensed under the Artistic License 2.0.

worker.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.5
+#!/usr/bin/env python3
 
 # Copyright (C) 2015-2017 Andrew Hamilton. All rights reserved.
 # Licensed under the Artistic License 2.0.
@@ -20,7 +20,8 @@ class Worker:
         self.process = None
         self.child_pgid = None
 
-    async def create_process(self):
+    @asyncio.coroutine
+    def create_process(self):
         if self.is_sandboxed:
             sandbox_fs_path = os.path.join(os.path.dirname(__file__),
                                            "sandbox_fs")
@@ -32,22 +33,24 @@ class Worker:
             *command, stdin=asyncio.subprocess.PIPE,
             stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
             preexec_fn=os.setsid)
-        self.process = await create
+        self.process = yield from create
-        pid_line = await self.process.stdout.readline()
+        pid_line = yield from self.process.stdout.readline()
         self.child_pgid = int(pid_line.strip())
         os.setpriority(os.PRIO_PGRP, self.child_pgid, 19)
 
-    async def run_tool(self, path, tool):
+    @asyncio.coroutine
+    def run_tool(self, path, tool):
         self.process.stdin.write(("%s\n%s\n" %
                                   (tool.__qualname__, path)).encode("utf-8"))
-        data = await self.process.stdout.readline()
+        data = yield from self.process.stdout.readline()
         return tools.Status(int(data))
 
-    async def job_runner(self, summary, log, jobs_added_event,
+    @asyncio.coroutine
-                         appearance_changed_event):
+    def job_runner(self, summary, log, jobs_added_event,
-        await self.create_process()
+                   appearance_changed_event):
+        yield from self.create_process()
         while True:
-            await jobs_added_event.wait()
+            yield from jobs_added_event.wait()
             while True:
                 try:
                     self.result = summary.get_closest_placeholder()
@@ -58,8 +61,7 @@ class Worker:
                         if self.is_being_tested:
                             os.kill(os.getpid(), signal.SIGINT)
                     break
-                await self.result.run(log, appearance_changed_event,
+                yield from self.result.run(log, appearance_changed_event, self)
-                                      self)
                 summary.completed_total += 1
             jobs_added_event.clear()
 

worker_test.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.5
+#!/usr/bin/env python3
 
 # Copyright (C) 2017 Andrew Hamilton. All rights reserved.
 # Licensed under the Artistic License 2.0.