diff --git a/README.md b/README.md index c0b8a3d..3b56246 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ e.g. After cloning do: Extensions | Tools ---------- | ----- -.py | [python_syntax](https://en.wikipedia.org/wiki/Python_syntax_and_semantics) • [python_unittests](https://docs.python.org/3/library/unittest.html) • [pydoc](https://docs.python.org/3/library/pydoc.html) • [mypy](http://www.mypy-lang.org/) • [python_coverage](http://nedbatchelder.com/code/coverage/) • [python_profile](https://docs.python.org/3/library/profile.html) • [pycodestyle](https://pypi.python.org/pypi/pycodestyle) • [pyflakes](https://launchpad.net/pyflakes) • [pylint](http://www.pylint.org/) • [python_gut](https://github.com/ahamilton/vigil/blob/master/gut.py) • [python_modulefinder](https://docs.python.org/3/library/modulefinder.html) • [python_mccabe](https://github.com/flintwork/mccabe) • [bandit](https://wiki.openstack.org/wiki/Security/Projects/Bandit) +.py | [python_syntax](https://en.wikipedia.org/wiki/Python_syntax_and_semantics) • [python_unittests](https://docs.python.org/3/library/unittest.html) • [pydoc](https://docs.python.org/3/library/pydoc.html) • [mypy](http://www.mypy-lang.org/) • [python_coverage](http://nedbatchelder.com/code/coverage/) • [pycodestyle](https://pypi.python.org/pypi/pycodestyle) • [pyflakes](https://launchpad.net/pyflakes) • [pylint](http://www.pylint.org/) • [python_gut](https://github.com/ahamilton/vigil/blob/master/gut.py) • [python_modulefinder](https://docs.python.org/3/library/modulefinder.html) • [python_mccabe](https://github.com/flintwork/mccabe) • [bandit](https://wiki.openstack.org/wiki/Security/Projects/Bandit) .pyc | [disassemble_pyc](https://docs.python.org/3/library/dis.html) .pl .pm .t | [perl_syntax](https://en.wikipedia.org/wiki/Perl) • [perldoc](http://perldoc.perl.org/) • [perltidy](http://perltidy.sourceforge.net/) .pod .pod6 | [perldoc](http://perldoc.perl.org/) diff --git a/in-directory b/in-directory deleted file mode 100755 index b8629f4..0000000 --- a/in-directory +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - - -set -e - - -cd $1 -shift -exec $@ diff --git a/sandbox_fs b/sandbox_fs deleted file mode 100755 index c5a7c3b..0000000 --- a/sandbox_fs +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - - -set -e - - -SCRIPT_PATH=$(realpath $0) -SCRIPT_DIR=$(dirname $SCRIPT_PATH) -exec unshare --mount --map-root-user "$SCRIPT_DIR/sandbox_fs.py" "$@" diff --git a/sandbox_fs.py b/sandbox_fs.py deleted file mode 100755 index ae87da1..0000000 --- a/sandbox_fs.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (C) 2017 Andrew Hamilton. All rights reserved. -# Licensed under the Artistic License 2.0. - -import os -import subprocess -import sys -import tempfile - - -class OverlayfsMount(): - - def __init__(self, lower_dir, mount_point): - self.lower_dir = lower_dir - self.mount_point = mount_point - - def __repr__(self): - return "" % (self.mount_point, - self.lower_dir) - - def __enter__(self): - self.upper_dir = tempfile.TemporaryDirectory() - self.work_dir = tempfile.TemporaryDirectory() - option_string = ("lowerdir=%s,upperdir=%s,workdir=%s" % - (self.lower_dir, self.upper_dir.name, - self.work_dir.name)) - subprocess.check_call(["mount", "-t", "overlay", "-o", - option_string, "overlay", self.mount_point], - stderr=subprocess.PIPE) - return self - - def __exit__(self, exc_type, exc_value, traceback): - subprocess.check_call(["umount", "--lazy", self.mount_point]) - - -_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -_IN_DIRECTORY_SCRIPT = os.path.join(_SCRIPT_DIR, "in-directory") - - -def _in_directory(directory_path, command): - return [_IN_DIRECTORY_SCRIPT, directory_path] + command - - -def _parse_proc_mounts(): - with open("/proc/mounts") as file_: - for line in file_: - yield line.split() - - -def _find_mounts(): - all_mounts = set(part[1] for part in _parse_proc_mounts()) - mount_points = {"/", "/usr", "/bin", "/etc", "/lib", "/dev", "/home", - "/boot", "/opt", "/run", "/root", "/var", "/vigil"} - return all_mounts.intersection(mount_points) - - -class SandboxFs: - - def __init__(self, holes=None): - self.holes = [] if holes is None else holes - self.holes += ["/dev/null"] - for hole in self.holes: - if not hole.startswith("/"): - raise ValueError("Holes must be absolute paths: %r" % hole) - self.temp_dir = tempfile.TemporaryDirectory() - self.mount_point = self.temp_dir.name - self.overlay_mounts = [] - - def __repr__(self): - return ("" % - (self.temp_dir.name, len(self.overlay_mounts))) - - def __enter__(self): - self.overlay_mounts = [ - OverlayfsMount(mount_point, - self.mount_point + mount_point).__enter__() - for mount_point in sorted(_find_mounts())] - for hole in self.holes: - subprocess.check_call(["mount", "--bind", hole, - self.mount_point + hole]) - return self - - def __exit__(self, exc_type, exc_value, traceback): - for hole in reversed(self.holes): - subprocess.check_call(["umount", self.mount_point + hole]) - for mount in reversed(self.overlay_mounts): - mount.__exit__(None, None, None) - self.overlay_mounts = [] - - def command(self, command, env=None): - return (["chroot", self.mount_point] + - _in_directory(os.getcwd(), command)) - - -if __name__ == "__main__": - try: - divider_index = sys.argv.index("--") - holes, command = sys.argv[1:divider_index], sys.argv[divider_index+1:] - except ValueError: - holes, command = None, sys.argv[1:] - with SandboxFs(holes) as sandbox: - subprocess.check_call(sandbox.command(command)) diff --git a/sandbox_fs_test.py b/sandbox_fs_test.py deleted file mode 100755 index 89725af..0000000 --- a/sandbox_fs_test.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (C) 2017 Andrew Hamilton. All rights reserved. -# Licensed under the Artistic License 2.0. - -import os -import sys -import subprocess -import tempfile -import unittest - - -tempfile.tempdir = os.getcwd() # This tests fails when using /tmp. -VIGIL_ROOT = os.path.dirname(__file__) - - -def _get_test_paths(temp_dir): - a_dir = os.path.join(temp_dir, "a") - foo_path = os.path.join(a_dir, "foo") - bar_path = os.path.join(temp_dir, "bar") - return a_dir, foo_path, bar_path - - -class SandboxFilesystemTestCase(unittest.TestCase): - - def test_sandbox(self): - with tempfile.TemporaryDirectory() as temp_dir: - a_dir, foo_path, bar_path = _get_test_paths(temp_dir) - os.mkdir(a_dir) - sandbox_fs_path = os.path.join(VIGIL_ROOT, "sandbox_fs") - subprocess.check_call([sandbox_fs_path, a_dir, "--", __file__, - temp_dir]) - self.assertTrue(os.path.exists(foo_path)) - self.assertFalse(os.path.exists(bar_path)) - - -if __name__ == "__main__": - if len(sys.argv) > 1: - temp_dir = sys.argv[1] - a_dir, foo_path, bar_path = _get_test_paths(temp_dir) - subprocess.check_call(["touch", foo_path]) - subprocess.check_call(["touch", bar_path]) - else: - unittest.main() diff --git a/tools.py b/tools.py index b3979a1..f484290 100644 --- a/tools.py +++ b/tools.py @@ -393,14 +393,6 @@ def python_coverage(path): "No corresponding test file: " + os.path.normpath(test_path)) -@deps(deps={"python", "python3"}, gentoo_deps={"python"}, - url="https://docs.python.org/3/library/profile.html") -def python_profile(path): - stdout, *rest = _do_command([_python_version(path), "-m", "cProfile", - "--sort=cumulative", path], timeout=TIMEOUT) - return Status.normal, fill3.Text(stdout) - - @deps(deps={"python-pycodestyle", "python3-pycodestyle"}, fedora_deps={"python2-pycodestyle", "python3-pycodestyle"}, debian_deps={"pip/pycodestyle", "pip3/pycodestyle"}, @@ -847,9 +839,8 @@ IMAGE_EXTENSIONS = ["png", "jpg", "gif", "bmp", "ppm", "tiff", "tga"] TOOLS_FOR_EXTENSIONS = \ [ (["py"], [python_syntax, python_unittests, pydoc, mypy, - python_coverage, python_profile, pycodestyle, pyflakes, - pylint, python_gut, python_modulefinder, python_mccabe, - bandit]), + python_coverage, pycodestyle, pyflakes, pylint, python_gut, + python_modulefinder, python_mccabe, bandit]), (["pyc"], [disassemble_pyc]), (["pl", "pm", "t"], [perl_syntax, perldoc, perltidy]), # (["p6", "pm6"], [perl6_syntax, perldoc]), diff --git a/vigil b/vigil index 4bcb632..9739c9f 100755 --- a/vigil +++ b/vigil @@ -51,8 +51,6 @@ Example: Options: -h, --help Show this screen and exit. - -s on|off, --sandbox=on|off Use a sandbox to prevent changes to the - filesystem. The sandbox is on by default. -w COUNT, --workers=COUNT The number of processes working in parallel. By default it is the number of cpus minus 1. -e "COMMAND", --editor="COMMAND" The command used to start the editor, in @@ -568,11 +566,10 @@ class Screen: self._make_widgets() self._key_map = make_key_map(Screen._KEY_DATA) - def make_workers(self, worker_count, is_sandboxed, is_being_tested): + def make_workers(self, worker_count, is_being_tested): workers = [] for index in range(worker_count): - worker_ = worker.Worker(is_sandboxed, self._is_paused, - is_being_tested) + worker_ = worker.Worker(self._is_paused, is_being_tested) workers.append(worker_) future = worker_.job_runner( self._summary, self._log, self._summary._jobs_added_event, @@ -947,8 +944,8 @@ def save_state(pickle_path, summary, screen, log): tools.dump_pickle_safe(screen, pickle_path, open=open_compressed) -def main(root_path, loop, worker_count=None, is_sandboxed=True, - editor_command=None, theme=None, is_being_tested=False): +def main(root_path, loop, worker_count=None, editor_command=None, theme=None, + is_being_tested=False): if worker_count is None: worker_count = max(multiprocessing.cpu_count() - 1, 1) if theme is None: @@ -974,7 +971,7 @@ def main(root_path, loop, worker_count=None, is_sandboxed=True, root_path, loop, on_filesystem_change, is_path_excluded) try: log.log_message("Starting workers (%s) ..." % worker_count) - screen.make_workers(worker_count, is_sandboxed, is_being_tested) + screen.make_workers(worker_count, is_being_tested) def exit_loop(): log.log_command("Exiting...") @@ -1034,10 +1031,6 @@ def check_arguments(): if not os.path.isdir(root_path): print("File is not a directory:", root_path) sys.exit(1) - if arguments["--sandbox"] not in ["on", "off", None]: - print("--sandbox argument must be 'on' or 'off'") - sys.exit(1) - is_sandboxed = arguments["--sandbox"] in ["on", None] if arguments["--theme"] is not None: themes = list(pygments.styles.get_all_styles()) if arguments["--theme"] not in themes: @@ -1045,17 +1038,14 @@ def check_arguments(): sys.exit(1) editor_command = arguments["--editor"] or os.environ.get("EDITOR", None)\ or os.environ.get("VISUAL", None) - return root_path, worker_count, is_sandboxed, editor_command, \ - arguments["--theme"] + return root_path, worker_count, editor_command, arguments["--theme"] if __name__ == "__main__": - root_path, worker_count, is_sandboxed, editor_command, theme = \ - check_arguments() + root_path, worker_count, editor_command, theme = check_arguments() with terminal.console_title("vigil: " + os.path.basename(root_path)): manage_cache(root_path) with chdir(root_path): # FIX: Don't change directory if possible. loop = asyncio.get_event_loop() - main(root_path, loop, worker_count, is_sandboxed, editor_command, - theme) + main(root_path, loop, worker_count, editor_command, theme) os._exit(0) diff --git a/vigil_test.py b/vigil_test.py index d71a398..88fb5e3 100755 --- a/vigil_test.py +++ b/vigil_test.py @@ -223,7 +223,7 @@ class MainTestCase(unittest.TestCase): with vigil.chdir(root_path): with contextlib.redirect_stdout(io.StringIO()): vigil.main(root_path, loop, worker_count=2, - is_sandboxed=True, is_being_tested=True) + is_being_tested=True) for file_name in ["summary.pickle", "creation_time", "log", "foo-metadata", "foo-contents"]: self.assertTrue(os.path.exists(".vigil/" + file_name)) diff --git a/worker.py b/worker.py index 24d6593..3af2f16 100755 --- a/worker.py +++ b/worker.py @@ -12,8 +12,7 @@ import tools class Worker: - def __init__(self, is_sandboxed, is_already_paused, is_being_tested): - self.is_sandboxed = is_sandboxed + def __init__(self, is_already_paused, is_being_tested): self.is_already_paused = is_already_paused self.is_being_tested = is_being_tested self.result = None @@ -22,15 +21,8 @@ class Worker: @asyncio.coroutine def create_process(self): - if self.is_sandboxed: - sandbox_fs_path = os.path.join(os.path.dirname(__file__), - "sandbox_fs") - cache_path = os.path.join(os.getcwd(), tools.CACHE_PATH) - command = [sandbox_fs_path, cache_path, "--", __file__] - else: - command = [__file__] create = asyncio.create_subprocess_exec( - *command, stdin=asyncio.subprocess.PIPE, + *[__file__], stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, preexec_fn=os.setsid) self.process = yield from create diff --git a/worker_test.py b/worker_test.py index 4fe5102..8ec7638 100755 --- a/worker_test.py +++ b/worker_test.py @@ -27,9 +27,9 @@ class WorkerTestCase(unittest.TestCase): shutil.rmtree(self.temp_dir) os.chdir(self.original_working_dir) - def _test_worker(self, is_sandboxed): + def test_run_job(self): loop = asyncio.get_event_loop() - worker_ = worker.Worker(is_sandboxed, False, False) + worker_ = worker.Worker(False, False) loop.run_until_complete(worker_.create_process()) future = worker_.run_tool("foo", tools.metadata) status = loop.run_until_complete(future) @@ -37,12 +37,6 @@ class WorkerTestCase(unittest.TestCase): result_path = os.path.join(tools.CACHE_PATH, "foo-metadata") self.assertTrue(os.path.exists(result_path)) - def test_run_job_without_sandbox(self): - self._test_worker(False) - - def test_run_job_with_sandbox(self): - self._test_worker(True) - if __name__ == "__main__": unittest.main()