From 72b5f3750e07ee293207b444c87d4217f66d5b41 Mon Sep 17 00:00:00 2001 From: Andrew Hamilton Date: Fri, 23 Jun 2017 23:23:32 +0100 Subject: [PATCH] Dropped the sandbox. - It wasn't worth the complexity. It was only useful when running the python profile tool, which executes python scripts. But that tool isn't very useful when run against normal scripts. It has been removed too. - The sandbox also wasn't working inside snaps, appimages, systemd-nspawn or chroots. --- README.md | 2 +- in-directory | 9 ---- sandbox_fs | 9 ---- sandbox_fs.py | 103 --------------------------------------------- sandbox_fs_test.py | 44 ------------------- tools.py | 13 +----- vigil | 26 ++++-------- vigil_test.py | 2 +- worker.py | 12 +----- worker_test.py | 10 +---- 10 files changed, 16 insertions(+), 214 deletions(-) delete mode 100755 in-directory delete mode 100755 sandbox_fs delete mode 100755 sandbox_fs.py delete mode 100755 sandbox_fs_test.py diff --git a/README.md b/README.md index c0b8a3d..3b56246 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ e.g. After cloning do: Extensions | Tools ---------- | ----- -.py | [python_syntax](https://en.wikipedia.org/wiki/Python_syntax_and_semantics) • [python_unittests](https://docs.python.org/3/library/unittest.html) • [pydoc](https://docs.python.org/3/library/pydoc.html) • [mypy](http://www.mypy-lang.org/) • [python_coverage](http://nedbatchelder.com/code/coverage/) • [python_profile](https://docs.python.org/3/library/profile.html) • [pycodestyle](https://pypi.python.org/pypi/pycodestyle) • [pyflakes](https://launchpad.net/pyflakes) • [pylint](http://www.pylint.org/) • [python_gut](https://github.com/ahamilton/vigil/blob/master/gut.py) • [python_modulefinder](https://docs.python.org/3/library/modulefinder.html) • [python_mccabe](https://github.com/flintwork/mccabe) • [bandit](https://wiki.openstack.org/wiki/Security/Projects/Bandit) +.py | [python_syntax](https://en.wikipedia.org/wiki/Python_syntax_and_semantics) • [python_unittests](https://docs.python.org/3/library/unittest.html) • [pydoc](https://docs.python.org/3/library/pydoc.html) • [mypy](http://www.mypy-lang.org/) • [python_coverage](http://nedbatchelder.com/code/coverage/) • [pycodestyle](https://pypi.python.org/pypi/pycodestyle) • [pyflakes](https://launchpad.net/pyflakes) • [pylint](http://www.pylint.org/) • [python_gut](https://github.com/ahamilton/vigil/blob/master/gut.py) • [python_modulefinder](https://docs.python.org/3/library/modulefinder.html) • [python_mccabe](https://github.com/flintwork/mccabe) • [bandit](https://wiki.openstack.org/wiki/Security/Projects/Bandit) .pyc | [disassemble_pyc](https://docs.python.org/3/library/dis.html) .pl .pm .t | [perl_syntax](https://en.wikipedia.org/wiki/Perl) • [perldoc](http://perldoc.perl.org/) • [perltidy](http://perltidy.sourceforge.net/) .pod .pod6 | [perldoc](http://perldoc.perl.org/) diff --git a/in-directory b/in-directory deleted file mode 100755 index b8629f4..0000000 --- a/in-directory +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - - -set -e - - -cd $1 -shift -exec $@ diff --git a/sandbox_fs b/sandbox_fs deleted file mode 100755 index c5a7c3b..0000000 --- a/sandbox_fs +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - - -set -e - - -SCRIPT_PATH=$(realpath $0) -SCRIPT_DIR=$(dirname $SCRIPT_PATH) -exec unshare --mount --map-root-user "$SCRIPT_DIR/sandbox_fs.py" "$@" diff --git a/sandbox_fs.py b/sandbox_fs.py deleted file mode 100755 index ae87da1..0000000 --- a/sandbox_fs.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (C) 2017 Andrew Hamilton. All rights reserved. -# Licensed under the Artistic License 2.0. - -import os -import subprocess -import sys -import tempfile - - -class OverlayfsMount(): - - def __init__(self, lower_dir, mount_point): - self.lower_dir = lower_dir - self.mount_point = mount_point - - def __repr__(self): - return "" % (self.mount_point, - self.lower_dir) - - def __enter__(self): - self.upper_dir = tempfile.TemporaryDirectory() - self.work_dir = tempfile.TemporaryDirectory() - option_string = ("lowerdir=%s,upperdir=%s,workdir=%s" % - (self.lower_dir, self.upper_dir.name, - self.work_dir.name)) - subprocess.check_call(["mount", "-t", "overlay", "-o", - option_string, "overlay", self.mount_point], - stderr=subprocess.PIPE) - return self - - def __exit__(self, exc_type, exc_value, traceback): - subprocess.check_call(["umount", "--lazy", self.mount_point]) - - -_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -_IN_DIRECTORY_SCRIPT = os.path.join(_SCRIPT_DIR, "in-directory") - - -def _in_directory(directory_path, command): - return [_IN_DIRECTORY_SCRIPT, directory_path] + command - - -def _parse_proc_mounts(): - with open("/proc/mounts") as file_: - for line in file_: - yield line.split() - - -def _find_mounts(): - all_mounts = set(part[1] for part in _parse_proc_mounts()) - mount_points = {"/", "/usr", "/bin", "/etc", "/lib", "/dev", "/home", - "/boot", "/opt", "/run", "/root", "/var", "/vigil"} - return all_mounts.intersection(mount_points) - - -class SandboxFs: - - def __init__(self, holes=None): - self.holes = [] if holes is None else holes - self.holes += ["/dev/null"] - for hole in self.holes: - if not hole.startswith("/"): - raise ValueError("Holes must be absolute paths: %r" % hole) - self.temp_dir = tempfile.TemporaryDirectory() - self.mount_point = self.temp_dir.name - self.overlay_mounts = [] - - def __repr__(self): - return ("" % - (self.temp_dir.name, len(self.overlay_mounts))) - - def __enter__(self): - self.overlay_mounts = [ - OverlayfsMount(mount_point, - self.mount_point + mount_point).__enter__() - for mount_point in sorted(_find_mounts())] - for hole in self.holes: - subprocess.check_call(["mount", "--bind", hole, - self.mount_point + hole]) - return self - - def __exit__(self, exc_type, exc_value, traceback): - for hole in reversed(self.holes): - subprocess.check_call(["umount", self.mount_point + hole]) - for mount in reversed(self.overlay_mounts): - mount.__exit__(None, None, None) - self.overlay_mounts = [] - - def command(self, command, env=None): - return (["chroot", self.mount_point] + - _in_directory(os.getcwd(), command)) - - -if __name__ == "__main__": - try: - divider_index = sys.argv.index("--") - holes, command = sys.argv[1:divider_index], sys.argv[divider_index+1:] - except ValueError: - holes, command = None, sys.argv[1:] - with SandboxFs(holes) as sandbox: - subprocess.check_call(sandbox.command(command)) diff --git a/sandbox_fs_test.py b/sandbox_fs_test.py deleted file mode 100755 index 89725af..0000000 --- a/sandbox_fs_test.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (C) 2017 Andrew Hamilton. All rights reserved. -# Licensed under the Artistic License 2.0. - -import os -import sys -import subprocess -import tempfile -import unittest - - -tempfile.tempdir = os.getcwd() # This tests fails when using /tmp. -VIGIL_ROOT = os.path.dirname(__file__) - - -def _get_test_paths(temp_dir): - a_dir = os.path.join(temp_dir, "a") - foo_path = os.path.join(a_dir, "foo") - bar_path = os.path.join(temp_dir, "bar") - return a_dir, foo_path, bar_path - - -class SandboxFilesystemTestCase(unittest.TestCase): - - def test_sandbox(self): - with tempfile.TemporaryDirectory() as temp_dir: - a_dir, foo_path, bar_path = _get_test_paths(temp_dir) - os.mkdir(a_dir) - sandbox_fs_path = os.path.join(VIGIL_ROOT, "sandbox_fs") - subprocess.check_call([sandbox_fs_path, a_dir, "--", __file__, - temp_dir]) - self.assertTrue(os.path.exists(foo_path)) - self.assertFalse(os.path.exists(bar_path)) - - -if __name__ == "__main__": - if len(sys.argv) > 1: - temp_dir = sys.argv[1] - a_dir, foo_path, bar_path = _get_test_paths(temp_dir) - subprocess.check_call(["touch", foo_path]) - subprocess.check_call(["touch", bar_path]) - else: - unittest.main() diff --git a/tools.py b/tools.py index b3979a1..f484290 100644 --- a/tools.py +++ b/tools.py @@ -393,14 +393,6 @@ def python_coverage(path): "No corresponding test file: " + os.path.normpath(test_path)) -@deps(deps={"python", "python3"}, gentoo_deps={"python"}, - url="https://docs.python.org/3/library/profile.html") -def python_profile(path): - stdout, *rest = _do_command([_python_version(path), "-m", "cProfile", - "--sort=cumulative", path], timeout=TIMEOUT) - return Status.normal, fill3.Text(stdout) - - @deps(deps={"python-pycodestyle", "python3-pycodestyle"}, fedora_deps={"python2-pycodestyle", "python3-pycodestyle"}, debian_deps={"pip/pycodestyle", "pip3/pycodestyle"}, @@ -847,9 +839,8 @@ IMAGE_EXTENSIONS = ["png", "jpg", "gif", "bmp", "ppm", "tiff", "tga"] TOOLS_FOR_EXTENSIONS = \ [ (["py"], [python_syntax, python_unittests, pydoc, mypy, - python_coverage, python_profile, pycodestyle, pyflakes, - pylint, python_gut, python_modulefinder, python_mccabe, - bandit]), + python_coverage, pycodestyle, pyflakes, pylint, python_gut, + python_modulefinder, python_mccabe, bandit]), (["pyc"], [disassemble_pyc]), (["pl", "pm", "t"], [perl_syntax, perldoc, perltidy]), # (["p6", "pm6"], [perl6_syntax, perldoc]), diff --git a/vigil b/vigil index 4bcb632..9739c9f 100755 --- a/vigil +++ b/vigil @@ -51,8 +51,6 @@ Example: Options: -h, --help Show this screen and exit. - -s on|off, --sandbox=on|off Use a sandbox to prevent changes to the - filesystem. The sandbox is on by default. -w COUNT, --workers=COUNT The number of processes working in parallel. By default it is the number of cpus minus 1. -e "COMMAND", --editor="COMMAND" The command used to start the editor, in @@ -568,11 +566,10 @@ class Screen: self._make_widgets() self._key_map = make_key_map(Screen._KEY_DATA) - def make_workers(self, worker_count, is_sandboxed, is_being_tested): + def make_workers(self, worker_count, is_being_tested): workers = [] for index in range(worker_count): - worker_ = worker.Worker(is_sandboxed, self._is_paused, - is_being_tested) + worker_ = worker.Worker(self._is_paused, is_being_tested) workers.append(worker_) future = worker_.job_runner( self._summary, self._log, self._summary._jobs_added_event, @@ -947,8 +944,8 @@ def save_state(pickle_path, summary, screen, log): tools.dump_pickle_safe(screen, pickle_path, open=open_compressed) -def main(root_path, loop, worker_count=None, is_sandboxed=True, - editor_command=None, theme=None, is_being_tested=False): +def main(root_path, loop, worker_count=None, editor_command=None, theme=None, + is_being_tested=False): if worker_count is None: worker_count = max(multiprocessing.cpu_count() - 1, 1) if theme is None: @@ -974,7 +971,7 @@ def main(root_path, loop, worker_count=None, is_sandboxed=True, root_path, loop, on_filesystem_change, is_path_excluded) try: log.log_message("Starting workers (%s) ..." % worker_count) - screen.make_workers(worker_count, is_sandboxed, is_being_tested) + screen.make_workers(worker_count, is_being_tested) def exit_loop(): log.log_command("Exiting...") @@ -1034,10 +1031,6 @@ def check_arguments(): if not os.path.isdir(root_path): print("File is not a directory:", root_path) sys.exit(1) - if arguments["--sandbox"] not in ["on", "off", None]: - print("--sandbox argument must be 'on' or 'off'") - sys.exit(1) - is_sandboxed = arguments["--sandbox"] in ["on", None] if arguments["--theme"] is not None: themes = list(pygments.styles.get_all_styles()) if arguments["--theme"] not in themes: @@ -1045,17 +1038,14 @@ def check_arguments(): sys.exit(1) editor_command = arguments["--editor"] or os.environ.get("EDITOR", None)\ or os.environ.get("VISUAL", None) - return root_path, worker_count, is_sandboxed, editor_command, \ - arguments["--theme"] + return root_path, worker_count, editor_command, arguments["--theme"] if __name__ == "__main__": - root_path, worker_count, is_sandboxed, editor_command, theme = \ - check_arguments() + root_path, worker_count, editor_command, theme = check_arguments() with terminal.console_title("vigil: " + os.path.basename(root_path)): manage_cache(root_path) with chdir(root_path): # FIX: Don't change directory if possible. loop = asyncio.get_event_loop() - main(root_path, loop, worker_count, is_sandboxed, editor_command, - theme) + main(root_path, loop, worker_count, editor_command, theme) os._exit(0) diff --git a/vigil_test.py b/vigil_test.py index d71a398..88fb5e3 100755 --- a/vigil_test.py +++ b/vigil_test.py @@ -223,7 +223,7 @@ class MainTestCase(unittest.TestCase): with vigil.chdir(root_path): with contextlib.redirect_stdout(io.StringIO()): vigil.main(root_path, loop, worker_count=2, - is_sandboxed=True, is_being_tested=True) + is_being_tested=True) for file_name in ["summary.pickle", "creation_time", "log", "foo-metadata", "foo-contents"]: self.assertTrue(os.path.exists(".vigil/" + file_name)) diff --git a/worker.py b/worker.py index 24d6593..3af2f16 100755 --- a/worker.py +++ b/worker.py @@ -12,8 +12,7 @@ import tools class Worker: - def __init__(self, is_sandboxed, is_already_paused, is_being_tested): - self.is_sandboxed = is_sandboxed + def __init__(self, is_already_paused, is_being_tested): self.is_already_paused = is_already_paused self.is_being_tested = is_being_tested self.result = None @@ -22,15 +21,8 @@ class Worker: @asyncio.coroutine def create_process(self): - if self.is_sandboxed: - sandbox_fs_path = os.path.join(os.path.dirname(__file__), - "sandbox_fs") - cache_path = os.path.join(os.getcwd(), tools.CACHE_PATH) - command = [sandbox_fs_path, cache_path, "--", __file__] - else: - command = [__file__] create = asyncio.create_subprocess_exec( - *command, stdin=asyncio.subprocess.PIPE, + *[__file__], stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, preexec_fn=os.setsid) self.process = yield from create diff --git a/worker_test.py b/worker_test.py index 4fe5102..8ec7638 100755 --- a/worker_test.py +++ b/worker_test.py @@ -27,9 +27,9 @@ class WorkerTestCase(unittest.TestCase): shutil.rmtree(self.temp_dir) os.chdir(self.original_working_dir) - def _test_worker(self, is_sandboxed): + def test_run_job(self): loop = asyncio.get_event_loop() - worker_ = worker.Worker(is_sandboxed, False, False) + worker_ = worker.Worker(False, False) loop.run_until_complete(worker_.create_process()) future = worker_.run_tool("foo", tools.metadata) status = loop.run_until_complete(future) @@ -37,12 +37,6 @@ class WorkerTestCase(unittest.TestCase): result_path = os.path.join(tools.CACHE_PATH, "foo-metadata") self.assertTrue(os.path.exists(result_path)) - def test_run_job_without_sandbox(self): - self._test_worker(False) - - def test_run_job_with_sandbox(self): - self._test_worker(True) - if __name__ == "__main__": unittest.main()