ahamilton/eris
1
0
Fork 0
Code Releases Activity

Dropped the sandbox.

Browse source 
- It wasn't worth the complexity. It was only useful when running the
  python profile tool, which executes python scripts. But that tool
  isn't very useful when run against normal scripts. It has been
  removed too.
- The sandbox also wasn't working inside snaps, appimages,
  systemd-nspawn or chroots.
This commit is contained in:
Andrew Hamilton 2017-06-23 23:23:32 +01:00
parent 814dbfee5e
commit 72b5f3750e
10 changed files with 16 additions and 214 deletions
Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -26,7 +26,7 @@ e.g. After cloning do:
Extensions | Tools
---------- | -----
.py | [python_syntax](https://en.wikipedia.org/wiki/Python_syntax_and_semantics) • [python_unittests](https://docs.python.org/3/library/unittest.html) • [pydoc](https://docs.python.org/3/library/pydoc.html) • [mypy](http://www.mypy-lang.org/) • [python_coverage](http://nedbatchelder.com/code/coverage/) • [python_profile](https://docs.python.org/3/library/profile.html) • [pycodestyle](https://pypi.python.org/pypi/pycodestyle) • [pyflakes](https://launchpad.net/pyflakes) • [pylint](http://www.pylint.org/) • [python_gut](https://github.com/ahamilton/vigil/blob/master/gut.py) • [python_modulefinder](https://docs.python.org/3/library/modulefinder.html) • [python_mccabe](https://github.com/flintwork/mccabe) • [bandit](https://wiki.openstack.org/wiki/Security/Projects/Bandit)
.py | [python_syntax](https://en.wikipedia.org/wiki/Python_syntax_and_semantics) • [python_unittests](https://docs.python.org/3/library/unittest.html) • [pydoc](https://docs.python.org/3/library/pydoc.html) • [mypy](http://www.mypy-lang.org/) • [python_coverage](http://nedbatchelder.com/code/coverage/) • [pycodestyle](https://pypi.python.org/pypi/pycodestyle) • [pyflakes](https://launchpad.net/pyflakes) • [pylint](http://www.pylint.org/) • [python_gut](https://github.com/ahamilton/vigil/blob/master/gut.py) • [python_modulefinder](https://docs.python.org/3/library/modulefinder.html) • [python_mccabe](https://github.com/flintwork/mccabe) • [bandit](https://wiki.openstack.org/wiki/Security/Projects/Bandit)
.pyc | [disassemble_pyc](https://docs.python.org/3/library/dis.html)
.pl .pm .t | [perl_syntax](https://en.wikipedia.org/wiki/Perl) • [perldoc](http://perldoc.perl.org/) • [perltidy](http://perltidy.sourceforge.net/)
.pod .pod6 | [perldoc](http://perldoc.perl.org/)

9
in-directory
View file

@ -1,9 +0,0 @@
#!/bin/bash
set -e
cd $1
shift
exec $@

9
sandbox_fs
View file

@ -1,9 +0,0 @@
#!/bin/bash
set -e
SCRIPT_PATH=$(realpath $0)
SCRIPT_DIR=$(dirname $SCRIPT_PATH)
exec unshare --mount --map-root-user "$SCRIPT_DIR/sandbox_fs.py" "$@"

103
sandbox_fs.py
View file

@ -1,103 +0,0 @@
#!/usr/bin/env python3
# Copyright (C) 2017 Andrew Hamilton. All rights reserved.
# Licensed under the Artistic License 2.0.
import os
import subprocess
import sys
import tempfile
class OverlayfsMount():
def __init__(self, lower_dir, mount_point):
self.lower_dir = lower_dir
self.mount_point = mount_point
def __repr__(self):
return "<OverlayfsMount:%r over %r>" % (self.mount_point,
self.lower_dir)
def __enter__(self):
self.upper_dir = tempfile.TemporaryDirectory()
self.work_dir = tempfile.TemporaryDirectory()
option_string = ("lowerdir=%s,upperdir=%s,workdir=%s" %
(self.lower_dir, self.upper_dir.name,
self.work_dir.name))
subprocess.check_call(["mount", "-t", "overlay", "-o",
option_string, "overlay", self.mount_point],
stderr=subprocess.PIPE)
return self
def __exit__(self, exc_type, exc_value, traceback):
subprocess.check_call(["umount", "--lazy", self.mount_point])
_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
_IN_DIRECTORY_SCRIPT = os.path.join(_SCRIPT_DIR, "in-directory")
def _in_directory(directory_path, command):
return [_IN_DIRECTORY_SCRIPT, directory_path] + command
def _parse_proc_mounts():
with open("/proc/mounts") as file_:
for line in file_:
yield line.split()
def _find_mounts():
all_mounts = set(part[1] for part in _parse_proc_mounts())
mount_points = {"/", "/usr", "/bin", "/etc", "/lib", "/dev", "/home",
"/boot", "/opt", "/run", "/root", "/var", "/vigil"}
return all_mounts.intersection(mount_points)
class SandboxFs:
def __init__(self, holes=None):
self.holes = [] if holes is None else holes
self.holes += ["/dev/null"]
for hole in self.holes:
if not hole.startswith("/"):
raise ValueError("Holes must be absolute paths: %r" % hole)
self.temp_dir = tempfile.TemporaryDirectory()
self.mount_point = self.temp_dir.name
self.overlay_mounts = []
def __repr__(self):
return ("<SandboxFs:%r mounts:%r>" %
(self.temp_dir.name, len(self.overlay_mounts)))
def __enter__(self):
self.overlay_mounts = [
OverlayfsMount(mount_point,
self.mount_point + mount_point).__enter__()
for mount_point in sorted(_find_mounts())]
for hole in self.holes:
subprocess.check_call(["mount", "--bind", hole,
self.mount_point + hole])
return self
def __exit__(self, exc_type, exc_value, traceback):
for hole in reversed(self.holes):
subprocess.check_call(["umount", self.mount_point + hole])
for mount in reversed(self.overlay_mounts):
mount.__exit__(None, None, None)
self.overlay_mounts = []
def command(self, command, env=None):
return (["chroot", self.mount_point] +
_in_directory(os.getcwd(), command))
if __name__ == "__main__":
try:
divider_index = sys.argv.index("--")
holes, command = sys.argv[1:divider_index], sys.argv[divider_index+1:]
except ValueError:
holes, command = None, sys.argv[1:]
with SandboxFs(holes) as sandbox:
subprocess.check_call(sandbox.command(command))

44
sandbox_fs_test.py
View file

@ -1,44 +0,0 @@
#!/usr/bin/env python3
# Copyright (C) 2017 Andrew Hamilton. All rights reserved.
# Licensed under the Artistic License 2.0.
import os
import sys
import subprocess
import tempfile
import unittest
tempfile.tempdir = os.getcwd() # This tests fails when using /tmp.
VIGIL_ROOT = os.path.dirname(__file__)
def _get_test_paths(temp_dir):
a_dir = os.path.join(temp_dir, "a")
foo_path = os.path.join(a_dir, "foo")
bar_path = os.path.join(temp_dir, "bar")
return a_dir, foo_path, bar_path
class SandboxFilesystemTestCase(unittest.TestCase):
def test_sandbox(self):
with tempfile.TemporaryDirectory() as temp_dir:
a_dir, foo_path, bar_path = _get_test_paths(temp_dir)
os.mkdir(a_dir)
sandbox_fs_path = os.path.join(VIGIL_ROOT, "sandbox_fs")
subprocess.check_call([sandbox_fs_path, a_dir, "--", __file__,
temp_dir])
self.assertTrue(os.path.exists(foo_path))
self.assertFalse(os.path.exists(bar_path))
if __name__ == "__main__":
if len(sys.argv) > 1:
temp_dir = sys.argv[1]
a_dir, foo_path, bar_path = _get_test_paths(temp_dir)
subprocess.check_call(["touch", foo_path])
subprocess.check_call(["touch", bar_path])
else:
unittest.main()

13
tools.py
View file

@ -393,14 +393,6 @@ def python_coverage(path):
"No corresponding test file: " + os.path.normpath(test_path))
@deps(deps={"python", "python3"}, gentoo_deps={"python"},
url="https://docs.python.org/3/library/profile.html")
def python_profile(path):
stdout, *rest = _do_command([_python_version(path), "-m", "cProfile",
"--sort=cumulative", path], timeout=TIMEOUT)
return Status.normal, fill3.Text(stdout)
@deps(deps={"python-pycodestyle", "python3-pycodestyle"},
fedora_deps={"python2-pycodestyle", "python3-pycodestyle"},
debian_deps={"pip/pycodestyle", "pip3/pycodestyle"},
@ -847,9 +839,8 @@ IMAGE_EXTENSIONS = ["png", "jpg", "gif", "bmp", "ppm", "tiff", "tga"]
TOOLS_FOR_EXTENSIONS = \
[
(["py"], [python_syntax, python_unittests, pydoc, mypy,
python_coverage, python_profile, pycodestyle, pyflakes,
pylint, python_gut, python_modulefinder, python_mccabe,
bandit]),
python_coverage, pycodestyle, pyflakes, pylint, python_gut,
python_modulefinder, python_mccabe, bandit]),
(["pyc"], [disassemble_pyc]),
(["pl", "pm", "t"], [perl_syntax, perldoc, perltidy]),
# (["p6", "pm6"], [perl6_syntax, perldoc]),

26
vigil
View file

@ -51,8 +51,6 @@ Example:
Options:
-h, --help Show this screen and exit.
-s on|off, --sandbox=on|off Use a sandbox to prevent changes to the
filesystem. The sandbox is on by default.
-w COUNT, --workers=COUNT The number of processes working in parallel.
By default it is the number of cpus minus 1.
-e "COMMAND", --editor="COMMAND" The command used to start the editor, in
@ -568,11 +566,10 @@ class Screen:
self._make_widgets()
self._key_map = make_key_map(Screen._KEY_DATA)
def make_workers(self, worker_count, is_sandboxed, is_being_tested):
def make_workers(self, worker_count, is_being_tested):
workers = []
for index in range(worker_count):
worker_ = worker.Worker(is_sandboxed, self._is_paused,
is_being_tested)
worker_ = worker.Worker(self._is_paused, is_being_tested)
workers.append(worker_)
future = worker_.job_runner(
self._summary, self._log, self._summary._jobs_added_event,
@ -947,8 +944,8 @@ def save_state(pickle_path, summary, screen, log):
tools.dump_pickle_safe(screen, pickle_path, open=open_compressed)
def main(root_path, loop, worker_count=None, is_sandboxed=True,
editor_command=None, theme=None, is_being_tested=False):
def main(root_path, loop, worker_count=None, editor_command=None, theme=None,
is_being_tested=False):
if worker_count is None:
worker_count = max(multiprocessing.cpu_count() - 1, 1)
if theme is None:
@ -974,7 +971,7 @@ def main(root_path, loop, worker_count=None, is_sandboxed=True,
root_path, loop, on_filesystem_change, is_path_excluded)
try:
log.log_message("Starting workers (%s) ..." % worker_count)
screen.make_workers(worker_count, is_sandboxed, is_being_tested)
screen.make_workers(worker_count, is_being_tested)
def exit_loop():
log.log_command("Exiting...")
@ -1034,10 +1031,6 @@ def check_arguments():
if not os.path.isdir(root_path):
print("File is not a directory:", root_path)
sys.exit(1)
if arguments["--sandbox"] not in ["on", "off", None]:
print("--sandbox argument must be 'on' or 'off'")
sys.exit(1)
is_sandboxed = arguments["--sandbox"] in ["on", None]
if arguments["--theme"] is not None:
themes = list(pygments.styles.get_all_styles())
if arguments["--theme"] not in themes:
@ -1045,17 +1038,14 @@ def check_arguments():
sys.exit(1)
editor_command = arguments["--editor"] or os.environ.get("EDITOR", None)\
or os.environ.get("VISUAL", None)
return root_path, worker_count, is_sandboxed, editor_command, \
arguments["--theme"]
return root_path, worker_count, editor_command, arguments["--theme"]
if __name__ == "__main__":
root_path, worker_count, is_sandboxed, editor_command, theme = \
check_arguments()
root_path, worker_count, editor_command, theme = check_arguments()
with terminal.console_title("vigil: " + os.path.basename(root_path)):
manage_cache(root_path)
with chdir(root_path): # FIX: Don't change directory if possible.
loop = asyncio.get_event_loop()
main(root_path, loop, worker_count, is_sandboxed, editor_command,
theme)
main(root_path, loop, worker_count, editor_command, theme)
os._exit(0)

2
vigil_test.py
View file

@ -223,7 +223,7 @@ class MainTestCase(unittest.TestCase):
with vigil.chdir(root_path):
with contextlib.redirect_stdout(io.StringIO()):
vigil.main(root_path, loop, worker_count=2,
is_sandboxed=True, is_being_tested=True)
is_being_tested=True)
for file_name in ["summary.pickle", "creation_time", "log",
"foo-metadata", "foo-contents"]:
self.assertTrue(os.path.exists(".vigil/" + file_name))

12
worker.py
View file

@ -12,8 +12,7 @@ import tools
class Worker:
def __init__(self, is_sandboxed, is_already_paused, is_being_tested):
self.is_sandboxed = is_sandboxed
def __init__(self, is_already_paused, is_being_tested):
self.is_already_paused = is_already_paused
self.is_being_tested = is_being_tested
self.result = None
@ -22,15 +21,8 @@ class Worker:
@asyncio.coroutine
def create_process(self):
if self.is_sandboxed:
sandbox_fs_path = os.path.join(os.path.dirname(__file__),
"sandbox_fs")
cache_path = os.path.join(os.getcwd(), tools.CACHE_PATH)
command = [sandbox_fs_path, cache_path, "--", __file__]
else:
command = [__file__]
create = asyncio.create_subprocess_exec(
*command, stdin=asyncio.subprocess.PIPE,
*[__file__], stdin=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
preexec_fn=os.setsid)
self.process = yield from create

10
worker_test.py
View file

@ -27,9 +27,9 @@ class WorkerTestCase(unittest.TestCase):
shutil.rmtree(self.temp_dir)
os.chdir(self.original_working_dir)
def _test_worker(self, is_sandboxed):
def test_run_job(self):
loop = asyncio.get_event_loop()
worker_ = worker.Worker(is_sandboxed, False, False)
worker_ = worker.Worker(False, False)
loop.run_until_complete(worker_.create_process())
future = worker_.run_tool("foo", tools.metadata)
status = loop.run_until_complete(future)
@ -37,12 +37,6 @@ class WorkerTestCase(unittest.TestCase):
result_path = os.path.join(tools.CACHE_PATH, "foo-metadata")
self.assertTrue(os.path.exists(result_path))
def test_run_job_without_sandbox(self):
self._test_worker(False)
def test_run_job_with_sandbox(self):
self._test_worker(True)
if __name__ == "__main__":
unittest.main()