ahamilton/eris
1
0
Fork 0
Code Releases Activity

Coding style.

Browse source 
Moved the Result class into tools, which simplifies worker.py and makes the
circular import problem go away.
This commit is contained in:
Andrew Hamilton 2016-02-13 18:48:53 +00:00
parent cc95534bd7
commit 2373e78cbd
5 changed files with 151 additions and 149 deletions
Download patch file Download diff file Expand all files Collapse all files

117
tools.py
View file

@ -8,6 +8,7 @@ import contextlib
import dis
import enum
import functools
import gzip
import hashlib
import io
import math
@ -31,6 +32,9 @@ import gut
import termstr
_CACHE_PATH = ".vigil"
class Status(enum.IntEnum):
ok = 1
@ -546,6 +550,119 @@ def php5_syntax(path):
php5_syntax.dependencies = {"php5"}
def lru_cache_with_eviction(maxsize=128, typed=False):
versions = {}
make_key = functools._make_key
def evict(*args, **kwds):
key = make_key(args, kwds, typed)
if key in versions:
versions[key] += 1
def decorating_function(user_function):
def remove_version(*args, **kwds):
return user_function(*args[1:], **kwds)
new_func = functools.lru_cache(maxsize=maxsize, typed=typed)(
remove_version)
def add_version(*args, **kwds):
key = make_key(args, kwds, typed)
return new_func(*((versions.setdefault(key, 0),) + args), **kwds)
add_version.versions = versions
add_version.cache_info = new_func.cache_info
add_version.evict = evict
return functools.update_wrapper(add_version, user_function)
return decorating_function
def dump_pickle_safe(object_, path, protocol=pickle.HIGHEST_PROTOCOL,
open=open):
tmp_path = path + ".tmp"
try:
with open(tmp_path, "wb") as file_:
pickle.dump(object_, file_, protocol=protocol)
except (OSError, KeyboardInterrupt):
os.remove(tmp_path)
else:
os.rename(tmp_path, path)
def status_to_str(status, is_status_simple):
if isinstance(status, enum.Enum):
dict_ = (_STATUS_TO_TERMSTR_SIMPLE if is_status_simple
else _STATUS_TO_TERMSTR)
return dict_[status]
else:
return status
class Result:
def __init__(self, path, tool, is_stored_compressed=True):
self.path = path
self.tool = tool
self._open_func = gzip.open if is_stored_compressed else open
self.pickle_path = os.path.join(_CACHE_PATH,
path + "-" + tool.__name__)
self.scroll_position = (0, 0)
self.is_completed = False
self.is_placeholder = True
self.status = Status.pending
@property
@lru_cache_with_eviction(maxsize=50)
def result(self):
unknown_label = fill3.Text("?")
if self.is_placeholder:
return unknown_label
try:
with self._open_func(self.pickle_path, "rb") as pickle_file:
return pickle.load(pickle_file)
except FileNotFoundError:
return unknown_label
@result.setter
def result(self, value):
os.makedirs(os.path.dirname(self.pickle_path), exist_ok=True)
dump_pickle_safe(value, self.pickle_path, open=self._open_func)
Result.result.fget.evict(self)
def set_status(self, status):
self.status = status
self.entry.appearance_cache = None
def run(self, log, appearance_changed_event, worker, runner):
self.is_placeholder = False
tool_name = _tool_name_colored(self.tool, self.path)
path_colored = _path_colored(self.path)
log.log_message(["Running ", tool_name, " on ", path_colored, "..."])
self.set_status(Status.running)
if runner.is_already_paused:
runner.is_already_paused = False
runner.pause()
appearance_changed_event.set()
start_time = time.time()
new_status = worker.run_tool(self.path, self.tool)
Result.result.fget.evict(self)
end_time = time.time()
self.set_status(new_status)
appearance_changed_event.set()
self.is_completed = True
log.log_message(
["Finished running ", tool_name, " on ", path_colored, ". ",
status_to_str(new_status, self.entry.summary.is_status_simple),
" %s secs" % round(end_time - start_time, 2)])
def reset(self):
self.is_placeholder = True
self.set_status(Status.pending)
def appearance_min(self):
return [status_to_str(self.status,
self.entry.summary.is_status_simple)]
def generic_tools():
return [contents, metadata]

23
tools_test.py
View file

@ -188,5 +188,28 @@ class ToolsTestCase(unittest.TestCase):
self._test_tool(tools.php5_syntax, [("root.php", tools.Status.ok)])
class LruCacheWithEvictionTestCase(unittest.TestCase):
def _assert_cache(self, func, hits, misses, current_size):
cache_info = func.cache_info()
self.assertEqual(cache_info.hits, hits)
self.assertEqual(cache_info.misses, misses)
self.assertEqual(cache_info.currsize, current_size)
def test_lru_cache_with_eviction(self):
@tools.lru_cache_with_eviction()
def a(foo):
return foo
self._assert_cache(a, 0, 0, 0)
self.assertEqual(a(1), 1)
self._assert_cache(a, 0, 1, 1)
a(1)
self._assert_cache(a, 1, 1, 1)
a.evict(1)
self._assert_cache(a, 1, 1, 1)
a(1)
self._assert_cache(a, 1, 2, 2)
if __name__ == "__main__":
golden.main()

132
vigil
View file

@ -77,6 +77,7 @@ import sandbox_fs
import terminal
import termstr
import tools
import worker
_LOG_PATH = os.path.join(os.getcwd(), "vigil.log")
@ -88,123 +89,7 @@ def _log_error(message=None):
log_file.write(message)
_CACHE_PATH = ".vigil"
def lru_cache_with_eviction(maxsize=128, typed=False):
versions = {}
make_key = functools._make_key
def evict(*args, **kwds):
key = make_key(args, kwds, typed)
if key in versions:
versions[key] += 1
def decorating_function(user_function):
def remove_version(*args, **kwds):
return user_function(*args[1:], **kwds)
new_func = functools.lru_cache(maxsize=maxsize, typed=typed)(
remove_version)
def add_version(*args, **kwds):
key = make_key(args, kwds, typed)
return new_func(*((versions.setdefault(key, 0),) + args), **kwds)
add_version.versions = versions
add_version.cache_info = new_func.cache_info
add_version.evict = evict
return functools.update_wrapper(add_version, user_function)
return decorating_function
def dump_pickle_safe(object_, path, protocol=pickle.HIGHEST_PROTOCOL,
open=open):
tmp_path = path + ".tmp"
try:
with open(tmp_path, "wb") as file_:
pickle.dump(object_, file_, protocol=protocol)
except (OSError, KeyboardInterrupt):
os.remove(tmp_path)
else:
os.rename(tmp_path, path)
def status_to_str(status, is_status_simple):
if isinstance(status, enum.Enum):
dict_ = (tools._STATUS_TO_TERMSTR_SIMPLE if is_status_simple
else tools._STATUS_TO_TERMSTR)
return dict_[status]
else:
return status
class Result:
def __init__(self, path, tool, is_stored_compressed=True):
self.path = path
self.tool = tool
self._open_func = gzip.open if is_stored_compressed else open
self.pickle_path = os.path.join(_CACHE_PATH,
path + "-" + tool.__name__)
self.scroll_position = (0, 0)
self.is_completed = False
self.is_placeholder = True
self.status = tools.Status.pending
@property
@lru_cache_with_eviction(maxsize=50)
def result(self):
unknown_label = fill3.Text("?")
if self.is_placeholder:
return unknown_label
try:
with self._open_func(self.pickle_path, "rb") as pickle_file:
return pickle.load(pickle_file)
except FileNotFoundError:
return unknown_label
@result.setter
def result(self, value):
os.makedirs(os.path.dirname(self.pickle_path), exist_ok=True)
dump_pickle_safe(value, self.pickle_path, open=self._open_func)
Result.result.fget.evict(self)
def set_status(self, status):
self.status = status
self.entry.appearance_cache = None
def run(self, log, appearance_changed_event, worker, runner):
self.is_placeholder = False
tool_name = tools._tool_name_colored(self.tool, self.path)
path_colored = tools._path_colored(self.path)
log.log_message(["Running ", tool_name, " on ", path_colored, "..."])
self.set_status(tools.Status.running)
if runner.is_already_paused:
runner.is_already_paused = False
runner.pause()
appearance_changed_event.set()
start_time = time.time()
new_status = worker.run_tool(self.path, self.tool)
Result.result.fget.evict(self)
end_time = time.time()
self.set_status(new_status)
appearance_changed_event.set()
self.is_completed = True
log.log_message(
["Finished running ", tool_name, " on ", path_colored, ". ",
status_to_str(new_status, self.entry.summary.is_status_simple),
" %s secs" % round(end_time - start_time, 2)])
def reset(self):
self.is_placeholder = True
self.set_status(tools.Status.pending)
def appearance_min(self):
return [status_to_str(self.status,
self.entry.summary.is_status_simple)]
import worker # Avoid a circular import. worker.py needs the Result class.
_CACHE_PATH = tools._CACHE_PATH
def reverse_style(style):
@ -383,7 +268,7 @@ class Summary:
result = self._cache[file_key][tool_key]
result.tool = tool
else:
result = Result(path, tool)
result = tools.Result(path, tool)
jobs_added = True
all_results.add(result)
if result.is_completed:
@ -609,9 +494,10 @@ def _highlight_chars(str_, style, marker="*"):
@functools.lru_cache()
def _get_help_text(is_status_simple=True):
usage = _highlight_chars(__doc__, Log.GREEN_STYLE)
return fill3.join("\n", [usage, "Statuses:"] +
[" " + status_to_str(status, is_status_simple) + " " +
meaning for status, meaning in tools.STATUS_MEANINGS])
return fill3.join(
"\n", [usage, "Statuses:"] +
[" " + tools.status_to_str(status, is_status_simple) + " " + meaning
for status, meaning in tools.STATUS_MEANINGS])
def _make_key_map(key_data):
@ -907,7 +793,7 @@ class Screen:
tool_name = tools._tool_name_colored(widget.tool, widget.path)
self._listing.title = (
tools._path_colored(widget.path) + " ─── " + tool_name + " " +
status_to_str(widget.status, self._summary.is_status_simple))
tools.status_to_str(widget.status, self._summary.is_status_simple))
incomplete = self._summary.result_total - self._summary.completed_total
progress_bar_size = max(0, width * incomplete //
self._summary.result_total)
@ -1122,7 +1008,7 @@ def main(root_path, worker_count=multiprocessing.cpu_count()*2,
screen._main_loop, screen.runners,
log._appearance_changed_event) = [None] * 7
open_compressed = functools.partial(gzip.open, compresslevel=1)
dump_pickle_safe(screen, pickle_path, open=open_compressed)
tools.dump_pickle_safe(screen, pickle_path, open=open_compressed)
finally:
if is_sandboxed:
sandbox.umount()

23
vigil_test.py
View file

@ -18,29 +18,6 @@ import golden
import vigil
class LruCacheWithEvictionTestCase(unittest.TestCase):
def _assert_cache(self, func, hits, misses, current_size):
cache_info = func.cache_info()
self.assertEqual(cache_info.hits, hits)
self.assertEqual(cache_info.misses, misses)
self.assertEqual(cache_info.currsize, current_size)
def test_lru_cache_with_eviction(self):
@vigil.lru_cache_with_eviction()
def a(foo):
return foo
self._assert_cache(a, 0, 0, 0)
self.assertEqual(a(1), 1)
self._assert_cache(a, 0, 1, 1)
a(1)
self._assert_cache(a, 1, 1, 1)
a.evict(1)
self._assert_cache(a, 1, 1, 1)
a(1)
self._assert_cache(a, 1, 2, 2)
_DIMENSIONS = (100, 60)

5
worker.py
View file

@ -10,7 +10,6 @@ import subprocess
import psutil
import tools
import vigil
def make_process_nicest(pid):
@ -28,7 +27,7 @@ class Worker:
[__file__], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
else:
cache_path = os.path.join(os.getcwd(), vigil._CACHE_PATH)
cache_path = os.path.join(os.getcwd(), tools._CACHE_PATH)
self.cache_mount = sandbox.mount_point + cache_path
subprocess.check_call(["sudo", "mount", "--bind", cache_path,
self.cache_mount])
@ -54,7 +53,7 @@ def main():
while True:
tool_name, path = input(), input()
tool = getattr(tools, tool_name)
result = vigil.Result(path, tool)
result = tools.Result(path, tool)
status, result.result = tools.run_tool_no_error(path, tool)
print(status.value, flush=True)