From 2373e78cbd85622c59673fb5b921274e86ec222f Mon Sep 17 00:00:00 2001 From: Andrew Hamilton Date: Sat, 13 Feb 2016 18:48:53 +0000 Subject: [PATCH] Coding style. Moved the Result class into tools, which simplifies worker.py and makes the circular import problem go away. --- tools.py | 117 ++++++++++++++++++++++++++++++++++++++++++++ tools_test.py | 23 +++++++++ vigil | 132 ++++---------------------------------------------- vigil_test.py | 23 --------- worker.py | 5 +- 5 files changed, 151 insertions(+), 149 deletions(-) diff --git a/tools.py b/tools.py index 3b5bd41..a1e4554 100644 --- a/tools.py +++ b/tools.py @@ -8,6 +8,7 @@ import contextlib import dis import enum import functools +import gzip import hashlib import io import math @@ -31,6 +32,9 @@ import gut import termstr +_CACHE_PATH = ".vigil" + + class Status(enum.IntEnum): ok = 1 @@ -546,6 +550,119 @@ def php5_syntax(path): php5_syntax.dependencies = {"php5"} +def lru_cache_with_eviction(maxsize=128, typed=False): + versions = {} + make_key = functools._make_key + + def evict(*args, **kwds): + key = make_key(args, kwds, typed) + if key in versions: + versions[key] += 1 + + def decorating_function(user_function): + + def remove_version(*args, **kwds): + return user_function(*args[1:], **kwds) + new_func = functools.lru_cache(maxsize=maxsize, typed=typed)( + remove_version) + + def add_version(*args, **kwds): + key = make_key(args, kwds, typed) + return new_func(*((versions.setdefault(key, 0),) + args), **kwds) + add_version.versions = versions + add_version.cache_info = new_func.cache_info + add_version.evict = evict + return functools.update_wrapper(add_version, user_function) + return decorating_function + + +def dump_pickle_safe(object_, path, protocol=pickle.HIGHEST_PROTOCOL, + open=open): + tmp_path = path + ".tmp" + try: + with open(tmp_path, "wb") as file_: + pickle.dump(object_, file_, protocol=protocol) + except (OSError, KeyboardInterrupt): + os.remove(tmp_path) + else: + os.rename(tmp_path, path) + + +def status_to_str(status, is_status_simple): + if isinstance(status, enum.Enum): + dict_ = (_STATUS_TO_TERMSTR_SIMPLE if is_status_simple + else _STATUS_TO_TERMSTR) + return dict_[status] + else: + return status + + +class Result: + + def __init__(self, path, tool, is_stored_compressed=True): + self.path = path + self.tool = tool + self._open_func = gzip.open if is_stored_compressed else open + self.pickle_path = os.path.join(_CACHE_PATH, + path + "-" + tool.__name__) + self.scroll_position = (0, 0) + self.is_completed = False + self.is_placeholder = True + self.status = Status.pending + + @property + @lru_cache_with_eviction(maxsize=50) + def result(self): + unknown_label = fill3.Text("?") + if self.is_placeholder: + return unknown_label + try: + with self._open_func(self.pickle_path, "rb") as pickle_file: + return pickle.load(pickle_file) + except FileNotFoundError: + return unknown_label + + @result.setter + def result(self, value): + os.makedirs(os.path.dirname(self.pickle_path), exist_ok=True) + dump_pickle_safe(value, self.pickle_path, open=self._open_func) + Result.result.fget.evict(self) + + def set_status(self, status): + self.status = status + self.entry.appearance_cache = None + + def run(self, log, appearance_changed_event, worker, runner): + self.is_placeholder = False + tool_name = _tool_name_colored(self.tool, self.path) + path_colored = _path_colored(self.path) + log.log_message(["Running ", tool_name, " on ", path_colored, "..."]) + self.set_status(Status.running) + if runner.is_already_paused: + runner.is_already_paused = False + runner.pause() + appearance_changed_event.set() + start_time = time.time() + new_status = worker.run_tool(self.path, self.tool) + Result.result.fget.evict(self) + end_time = time.time() + self.set_status(new_status) + appearance_changed_event.set() + self.is_completed = True + log.log_message( + ["Finished running ", tool_name, " on ", path_colored, ". ", + status_to_str(new_status, self.entry.summary.is_status_simple), + " %s secs" % round(end_time - start_time, 2)]) + + def reset(self): + self.is_placeholder = True + self.set_status(Status.pending) + + def appearance_min(self): + return [status_to_str(self.status, + self.entry.summary.is_status_simple)] + + def generic_tools(): return [contents, metadata] diff --git a/tools_test.py b/tools_test.py index 0c62410..7b12875 100755 --- a/tools_test.py +++ b/tools_test.py @@ -188,5 +188,28 @@ class ToolsTestCase(unittest.TestCase): self._test_tool(tools.php5_syntax, [("root.php", tools.Status.ok)]) +class LruCacheWithEvictionTestCase(unittest.TestCase): + + def _assert_cache(self, func, hits, misses, current_size): + cache_info = func.cache_info() + self.assertEqual(cache_info.hits, hits) + self.assertEqual(cache_info.misses, misses) + self.assertEqual(cache_info.currsize, current_size) + + def test_lru_cache_with_eviction(self): + @tools.lru_cache_with_eviction() + def a(foo): + return foo + self._assert_cache(a, 0, 0, 0) + self.assertEqual(a(1), 1) + self._assert_cache(a, 0, 1, 1) + a(1) + self._assert_cache(a, 1, 1, 1) + a.evict(1) + self._assert_cache(a, 1, 1, 1) + a(1) + self._assert_cache(a, 1, 2, 2) + + if __name__ == "__main__": golden.main() diff --git a/vigil b/vigil index 7611dd2..a58adf8 100755 --- a/vigil +++ b/vigil @@ -77,6 +77,7 @@ import sandbox_fs import terminal import termstr import tools +import worker _LOG_PATH = os.path.join(os.getcwd(), "vigil.log") @@ -88,123 +89,7 @@ def _log_error(message=None): log_file.write(message) -_CACHE_PATH = ".vigil" - - -def lru_cache_with_eviction(maxsize=128, typed=False): - versions = {} - make_key = functools._make_key - - def evict(*args, **kwds): - key = make_key(args, kwds, typed) - if key in versions: - versions[key] += 1 - - def decorating_function(user_function): - - def remove_version(*args, **kwds): - return user_function(*args[1:], **kwds) - new_func = functools.lru_cache(maxsize=maxsize, typed=typed)( - remove_version) - - def add_version(*args, **kwds): - key = make_key(args, kwds, typed) - return new_func(*((versions.setdefault(key, 0),) + args), **kwds) - add_version.versions = versions - add_version.cache_info = new_func.cache_info - add_version.evict = evict - return functools.update_wrapper(add_version, user_function) - return decorating_function - - -def dump_pickle_safe(object_, path, protocol=pickle.HIGHEST_PROTOCOL, - open=open): - tmp_path = path + ".tmp" - try: - with open(tmp_path, "wb") as file_: - pickle.dump(object_, file_, protocol=protocol) - except (OSError, KeyboardInterrupt): - os.remove(tmp_path) - else: - os.rename(tmp_path, path) - - -def status_to_str(status, is_status_simple): - if isinstance(status, enum.Enum): - dict_ = (tools._STATUS_TO_TERMSTR_SIMPLE if is_status_simple - else tools._STATUS_TO_TERMSTR) - return dict_[status] - else: - return status - - -class Result: - - def __init__(self, path, tool, is_stored_compressed=True): - self.path = path - self.tool = tool - self._open_func = gzip.open if is_stored_compressed else open - self.pickle_path = os.path.join(_CACHE_PATH, - path + "-" + tool.__name__) - self.scroll_position = (0, 0) - self.is_completed = False - self.is_placeholder = True - self.status = tools.Status.pending - - @property - @lru_cache_with_eviction(maxsize=50) - def result(self): - unknown_label = fill3.Text("?") - if self.is_placeholder: - return unknown_label - try: - with self._open_func(self.pickle_path, "rb") as pickle_file: - return pickle.load(pickle_file) - except FileNotFoundError: - return unknown_label - - @result.setter - def result(self, value): - os.makedirs(os.path.dirname(self.pickle_path), exist_ok=True) - dump_pickle_safe(value, self.pickle_path, open=self._open_func) - Result.result.fget.evict(self) - - def set_status(self, status): - self.status = status - self.entry.appearance_cache = None - - def run(self, log, appearance_changed_event, worker, runner): - self.is_placeholder = False - tool_name = tools._tool_name_colored(self.tool, self.path) - path_colored = tools._path_colored(self.path) - log.log_message(["Running ", tool_name, " on ", path_colored, "..."]) - self.set_status(tools.Status.running) - if runner.is_already_paused: - runner.is_already_paused = False - runner.pause() - appearance_changed_event.set() - start_time = time.time() - new_status = worker.run_tool(self.path, self.tool) - Result.result.fget.evict(self) - end_time = time.time() - self.set_status(new_status) - appearance_changed_event.set() - self.is_completed = True - log.log_message( - ["Finished running ", tool_name, " on ", path_colored, ". ", - status_to_str(new_status, self.entry.summary.is_status_simple), - " %s secs" % round(end_time - start_time, 2)]) - - def reset(self): - self.is_placeholder = True - self.set_status(tools.Status.pending) - - def appearance_min(self): - return [status_to_str(self.status, - self.entry.summary.is_status_simple)] - - -import worker # Avoid a circular import. worker.py needs the Result class. +_CACHE_PATH = tools._CACHE_PATH def reverse_style(style): @@ -383,7 +268,7 @@ class Summary: result = self._cache[file_key][tool_key] result.tool = tool else: - result = Result(path, tool) + result = tools.Result(path, tool) jobs_added = True all_results.add(result) if result.is_completed: @@ -609,9 +494,10 @@ def _highlight_chars(str_, style, marker="*"): @functools.lru_cache() def _get_help_text(is_status_simple=True): usage = _highlight_chars(__doc__, Log.GREEN_STYLE) - return fill3.join("\n", [usage, "Statuses:"] + - [" " + status_to_str(status, is_status_simple) + " " + - meaning for status, meaning in tools.STATUS_MEANINGS]) + return fill3.join( + "\n", [usage, "Statuses:"] + + [" " + tools.status_to_str(status, is_status_simple) + " " + meaning + for status, meaning in tools.STATUS_MEANINGS]) def _make_key_map(key_data): @@ -907,7 +793,7 @@ class Screen: tool_name = tools._tool_name_colored(widget.tool, widget.path) self._listing.title = ( tools._path_colored(widget.path) + " ─── " + tool_name + " " + - status_to_str(widget.status, self._summary.is_status_simple)) + tools.status_to_str(widget.status, self._summary.is_status_simple)) incomplete = self._summary.result_total - self._summary.completed_total progress_bar_size = max(0, width * incomplete // self._summary.result_total) @@ -1122,7 +1008,7 @@ def main(root_path, worker_count=multiprocessing.cpu_count()*2, screen._main_loop, screen.runners, log._appearance_changed_event) = [None] * 7 open_compressed = functools.partial(gzip.open, compresslevel=1) - dump_pickle_safe(screen, pickle_path, open=open_compressed) + tools.dump_pickle_safe(screen, pickle_path, open=open_compressed) finally: if is_sandboxed: sandbox.umount() diff --git a/vigil_test.py b/vigil_test.py index 409568c..3824b10 100755 --- a/vigil_test.py +++ b/vigil_test.py @@ -18,29 +18,6 @@ import golden import vigil -class LruCacheWithEvictionTestCase(unittest.TestCase): - - def _assert_cache(self, func, hits, misses, current_size): - cache_info = func.cache_info() - self.assertEqual(cache_info.hits, hits) - self.assertEqual(cache_info.misses, misses) - self.assertEqual(cache_info.currsize, current_size) - - def test_lru_cache_with_eviction(self): - @vigil.lru_cache_with_eviction() - def a(foo): - return foo - self._assert_cache(a, 0, 0, 0) - self.assertEqual(a(1), 1) - self._assert_cache(a, 0, 1, 1) - a(1) - self._assert_cache(a, 1, 1, 1) - a.evict(1) - self._assert_cache(a, 1, 1, 1) - a(1) - self._assert_cache(a, 1, 2, 2) - - _DIMENSIONS = (100, 60) diff --git a/worker.py b/worker.py index 5d79f43..5c4350e 100755 --- a/worker.py +++ b/worker.py @@ -10,7 +10,6 @@ import subprocess import psutil import tools -import vigil def make_process_nicest(pid): @@ -28,7 +27,7 @@ class Worker: [__file__], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: - cache_path = os.path.join(os.getcwd(), vigil._CACHE_PATH) + cache_path = os.path.join(os.getcwd(), tools._CACHE_PATH) self.cache_mount = sandbox.mount_point + cache_path subprocess.check_call(["sudo", "mount", "--bind", cache_path, self.cache_mount]) @@ -54,7 +53,7 @@ def main(): while True: tool_name, path = input(), input() tool = getattr(tools, tool_name) - result = vigil.Result(path, tool) + result = tools.Result(path, tool) status, result.result = tools.run_tool_no_error(path, tool) print(status.value, flush=True)