diff --git a/eris/__main__.py b/eris/__main__.py index 48bb5da..fbbd29e 100755 --- a/eris/__main__.py +++ b/eris/__main__.py @@ -22,6 +22,7 @@ import functools import gzip import importlib import importlib.resources +import itertools import math import multiprocessing import os @@ -43,6 +44,7 @@ from eris import terminal from eris import termstr from eris import tools from eris import worker +from eris import paged_list USAGE = """ @@ -86,10 +88,12 @@ KEYS_DOC = """Keys: class Entry: - def __init__(self, path, results, summary, highlighted=None, + MAX_WIDTH = 0 + + def __init__(self, path, results, change_time, highlighted=None, set_results=True): self.path = path - self.summary = summary + self.change_time = change_time self.highlighted = highlighted self.results = results if set_results: @@ -111,8 +115,8 @@ class Entry: def appearance_min(self): if self.appearance_cache is None \ - or self.last_width != self.summary._max_width: - self.last_width = self.summary._max_width + or self.last_width != Entry.MAX_WIDTH: + self.last_width = Entry.MAX_WIDTH if self.highlighted is not None: self.results[self.highlighted].is_highlighted = True row_appearance = self.widget.appearance_min() @@ -131,7 +135,7 @@ class Entry: html_parts.append(result_html) styles.update(result_styles) path = tools.path_colored(self.path) - padding = " " * (self.summary._max_width - len(self.widget) + 1) + padding = " " * (Entry.MAX_WIDTH - len(self.widget) + 1) path_html, path_styles = termstr.TermStr(padding + path).as_html() return "".join(html_parts) + path_html, styles.union(path_styles) @@ -206,22 +210,44 @@ class Summary: self._root_path = root_path self._jobs_added_event = jobs_added_event self._view_widget = fill3.View.from_widget(self) - self.__cursor_position = (0, 0) - self.closest_placeholder_generator = None - self._cache = {} self.is_directory_sort = True - self._max_width = 0 + self._old_entries = [] + self.reset() + + def reset(self): + self.__cursor_position = (0, 0) + Entry.MAX_WIDTH = 0 self._max_path_length = 0 - self._entries = sortedcontainers.SortedList([], key=directory_sort) self.result_total = 0 self.completed_total = 0 + self.is_loaded = False + self.closest_placeholder_generator = None + sort_func = directory_sort if self.is_directory_sort else type_sort + self._entries = sortedcontainers.SortedList([], key=sort_func) def __getstate__(self): state = self.__dict__.copy() state["closest_placeholder_generator"] = None state["_jobs_added_event"] = None + summary_path = os.path.join(tools.CACHE_PATH, "summary_dir") + open_compressed = functools.partial(gzip.open, compresslevel=1) + x, y = self.cursor_position() + if y == 0: + entries = [] + else: + current_entry = self._entries[y] + del self._entries[y] + entries = itertools.chain([current_entry], self._entries) + state["_old_entries"] = paged_list.PagedList( + entries, summary_path, 2000, 1, exist_ok=True, + open_func=open_compressed) + state["_entries"] = None return state + def __setstate__(self, state): + self.__dict__ = state + self.reset() + @property def _cursor_position(self): return self.__cursor_position @@ -238,24 +264,16 @@ class Summary: self._entries, key=key_func) self.closest_placeholder_generator = None - def file_added(self, path): - full_path = os.path.join(self._root_path, path) - try: - change_time = os.stat(full_path).st_ctime - except FileNotFoundError: + def add_entry(self, entry): + if entry in self._entries: return - row = [] - change_time = self._cache.setdefault(path, change_time) - for tool in tools.tools_for_path(path): - result = tools.Result(path, tool) + for result in entry: self.result_total += 1 if result.is_completed: self.completed_total += 1 - row.append(result) - self._max_width = max(len(row), self._max_width) - self._max_path_length = max(len(path) - len("./"), + Entry.MAX_WIDTH = max(len(entry), Entry.MAX_WIDTH) + self._max_path_length = max(len(entry.path) - len("./"), self._max_path_length) - entry = Entry(path, row, self) self._entries.add(entry) entry_index = self._entries.index(entry) x, y = self._cursor_position @@ -264,10 +282,20 @@ class Summary: self._jobs_added_event.set() self.closest_placeholder_generator = None - def file_deleted(self, path, check=True): - if check and os.path.exists(os.path.join(self._root_path, path)): + def on_file_added(self, path): + full_path = os.path.join(self._root_path, path) + try: + change_time = os.stat(full_path).st_ctime + except FileNotFoundError: return - entry = Entry(path, [], self) + row = [tools.Result(path, tool) for tool in tools.tools_for_path(path)] + entry = Entry(path, row, change_time) + self.add_entry(entry) + + def on_file_deleted(self, path): + if os.path.exists(os.path.join(self._root_path, path)): + return + entry = Entry(path, [], None) try: index = self._entries.index(entry) except ValueError: @@ -275,7 +303,6 @@ class Summary: x, y = self._cursor_position if index < y: self.scroll(0, 1) - del self._cache[path] for result in self._entries[index]: if result.is_completed: self.completed_total -= 1 @@ -283,8 +310,8 @@ class Summary: result.delete() row = self._entries[index] self._entries.pop(index) - if len(row) == self._max_width: - self._max_width = max((len(entry) for entry in self._entries), + if len(row) == Entry.MAX_WIDTH: + Entry.MAX_WIDTH = max((len(entry) for entry in self._entries), default=0) if (len(path) - 2) == self._max_path_length: self._max_path_length = max(((len(entry.path) - 2) @@ -294,14 +321,15 @@ class Summary: self._cursor_position = x, y - 1 self.closest_placeholder_generator = None - def file_modified(self, path): - entry = Entry(path, [], self) + def on_file_modified(self, path): + entry = Entry(path, [], None) try: entry_index = self._entries.index(entry) except ValueError: return for result in self._entries[entry_index]: self.refresh_result(result, only_completed=False) + self.closest_placeholder_generator = None @contextlib.contextmanager def keep_selection(self): @@ -320,25 +348,37 @@ class Summary: self._cursor_position = (x, len(self._entries) - 1) async def sync_with_filesystem(self, log=None): - log.log_message("Started syncing filesystem…") + start_time = time.time() + cache = {} + log.log_message("Started loading summary…") + for index, entry in enumerate(self._old_entries): + if index != 0 and index % 5000 == 0: + log.log_message(f"Loaded {index} files…") + await asyncio.sleep(0) + self.add_entry(entry) + cache[entry.path] = entry.change_time + duration = time.time() - start_time + log.log_message(f"Finished loading summary. {round(duration, 2)} secs") + self.is_loaded = True + log.log_message("Started sync with filesystem…") start_time = time.time() all_paths = set() for path in fix_paths(self._root_path, codebase_files(self._root_path)): await asyncio.sleep(0) all_paths.add(path) - if path in self._cache: + if path in cache: full_path = os.path.join(self._root_path, path) change_time = os.stat(full_path).st_ctime - if change_time != self._cache[path]: - self._cache[path] = change_time - self.file_modified(path) + if change_time != cache[path]: + cache[path] = change_time + self.on_file_modified(path) else: - self.file_added(path) - for path in self._cache.keys() - all_paths: + self.on_file_added(path) + for path in cache.keys() - all_paths: await asyncio.sleep(0) - self.file_deleted(path) + self.on_file_deleted(path) duration = time.time() - start_time - log.log_message(f"Finished syncing filesystem. {round(duration, 2)} secs") + log.log_message(f"Finished sync with filesystem. {round(duration, 2)} secs") def _sweep_up(self, x, y): yield from reversed(self._entries[y][:x]) @@ -375,7 +415,7 @@ class Summary: return await self.closest_placeholder_generator.asend(None) def appearance_dimensions(self): - return self._max_path_length + 1 + self._max_width, len(self._entries) + return self._max_path_length + 1 + Entry.MAX_WIDTH, len(self._entries) def appearance_interval(self, interval): start_y, end_y = interval @@ -1072,12 +1112,12 @@ def on_filesystem_event(event, summary, root_path, appearance_changed_event): path = list(fix_paths(root_path, [event.pathname]))[0] if is_path_excluded(path[2:]): return - inotify_actions = {pyinotify.IN_CREATE: summary.file_added, - pyinotify.IN_MOVED_TO: summary.file_added, - pyinotify.IN_DELETE: summary.file_deleted, - pyinotify.IN_MOVED_FROM: summary.file_deleted, - pyinotify.IN_ATTRIB: summary.file_modified, - pyinotify.IN_CLOSE_WRITE: summary.file_modified} + inotify_actions = {pyinotify.IN_CREATE: summary.on_file_added, + pyinotify.IN_MOVED_TO: summary.on_file_added, + pyinotify.IN_DELETE: summary.on_file_deleted, + pyinotify.IN_MOVED_FROM: summary.on_file_deleted, + pyinotify.IN_ATTRIB: summary.on_file_modified, + pyinotify.IN_CLOSE_WRITE: summary.on_file_modified} if event.mask not in inotify_actions: return try: @@ -1124,7 +1164,8 @@ def main(root_path, loop, worker_count=None, editor_command=None, theme=None, log.log_message("Program stopped.") finally: notifier.stop() - screen.save() + if summary.is_loaded: + screen.save() @contextlib.contextmanager diff --git a/eris/paged_list.py b/eris/paged_list.py index 2b23f91..189d467 100644 --- a/eris/paged_list.py +++ b/eris/paged_list.py @@ -29,6 +29,7 @@ class PagedList: shutil.rmtree(tmp_dir, ignore_errors=True) shutil.rmtree(pages_dir, ignore_errors=True) os.makedirs(tmp_dir) + index = 0 for index, page in enumerate(batch(list_, page_size)): pickle_path = os.path.join(tmp_dir, str(index)) with self.open_func(pickle_path, "wb") as file_: @@ -43,8 +44,11 @@ class PagedList: def _get_page_org(self, index): # This is cached, see setup_page_cache. pickle_path = os.path.join(self.pages_dir, str(index)) - with self.open_func(pickle_path, "rb") as file_: - return pickle.load(file_) + try: + with self.open_func(pickle_path, "rb") as file_: + return pickle.load(file_) + except FileNotFoundError: + raise IndexError def __getitem__(self, index): if isinstance(index, slice): diff --git a/eris/worker.py b/eris/worker.py index 39c695d..51e4ced 100755 --- a/eris/worker.py +++ b/eris/worker.py @@ -55,7 +55,7 @@ class Worker: await self.result.run(log, appearance_changed_event, self) self.result.compression = self.compression Worker.unsaved_jobs_total += 1 - if Worker.unsaved_jobs_total == 2000: + if Worker.unsaved_jobs_total == 5000 and summary.is_loaded: log.log_message(Worker.AUTOSAVE_MESSAGE) screen.save() summary.completed_total += 1 diff --git a/tests/__main___test.py b/tests/__main___test.py index 1e2dacf..04b4897 100755 --- a/tests/__main___test.py +++ b/tests/__main___test.py @@ -159,7 +159,7 @@ class SummarySyncWithFilesystemTestCase(unittest.TestCase): self.assertEqual(self.summary.completed_total, completed_total) self.assertEqual(self.summary.result_total, result_total) max_width = max((len(row) for row in self.summary._entries), default=0) - self.assertEqual(self.summary._max_width, max_width) + self.assertEqual(__main__.Entry.MAX_WIDTH, max_width) max_path_length = max( (len(row.path) - 2 for row in self.summary._entries), default=0) self.assertEqual(self.summary._max_path_length, max_path_length) diff --git a/tests/paged_list_test.py b/tests/paged_list_test.py index bfefc5f..7f15cc1 100755 --- a/tests/paged_list_test.py +++ b/tests/paged_list_test.py @@ -31,6 +31,12 @@ class PagedListTestCase(unittest.TestCase): self.assertEqual(list_[1:5], [4, 5, 6, 7]) self.assertEqual(list_[:2], [3, 4]) self.assertEqual(list_[2:], [5, 6, 7, 8]) + self.assertEqual(list(list_), [3, 4, 5, 6, 7, 8]) + self.assertRaises(IndexError, list_.__getitem__, 6) + with tempfile.TemporaryDirectory() as temp_dir: + list_ = paged_list.PagedList([], temp_dir, 2, 2) + self.assertRaises(IndexError, list_.__getitem__, 0) + # self.assertEqual(list_[3:4], []) FIX def test_pickling(self): with tempfile.TemporaryDirectory() as temp_dir: