Make large results display quickly by loading them in pieces.

eris/__main__.py

@@ -318,8 +318,7 @@ class Summary:
             if jobs_added:
                 self._jobs_added_event.set()
             for result in deleted_results:
-                with contextlib.suppress(FileNotFoundError):
-                    os.remove(result.pickle_path)
+                result.delete()
             self.sort_entries()
 
     def _sweep_up(self, x, y):
@@ -470,6 +469,7 @@ class Summary:
     def refresh_result(self, result):
         if result.is_completed:
             result.reset()
+            result.delete()
             self.closest_placeholder_generator = None
             self._jobs_added_event.set()
             self.completed_total -= 1

eris/fill3.py

@@ -334,18 +334,19 @@ class View:
         return result
 
 
+def str_to_appearance(text, pad_char=" "):
+    lines = text.splitlines()
+    if len(lines) == 0:
+        return []
+    max_width = max(len(line) for line in lines)
+    height = len(lines)
+    return appearance_resize(lines, (max_width, height), pad_char)
+
+
 class Text:
 
     def __init__(self, text, pad_char=" "):
-        lines = text.splitlines()
-        if len(lines) == 0:
-            self.text = []
-        elif len(lines) == 1:
-            self.text = [lines[0]]
-        else:
-            max_width = max(len(line) for line in lines)
-            height = len(lines)
-            self.text = appearance_resize(lines, (max_width, height), pad_char)
+        self.text = str_to_appearance(text, pad_char)
 
     def appearance_min(self):
         return self.text
@@ -434,12 +435,16 @@ class Placeholder:
 
 class Fixed:
 
-    def __init__(self, appearance):
-        self.appearance_min_ = appearance
+    def __init__(self, appearance_min):
+        self.appearance_min_ = appearance_min
+        self.dimensions = appearance_dimensions(appearance_min)
 
     def appearance_min(self):
         return self.appearance_min_
 
+    def appearance_dimensions(self):
+        return self.dimensions
+
 
 ##########################
 

eris/paged_list.py

@@ -0,0 +1,73 @@
+# Copyright (C) 2019 Andrew Hamilton. All rights reserved.
+# Licensed under the Artistic License 2.0.
+
+
+import functools
+import gzip
+import os
+import pickle
+
+
+class PagedList:
+
+    def __init__(self, list_, pages_dir, page_size, cache_size):
+        self.pages_dir = pages_dir  # An empty or non-existant directory.
+        self.page_size = page_size
+        self.cache_size = cache_size
+        self._len = len(list_)
+        tmp_dir = pages_dir + ".tmp"
+        os.makedirs(tmp_dir)
+        if len(list_) == 0:
+            pages = [[]]
+        else:
+            pages = [list_[start:start+self.page_size]
+                     for start in range(0, len(list_), self.page_size)]
+        for index, page in enumerate(pages):
+            pickle_path = os.path.join(tmp_dir, str(index))
+            with gzip.open(pickle_path, "wb") as file_:
+                pickle.dump(page, file_, protocol=pickle.HIGHEST_PROTOCOL)
+        self.page_count = len(pages)
+        os.rename(tmp_dir, self.pages_dir)
+        self._get_page = functools.lru_cache(maxsize=cache_size)(self._get_page)
+
+    def __len__(self):
+        return self._len
+
+    def _get_page(self, index):  # This is cached, see __init__.
+        pickle_path = os.path.join(self.pages_dir, str(index))
+        with gzip.open(pickle_path, "rb") as file_:
+            return pickle.load(file_)
+
+    def __getitem__(self, index):
+        if type(index) == slice:
+            start, stop, step = index.indices(self._len)
+            start_page_index, start_page_offset = divmod(start, self.page_size)
+            stop_page_index, stop_page_offset = divmod(stop, self.page_size)
+            if stop_page_index == self.page_count:
+                stop_page_index -= 1
+                stop_page_offset = self.page_size
+            if start_page_index == stop_page_index:
+                return (self._get_page(start_page_index)
+                        [start_page_offset:stop_page_offset])
+            else:
+                result = self._get_page(start_page_index)[start_page_offset:]
+                middle_pages = (self._get_page(page_index) for page_index in
+                                range(start_page_index+1, stop_page_index))
+                for page in middle_pages:
+                    result.extend(page)
+                result.extend(
+                    self._get_page(stop_page_index)[:stop_page_offset])
+                return result
+        else:
+            page_index, page_offset = divmod(index, self.page_size)
+            return self._get_page(page_index)[page_offset]
+
+    def __getstate__(self):  # Don't pickle the lru_cache.
+        state = self.__dict__.copy()
+        del state["_get_page"]
+        return state
+
+    def __setstate__(self, state):
+        self.__dict__ = state
+        self._get_page = \
+            functools.lru_cache(maxsize=self.cache_size)(self._get_page)

eris/tools.py

@@ -14,6 +14,7 @@ import os
 import os.path
 import pickle
 import pwd
+import shutil
 import stat
 import subprocess
 import tempfile
@@ -588,6 +589,15 @@ class Result:
     def appearance_min(self):
         return [status_to_str(self.status)]
 
+    def get_pages_dir(self):
+        return self.pickle_path + ".pages"
+
+    def delete(self):
+        with contextlib.suppress(FileNotFoundError):
+            os.remove(self.pickle_path)
+        with contextlib.suppress(FileNotFoundError):
+            shutil.rmtree(self.get_pages_dir())
+
     def as_html(self):
         html, styles = termstr.TermStr(status_to_str(self.status)).as_html()
         return (f'<a title="{self.tool.__name__}" '

eris/worker.py

@@ -9,6 +9,7 @@ import signal
 
 import eris.fill3 as fill3
 import eris.tools as tools
+import eris.paged_list
 
 
 class Worker:
@@ -80,6 +81,15 @@ class Worker:
             os.killpg(self.child_pgid, signal.SIGKILL)
 
 
+def make_result_widget(text, result):
+    appearance = fill3.str_to_appearance(text)
+    page_size = 500
+    if len(appearance) > page_size:
+        appearance = eris.paged_list.PagedList(
+            appearance, result.get_pages_dir(), page_size, cache_size=2)
+    return fill3.Fixed(appearance)
+
+
 def main():
     print(os.getpgid(os.getpid()), flush=True)
     try:
@@ -88,7 +98,7 @@ def main():
             tool = getattr(tools, tool_name)
             result = tools.Result(path, tool)
             status, text = tools.run_tool_no_error(path, tool)
-            result.result = fill3.Text(text)
+            result.result = make_result_widget(text, result)
             print(status.value, flush=True)
     except:
         tools.log_error()

tests/paged_list_test.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3.7
+
+# Copyright (C) 2019 Andrew Hamilton. All rights reserved.
+# Licensed under the Artistic License 2.0.
+
+
+import pickle
+import tempfile
+import unittest
+
+import eris.paged_list as paged_list
+
+
+class PagedListTestCase(unittest.TestCase):
+
+    def test_getitem(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            list_ = paged_list.PagedList([3, 4, 5, 6], temp_dir, 4, 2)
+            self.assertEqual(list_[1], 4)            
+            self.assertEqual(list_[1:3], [4, 5])
+            self.assertEqual(list_[0:4], [3, 4, 5, 6])        
+        with tempfile.TemporaryDirectory() as temp_dir:
+            list_ = paged_list.PagedList([3, 4, 5, 6], temp_dir, 2, 2)
+            self.assertEqual(list_[1:3], [4, 5])
+        with tempfile.TemporaryDirectory() as temp_dir:
+            list_ = paged_list.PagedList([3, 4, 5, 6, 7, 8], temp_dir, 2, 2)
+            self.assertEqual(list_[1:5], [4, 5, 6, 7])
+            self.assertEqual(list_[:2], [3, 4])
+            self.assertEqual(list_[2:], [5, 6, 7, 8])
+
+    def test_pickling(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            list_ = paged_list.PagedList([3, 4, 5], temp_dir, 2, 2)
+            list_b = pickle.loads(pickle.dumps(list_))
+            self.assertEqual(list_b[1], 4)
+
+
+if __name__ == "__main__":
+    unittest.main()