Can build AppImages more successfully.

- The meta/Recipes script doesn't run the post-install of debian
  packages, so some weren't installed correctly.
- This scheme lets vigil install into a container as usual.
- The files in the AppDir are calculated as the difference between
  the files in the initial debootstrapped container, and the
  container after the installation.

appimage/AppRun

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+
+# This is a modified version of the AppRun made by AppImage's meta/Recipe.
+
+
+set -e
+
+
+HERE="$(dirname "$(readlink -f "${0}")")"
+export UNION_PRELOAD="${HERE}"
+export LD_PRELOAD="${HERE}/libunionpreload.so"
+export LD_LIBRARY_PATH="${HERE}"/usr/lib/:"${HERE}"/usr/lib/i386-linux-gnu/:"${HERE}"/usr/lib/x86_64-linux-gnu/:"${HERE}"/usr/lib32/:"${HERE}"/usr/lib64/:"${HERE}"/lib/:"${HERE}"/lib/i386-linux-gnu/:"${HERE}"/lib/x86_64-linux-gnu/:"${HERE}"/lib32/:"${HERE}"/lib64/:"${LD_LIBRARY_PATH}"
+export PYTHONPATH="/usr/local/lib/python3.5/dist-packages:${HERE}"/usr/share/pyshared/:"${PYTHONPATH}"
+export XDG_DATA_DIRS="${HERE}"/usr/share/:"${XDG_DATA_DIRS}"
+export PERLLIB="${HERE}"/usr/share/perl5/:"${HERE}"/usr/lib/perl5/:"${PERLLIB}"
+export GSETTINGS_SCHEMA_DIR="${HERE}"/usr/share/glib-2.0/schemas/:"${GSETTINGS_SCHEMA_DIR}"
+export QT_PLUGIN_PATH="${HERE}"/usr/lib/qt4/plugins/:"${HERE}"/usr/lib/i386-linux-gnu/qt4/plugins/:"${HERE}"/usr/lib/x86_64-linux-gnu/qt4/plugins/:"${HERE}"/usr/lib32/qt4/plugins/:"${HERE}"/usr/lib64/qt4/plugins/:"${HERE}"/usr/lib/qt5/plugins/:"${HERE}"/usr/lib/i386-linux-gnu/qt5/plugins/:"${HERE}"/usr/lib/x86_64-linux-gnu/qt5/plugins/:"${HERE}"/usr/lib32/qt5/plugins/:"${HERE}"/usr/lib64/qt5/plugins/:"${QT_PLUGIN_PATH}"
+EXEC=$(grep -e '^Exec=.*' "${HERE}"/*.desktop | head -n 1 | cut -d "=" -f 2- | sed -e 's|%.||g')
+exec ${EXEC} $@

appimage/make-appimage

@@ -1,29 +0,0 @@
-#!/bin/bash
-
-# Copyright (C) 2017 Andrew Hamilton. All rights reserved.
-# Licensed under the Artistic License 2.0.
-
-
-set -e
-
-
-VIGIL_PATH=$(realpath $(dirname $0)/..)
-export VIGIL_PATH
-
-
-function get_patched_recipe {
-    wget -c "https://github.com/AppImage/AppImages/raw/94d39b68d8d7ab56f31fde6f59bc8d2813d13599/recipes/meta/Recipe"
-    cp Recipe Recipe-patched
-    sed -i -e 's|delete_blacklisted|# delete_blacklisted|g' Recipe-patched
-    chmod +x Recipe-patched
-}
-
-
-[ $# -eq 0 ] && WORK_PATH=$(mktemp -d --suffix=-appimage) || WORK_PATH="$1"
-cd "$WORK_PATH"
-$VIGIL_PATH/appimage/make-appimage-yml.py > vigil.yml
-get_patched_recipe
-./Recipe-patched --no-di vigil.yml
-mv out/Vigil* .
-echo
-echo "AppImage path: $(realpath Vigil*)"

appimage/make-appimage-yml.py

@@ -1,48 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright (C) 2017 Andrew Hamilton. All rights reserved.
-# Licensed under the Artistic License 2.0.
-
-
-import vigil.tools as tools
-
-
-dist_deps = set()
-for dependency in tools.dependencies("ubuntu"):
-    if "/" in dependency:
-        raise ValueError
-    else:
-        dist_deps.add(dependency)
-dist_deps.update({"python3-pygments", "python3-pyinotify", "python3-docopt",
-                  "util-linux", "python3-pil"})
-dep_list = "\n    - ".join(sorted(dist_deps))
-print("""app: vigil-code-monitor
-union: true
-
-ingredients:
-  packages:
-    - %s
-  dist: zesty
-  sources:
-    - deb http://archive.ubuntu.com/ubuntu/ zesty main universe
-
-script:
-  - export APPDIR=$(pwd)
-  - export PYTHONPATH=$APPDIR/usr/share/pyshared
-  - mkdir -p $PYTHONPATH
-  - (cd $VIGIL_PATH; /usr/bin/python3 setup.py install \
-     --prefix=$APPDIR/usr --install-lib=$PYTHONPATH)
-  - cp $VIGIL_PATH/appimage/vigil-icon.png .
-  - cp -a $VIGIL_PATH/tests .
-  - cp $VIGIL_PATH/test-all tests
-  - cat > vigil.desktop <<\EOF
-  - [Desktop Entry]
-  - Type=Application
-  - Name=Vigil Code Monitor
-  - Comment=Vigil maintains an up-to-date set of reports for every file in a codebase.
-  - Exec=vigil
-  - Terminal=true
-  - Icon=vigil-icon.png
-  - Categories=Application;
-  - EOF
-""" % dep_list)

appimage/vigil.desktop

@@ -0,0 +1,8 @@
+[Desktop Entry]
+Type=Application
+Name=Vigil
+Comment=
+Exec=vigil
+Terminal=true
+Icon=vigil
+Categories=Application;

make-appimage.py

@@ -0,0 +1,108 @@
+#!/usr/bin/python3
+
+# Copyright (C) 2017 Andrew Hamilton. All rights reserved.
+# Licensed under the Artistic License 2.0.
+
+
+import os
+import pickle
+import sys
+import tempfile
+
+import test_distributions
+
+
+VIGIL_PATH = os.path.realpath(os.path.dirname(__file__))
+cmd = test_distributions.cmd
+
+
+def all_paths(path):
+    for entry in os.scandir(path):
+        if entry.is_dir(follow_symlinks=False):
+            yield from all_paths(entry.path)
+        else:
+            yield entry.path
+
+
+def relative_paths(root_path, paths):
+    root_len = len(root_path)
+    for path in paths:
+        yield "." + path[root_len:]
+
+
+def make_sub_container(src_root, dest_root, paths):
+    for path in paths:
+        dest_path = os.path.join(dest_root, path)
+        os.makedirs(os.path.dirname(dest_path), exist_ok=True)
+        os.link(os.path.join(src_root, path), dest_path)
+
+
+def filter_paths(paths, excluded):
+    return [path for path in paths if not excluded in path]
+
+
+def make_ubuntu_base():
+    if os.path.exists("base_paths"):
+        with open("base_paths", "rb") as file_:
+            base_paths = pickle.load(file_)
+    else:
+        test_distributions.build_ubuntu()
+        base_paths = relative_paths("ubuntu", all_paths("ubuntu"))
+        base_paths = filter_paths(base_paths, "python3")
+        with open("base_paths", "wb") as file_:
+            pickle.dump(base_paths, file_)
+    return base_paths
+
+
+# FIX: This isn`t making the correct libunionpreload.
+# def make_libunionpreload():
+#     #See https://github.com/AppImage/AppImages/blob/master/recipes/meta/Recipe
+#     temp_dir = tempfile.mkdtemp()
+#     cmd("wget -q https://raw.githubusercontent.com/mikix/deb2snap/"
+#         "blob/847668c4a89e2d4a1711fe062a4bae0c7ab81bd0/src/preload.c "
+#         "-O - | sed -e 's|SNAPPY|UNION|g' | sed -e 's|SNAPP|UNION|g' | "
+#         "sed -e 's|SNAP|UNION|g' | sed -e 's|snappy|union|g' "
+#         "> %s/libunionpreload.c" % temp_dir)
+#     cmd("gcc -shared -fPIC %s/libunionpreload.c -o libunionpreload.so "
+#         '-ldl -DUNION_LIBNAME="libunionpreload.so"' % temp_dir)
+#     cmd("strip libunionpreload.so")
+
+
+def make_app_dir(app_dir, new_paths):
+    os.mkdir(app_dir)
+    make_sub_container("ubuntu", app_dir, new_paths)
+    cmd("cp -a %s/tests %s" % (VIGIL_PATH, app_dir))
+    cmd("cp -a %s/test-all %s/tests" % (VIGIL_PATH, app_dir))
+    cmd("cp %s/appimage/* %s" % (VIGIL_PATH, app_dir))
+    # if not os.path.exists("libunionpreload.so"):
+    #     make_libunionpreload()
+    # cmd("cp libunionpreload.so " + app_dir)
+
+
+def make_appimage(app_dir):
+    cmd("wget --continue https://github.com/AppImage/AppImageKit/releases/"
+        "download/continuous/appimagetool-x86_64.AppImage")
+    cmd("chmod +x appimagetool-x86_64.AppImage")
+    cmd("./appimagetool-x86_64.AppImage " + app_dir)
+
+
+def main(work_path):
+    assert os.getuid() == 0 and os.getgid() == 0, "Need to be root."
+    os.chdir(work_path)
+    base_paths = make_ubuntu_base()
+    test_distributions.run_in_container("ubuntu", "./install-dependencies")
+    test_distributions.run_in_container("ubuntu", "pip3 install -I .")
+    post_install_paths = relative_paths("ubuntu", all_paths("ubuntu"))
+    new_paths = set(post_install_paths) - set(base_paths)
+    new_paths = filter_paths(new_paths, "/var/cache/apt/archives")
+    app_dir = "vigil.AppDir"
+    if os.path.exists(app_dir):
+        cmd("sudo rm -rf " + app_dir)
+    make_app_dir(app_dir, new_paths)
+    make_appimage(app_dir)
+
+
+if __name__ == "__main__":
+    work_path = (tempfile.mkdtemp(prefix="make-appimage2-")
+                 if len(sys.argv) == 1 else sys.argv[1])
+    main(work_path)

test_distributions.py

@@ -126,9 +126,9 @@ def build_gentoo():
 
 
 def main():
-    WORK_PATH = (tempfile.mkdtemp(prefix="vigil-") if len(sys.argv) == 1
+    work_path = (tempfile.mkdtemp(prefix="test_distributions-")
-                 else sys.argv[1])
+                 if len(sys.argv) == 1 else sys.argv[1])
-    os.chdir(WORK_PATH)
+    os.chdir(work_path)
     cmd("sudo apt-get install -y systemd-container debootstrap xz-utils wget")
     # FIX: Reenable: fedora debian archlinux opensuse pixel gentoo
     for distribution in ["ubuntu"]:
@@ -151,7 +151,7 @@ def main():
             globals()["remove_" + distribution]()
         except KeyError:
             cmd("sudo rm -rf " + distribution)
-    os.rmdir(WORK_PATH)
+    os.rmdir(work_path)
     print("Finished.")
 
 

vigil/worker.py

@@ -18,14 +18,11 @@ class Worker:
         self.result = None
         self.process = None
         self.child_pgid = None
-        self.script_path = (os.path.join(
-            os.environ["APPDIR"], "usr", "bin", "vigil-worker")
-                            if "APPDIR" in os.environ else "vigil-worker")
 
     @asyncio.coroutine
     def create_process(self):
         create = asyncio.create_subprocess_exec(
-            self.script_path, stdin=asyncio.subprocess.PIPE,
+            "vigil-worker", stdin=asyncio.subprocess.PIPE,
             stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
             preexec_fn=os.setsid)
         self.process = yield from create