lots of little tweaks for compiling with pypy

ChangeLog.txt

@@ -4,9 +4,23 @@ v0.9.0
   THIS RELEASE INTRODUCES A NEW ON-DISK FILE LAYOUT - SEE BELOW FOR NOTES
 
     * Move the version directories, updates directory etc into an "appdata"
-      subdirectory, to avoid user confusion in the top-level directory.
-         * there are compatability hooks for the old file layout, but they
-           will be removed at the end of the 0.9.* series.
+      subdirectory, to avoid user confusion in the top-level directory. The
+      hope is that users will be much less inclined to poke around inside a
+      generic "appdata" directory than inside directories named after the app.
+         * to be clear, the new layout looks like this:
+               myapp.exe
+               appdata/
+                 myapp-X.Y.X.win32/
+                   myapp.exe
+                   esky-files/
+                 updates/
+         * the "bdist_esky" command still generates zipfiles using the old 
+           layout, so they can be unpacked correctly by previous versions of
+           esky.  To get the new layout, pass the "enable-appdata-dir" option.
+           This option will become the default (and will be removed) at the
+           end of the 0.9.* series.
+         * there are compatability hooks that upgrade the old file layout to
+           the new layout. They will be removed at the end of the 0.9.* series.
          * therefore, if you have deployed applications using esky 0.8.* or
            lower, you MUST deploy an update using the 0.9.* series before
            migrating to any newer versions of esky.
@@ -17,12 +31,15 @@ v0.9.0
             enable the compilation step.
           * any custom bootstrapping code must be valid RPython; check the
             variable __rpython__ to determine whether you're being compiled.
-          * obviously this only works if a recent "pypy" package is installed.
-    * Add "verify()" function to the bootstrap env, which is called to verify
+          * obviously this only works if a recent version of the "pypy" package
+            is installed and configured correctly.
+    * Add verify() function to the bootstrap env, which is called to verify
       files before they are chainloaded.  The default implementation is empty,
       but it could be overridden to perform e.g. signature checking.
     * py2exe:  never bundle the python dll into the bootstrap executable
-      unless it was bundled into the frozen app.
+      unless it was bundled into the frozen app, it caused too many issues
+      with unsuspecting third-party libraries.
+
 
 v0.8.5:
 
@@ -35,6 +52,7 @@ v0.8.5:
       strings are raw code, and all else is passed to inspect.getsource().
     * Made bdist_esky.Executable a subclass of unicode rather than str.
 
+
 v0.8.4:
 
     * changed to a decorator-based syntax for declaring lazy imports, it's

esky/__init__.py

@@ -154,7 +154,8 @@
                       is_version_dir, is_uninstalled_version_dir,\
                       parse_version, get_best_version, appdir_from_executable,\
                       copy_ownership_info, lock_version_dir, ESKY_CONTROL_DIR,\
-                      files_differ, lazy_import
+                      files_differ, lazy_import, ESKY_APPDATA_DIR, \
+                      get_all_versions
 
 #  Since all frozen apps are required to import this module and call the
 #  run_startup_hooks() function, we use a simple lazy import mechanism to 
@@ -259,8 +260,8 @@ def _init_from_appdir(self,appdir_or_exe):
         """
         if os.path.isfile(appdir_or_exe):
             self.appdir = appdir_from_executable(appdir_or_exe)
-            vdir = appdir_or_exe[len(self.appdir):].split(os.sep)[1]
-            #vdir = appdir_or_exe[len(self.appdir):].split(os.sep)[2]
+            vsdir = self._get_versions_dir()
+            vdir = appdir_or_exe[len(vsdir):].split(os.sep)[1]
             details = split_app_version(vdir)
             self.name,self.active_version,self.platform = details
         else:
@@ -280,20 +281,38 @@ def _set_version_finder(self,version_finder):
 
     def _get_update_dir(self):
         """Get the directory path in which self.version_finder can work."""
-        return os.path.join(self.appdir,"appdata","updates")
+        # TODO: remove compatability hooks for ESKY_APPDATA_DIR=""
+        try:
+            if os.listdir(os.path.join(self.appdir,ESKY_APPDATA_DIR)):
+                return os.path.join(self.appdir,ESKY_APPDATA_DIR,"updates")
+        except EnvironmentError:
+            return os.path.join(self.appdir,"updates")
+        else:
+            return os.path.join(self.appdir,"updates")
 
     def _get_versions_dir(self):
         """Get the directory path containing individual version dirs."""
-        #return os.path.join(self.appdir,"appdata")
-        return self.appdir
+        if not ESKY_APPDATA_DIR:
+            return self.appdir
+        # TODO: remove compatability hooks for ESKY_APPDATA_DIR=""
+        try:
+            if os.listdir(os.path.join(self.appdir,ESKY_APPDATA_DIR)):
+                return os.path.join(self.appdir,ESKY_APPDATA_DIR)
+        except EnvironmentError:
+            return self.appdir
+        else:
+            return self.appdir
 
     def get_abspath(self,relpath):
         """Get the absolute path of a file within the current version."""
         if self.active_version:
             v = join_app_version(self.name,self.active_version,self.platform)
         else:
             v = join_app_version(self.name,self.version,self.platform)
-        return os.path.join(self._get_versions_dir(),v,relpath)
+        # TODO: remove compatability hooks for ESKY_APPDATA_DIR=""
+        if os.path.exists(os.path.join(self._get_versions_dir(),v)):
+            return os.path.join(self._get_versions_dir(),v,relpath)
+        return os.path.join(self.appdir,v,relpath)
 
     def reinitialize(self):
         """Reinitialize internal state by poking around in the app directory.
@@ -496,6 +515,20 @@ def _cleanup_actions(self):
             (_,v,_) = split_app_version(new_version)
             yield (self.install_version,(v,))
             best_version = new_version
+        # TODO: remove compatability hooks for ESKY_APPDATA_DIR="".
+        if ESKY_APPDATA_DIR:
+            appdata_dir = os.path.join(self.appdir,ESKY_APPDATA_DIR)
+            if not os.path.exists(appdata_dir):
+                yield (os.makedirs, (appdata_dir,))
+            if not os.listdir(appdata_dir):
+                #  Careful to move all versions in order from newest to
+                #  oldest, so if we're interrupted we can safely launch
+                #  the best version from the new appdata directory.
+                for nm in get_all_versions(self.appdir) + ["updates"]:
+                    if os.path.exists(os.path.join(self.appdir,nm)):
+                        yield (os.rename, (os.path.join(self.appdir,nm),
+                                           os.path.join(vsdir,nm),))
+                vsdir = self._get_versions_dir()
         #  Now we can safely remove all the old versions.
         #  We except the currently-executing version, and silently
         #  ignore any locked versions.
@@ -506,6 +539,7 @@ def _cleanup_actions(self):
         if self.active_version and self.active_version != best_version:
             yield lambda: False
             manifest.add(self.active_version)
+        # TODO: remove compatability hooks for ESKY_APPDATA_DIR=""
         for tdir in (appdir,vsdir):
             for nm in os.listdir(tdir):
                 if nm not in manifest:
@@ -527,9 +561,15 @@ def _cleanup_actions(self):
                         #  It's a temporary backup file; remove it.
                         yield (self._try_remove, (tdir,nm,manifest,))
                     else:
-                        #  It's an unaccounted-for entry in the bootstrap env.
-                        #  Can't prove it's safe to remove, so leave it.
-                        pass
+                        for (_,_,filenms) in os.walk(fullnm):
+                            if filenms:
+                                #  It contains unaccounted-for files in the
+                                #  bootstrap env. Can't prove it's safe to
+                                #  remove, so leave it alone.
+                                break
+                        else:
+                            #  It's an empty directory structure, remove it.
+                            yield (self._try_remove, (tdir,nm,manifest,))
         #  If there are pending overwrites, try to do them.
         ovrdir = os.path.join(vsdir,best_version,ESKY_CONTROL_DIR,"overwrite")
         if os.path.exists(ovrdir):

esky/bdist_esky/__init__.py

@@ -32,7 +32,9 @@
 
 import esky.patch
 from esky.util import get_platform, is_core_dependency, create_zipfile, \
-                      split_app_version, join_app_version, ESKY_CONTROL_DIR
+                      split_app_version, join_app_version, ESKY_CONTROL_DIR, \
+                      ESKY_APPDATA_DIR
+
 if sys.platform == "win32":
     from esky import winres
     from xml.dom import minidom
@@ -239,9 +241,11 @@ class bdist_esky(Command):
                      "function to call just before starting to freeze the app"),
                     ('pre-zip-callback=', None,
                      "function to call just before starting to zip up the app"),
+                    ('enable-appdata-dir=', None,
+                     "enable new 'appdata' directory layout (will go away after the 0.9.X series)"),
                    ]
 
-    boolean_options = ["bundle-msvcrt","dont-run-startup-hooks","compile-bootstrap-exes"]
+    boolean_options = ["bundle-msvcrt","dont-run-startup-hooks","compile-bootstrap-exes","enable-appdata-dir"]
 
     def initialize_options(self):
         self.dist_dir = None
@@ -257,6 +261,7 @@ def initialize_options(self):
         self._compiled_exes = {}
         self.pre_freeze_callback = None
         self.pre_zip_callback = None
+        self.enable_appdata_dir = False
 
     def finalize_options(self):
         self.set_undefined_options('bdist',('dist_dir', 'dist_dir'))
@@ -320,10 +325,12 @@ def _run_initialise_dirs(self):
         platform = get_platform()
         self.bootstrap_dir = os.path.join(self.dist_dir,
                                           "%s.%s"%(fullname,platform,))
-        #self.freeze_dir = os.path.join(self.bootstrap_dir,"appdata",
-        #                               "%s.%s"%(fullname,platform,))
-        self.freeze_dir = os.path.join(self.bootstrap_dir,
-                                       "%s.%s"%(fullname,platform,))
+        if self.enable_appdata_dir:
+            self.freeze_dir = os.path.join(self.bootstrap_dir,ESKY_APPDATA_DIR,
+                                           "%s.%s"%(fullname,platform,))
+        else:
+            self.freeze_dir = os.path.join(self.bootstrap_dir,
+                                           "%s.%s"%(fullname,platform,))
         if os.path.exists(self.bootstrap_dir):
             shutil.rmtree(self.bootstrap_dir)
         os.makedirs(self.freeze_dir)
@@ -672,8 +679,7 @@ def compile_to_bootstrap_exe(self,exe,source):
         if COMPILED_BOOTSTRAP_CACHE is not None:
             outfile = os.path.join(COMPILED_BOOTSTRAP_CACHE,outname)
             if os.path.exists(outfile):
-                self.copy_to_bootstrap_env(outfile,exe.name)
-                return
+                return self.copy_to_bootstrap_env(outfile,exe.name)
         #  Otherwise we have to compile it anew.
         try:
             outfile = self._compiled_exes[source_hash]
@@ -685,14 +691,14 @@ def compile_to_bootstrap_exe(self,exe,source):
             opts = dict(gui_only=exe.gui_only)
             pypyc.compile_rpython(infile,outfile,**opts)
             self._compiled_exes[source_hash] = outfile
-        self.copy_to_bootstrap_env(outfile,exe.name)
         #  Try to save the compiled exe for future use.
         if COMPILED_BOOTSTRAP_CACHE is not None:
             cachedfile = os.path.join(COMPILED_BOOTSTRAP_CACHE,outname)
             try:
                 shutil.copy2(outfile,cachedfile)
             except EnvironmentError:
                 pass
+        return self.copy_to_bootstrap_env(outfile,exe.name)
 
     def copy_to_bootstrap_env(self,src,dst=None):
         """Copy the named file into the bootstrap environment.