Merge macOS compatibility updates from rgov/starter-pack

.gitignore

@@ -71,7 +71,7 @@ target/
 # Operating System Files
 # =========================
 
-# OSX
+# macOS
 # =========================
 
 .DS_Store

README.md

@@ -6,7 +6,7 @@ This project is NOT an actual pack, or intended for public use.
 There is *no support* for using this tool - it is designed for my own use,
 and released in the hope that others might find it useful.
 Bug reports are most welcome; feature requests are not
-(missing OS support is considered a bug - it should work on Windows, OSX, and Linux).
+(missing OS support is considered a bug - it should work on Windows, macOS, and Linux).
 
 **What it does:**
 
@@ -36,13 +36,10 @@ For anyone using these tools to assemble their own pack:
   set up and configured via these files, which are also commented.
 
 - You will need Python 3.5+, as I make extensive use of several
-  new features.  You will also need the `requests` and `pyaml`
-  libraries (both can be installed with `pip`).
-
-  Optional dependencies to unpack exotic archive types may be
-  added in future, but will not be required.
+  new features.  Dependencies can be installed with
+  `pip install -r requirements.txt`.
 
 - Many items in the provided config will only work on Windows
   (or when building for windows on another OS; tested on Debian).
-  If you are interested in helping support OSX or Linux, please
+  If you are interested in helping support macOS or Linux, please
   get in touch with my handle at gmail.

components.yml

@@ -26,6 +26,9 @@ files:
         os-linux:
             extract_to: |
                 PyLNP:build/starter-pack-launcher
+        os-osx:
+            extract_to: |
+                PyLNP.app:build/Starter Pack Launcher (PyLNP).app
     Stocksettings presets:
         bay12: 146213
         ident: 10170
@@ -41,6 +44,9 @@ files:
         os-linux:
             extract_to: |
                 {DFHACK_VER}/twbt.plug.so:plugins/
+        os-osx:
+            extract_to: |
+                {DFHACK_VER}/twbt.plug.dylib:plugins/
         needs_dfhack: True
         install_after: DFHack
     Quickfort_64:
@@ -70,6 +76,7 @@ utilities:
         manifest:
             tooltip: Announcement Window:Shows live announcements and combat reports in a separate window.
     Armok Vision:
+        requires_os: win linux
         bay12: 146473
         ident: JapaMala/armok-vision
         needs_dfhack: True
@@ -114,6 +121,7 @@ utilities:
         ident: robertjanetzko/LegendsBrowser
         manifest:
             tooltip: An in-browser legends utility, available for all operating systems.
+            osx_exe: Legends Browser.app/Contents/MacOS/JavaAppLauncher
     Legends Viewer:
         requires_os: win
         bay12: 154617
@@ -149,6 +157,7 @@ utilities:
         manifest:
             tooltip: A Sound+Music engine for DF - it reads the gamelog, and plays appropriate sounds and seasonal music.
             linux_exe: soundSense.sh
+            osx_exe: soundSense.sh
     World Viewer:
         requires_os: win
         bay12: 128932

requirements.txt

@@ -0,0 +1,3 @@
+pyyaml==3.13
+rarfile==3.0
+requests==2.19.1

starterpack/build.py

@@ -191,12 +191,12 @@ def teardown(message):
 
 
 def _exes_for(util):
-    """Find the best available match for Windows and OSX utilities."""
+    """Find the best available match for Windows and macOS utilities."""
     win_exe, osx_exe, linux_exe = '', '', ''
     for _, dirs, files in os.walk(paths.utilities(util.name)):
         # Windows: first .exe found, first .bat otherwise
         # Linux: first .jar found, otherwise .sh
-        # OSX: as for linux, but a .app directory wins
+        # macOS: as for linux, but a .app directory wins
         for f in files:
             if win_exe and osx_exe and linux_exe:
                 break
@@ -237,7 +237,8 @@ def create_utilities():
         if paths.HOST_OS != 'win':
             with open(paths.utilities(util.name, 'manifest.json')) as f:
                 exe = json.load(f)[paths.HOST_OS + '_exe']
-            os.chmod(exe, 0o110 | os.stat(exe).st_mode)
+            path = paths.utilities(util.name, exe)
+            os.chmod(path, 0o110 | os.stat(path).st_mode)
 
 
 # Configure graphics packs

starterpack/extract.py

@@ -23,10 +23,17 @@ def _copyfile(src, dest):
     """Copy the source file path or object to the dest path, creating dirs."""
     os.makedirs(os.path.dirname(dest), exist_ok=True)
     if isinstance(src, str):
-        shutil.copy2(src, dest)
-    else:
+        if os.path.isfile(src):
+            shutil.copy2(src, dest)
+        elif os.path.isdir(src):
+            copy_tree(src, dest, preserve_symlinks=True)
+        else:
+            raise IOError('Unexpected file type for %s' % src)
+    elif isinstance(src, zipfile.ZipExtFile):
         with open(dest, 'wb') as out:
             shutil.copyfileobj(src, out)
+    else:
+        raise NotImplementedException('Unexpected source type %s' % type(src))
 
 
 def unzip_to(filename, target_dir=None, path_pairs=None):
@@ -52,9 +59,9 @@ def unzip_to(filename, target_dir=None, path_pairs=None):
         files = dict(a for a in zip(zf.namelist(), zf.infolist())
                      if not a[0].endswith('/'))
         prefix = os.path.commonpath(list(files)) if len(files) > 1 else ''
-        for name in files:
+        for name, info in files.items():
             out = os.path.join(target_dir, os.path.relpath(name, prefix))
-            _copyfile(zf.open(files[name]), out)
+            _copyfile(zf.open(info), out)
 
 
 def nonzip_extract(filename, target_dir=None, path_pairs=None):
@@ -64,7 +71,7 @@ def nonzip_extract(filename, target_dir=None, path_pairs=None):
 
     Involves a lot of shelling out, as Python's `tarfile` cannot open
     the .tar.bz2 archived DF releases (complicated header issue).
-    OSX disk images (.dmg) are also unsupported by Python.
+    macOS disk images (.dmg) are also unsupported by Python.
     """
     if filename.endswith('.exe') and paths.HOST_OS == 'win' \
             or filename.endswith('.jar'):
@@ -79,26 +86,47 @@ def nonzip_extract(filename, target_dir=None, path_pairs=None):
         files = [os.path.join(root, f)
                  for root, _, files in os.walk(tmpdir) for f in files]
         prefix = os.path.commonpath(files) if len(files) > 1 else ''
+        # BAD HACK: It's an unsafe assumption (made above) that the common
+        # path should always be stripped.
+        if filename.endswith('.dmg') and 'Legends Browser' in target_dir:
+            prefix = ''
         if target_dir:
-            copy_tree(os.path.join(tmpdir, prefix), target_dir)
+            copy_tree(os.path.join(tmpdir, prefix), target_dir, preserve_symlinks=True)
         else:
             for inpath, outpath in path_pairs:
                 if outpath.endswith('/'):
                     outpath += os.path.basename(inpath)
-                if os.path.isfile(os.path.join(tmpdir, prefix, inpath)):
+                if os.path.exists(os.path.join(tmpdir, prefix, inpath)):
                     _copyfile(os.path.join(tmpdir, prefix, inpath), outpath)
                 else:
                     print('WARNING:  "{}" not found in "{}"'.format(
                           inpath, os.path.basename(filename)))
     return True
 
 
+def unpack_dmg(filename, dest):
+    """Extract a .dmg disk image on macOS into the dest dir."""
+    assert filename.endswith('.dmg') and paths.HOST_OS == 'osx'
+    with tempfile.TemporaryDirectory() as tmpdir:
+        try:
+            subprocess.check_call(['hdiutil', 'attach', '-quiet', '-readonly',
+                                   '-nobrowse', '-mountpoint', tmpdir,
+                                   filename])
+        except subprocess.CalledProcessError:
+            print('Failed to mount', filename, ' -- is it already mounted?')
+            raise
+        try:
+            copy_tree(tmpdir, dest, preserve_symlinks=True)
+        finally:
+            subprocess.check_call(['hdiutil', 'detach', '-quiet', tmpdir])
+
+
+
 def unpack_anything(filename, tmpdir):
     """Extract practically any archive format from src file to dest dir."""
     if filename.endswith('.dmg') and paths.HOST_OS == 'osx':
-        # TODO:  support .dmg extraction via shell on OSX
-        raise NotImplementedError(
-            'TODO: mount .dmg, copy contents to tmpdir, unmount')
+        unpack_dmg(filename, tmpdir)
+        return True
     elif zipfile.is_zipfile(filename):
         # Uses fast version above; handled here for completeness
         zipfile.ZipFile(filename).extractall(tmpdir)
@@ -142,8 +170,8 @@ def extract_comp(pool, comp):
         return pool.submit(unzip_to, comp.path, getattr(paths, dest)(*details))
     # else using the path_pairs option; extract pairs from string
     pairs = []
-    for pair in comp.extract_to.strip().split('\n'):
-        src, to = pair.split(':')
+    for pair in comp.extract_to.strip().splitlines():
+        src, _, to = pair.partition(':')
         dest, *details = to.split('/')
         # Note: can add format variables here as needed
         if '{DFHACK_VER}' in src:
@@ -197,7 +225,7 @@ def add_lnp_dirs():
     """Install the LNP subdirs that I can't create automatically."""
     # Should use https://github.com/Lazy-Newb-Pack/LNP-shared-core someday...
     for d in ('colors', 'embarks', 'extras', 'keybinds', 'tilesets'):
-        copy_tree(paths.base(d), paths.lnp(d))
+        copy_tree(paths.base(d), paths.lnp(d), preserve_symlinks=True)
 
 
 def main():

starterpack/metadata_api.py

@@ -26,7 +26,7 @@ def get_auth():
     return None
 
 
-def cache(method=lambda *_: None, *, saved={}, dump=False):
+def cache(method=lambda *_: None, *, saved={}, dump=False, expiration=30*60):
     """A caching decorator.
 
     Reads cache from local file if cache is empty.
@@ -37,10 +37,11 @@ def cache(method=lambda *_: None, *, saved={}, dump=False):
         try:
             with open('_cached.yml') as f:
                 saved.update(yaml.load(f))
+                print('Loaded metadata from cache file')
         except IOError:
-            print('Downloading metadata for components...\n')
             saved.update({'metadata': {}, 'timestamps': {}})
     elif dump:
+        print('Saving metadata to cache file')
         with open('_cached.yml', 'w') as f:
             yaml.dump(saved, f, indent=4)
 
@@ -49,12 +50,14 @@ def wrapper(self, ident):
         if isinstance(self, GitHubAssetMetadata):
             key = (not paths.ARGS.stable, ident)
             args = (self, ident, saved['timestamps'].get(key, 0),
-                    saved['metadata'].get(ident))
-        if (time.time() - saved['timestamps'].get(key, 0)) > 30*60:
+                    saved['metadata'].get(key))
+        if (time.time() - saved['timestamps'].get(key, 0)) > expiration:
+            print('Refreshing metadata for package', ident)
             new_json = method(*args)
-            if new_json is not None:
-                saved['metadata'][key] = new_json
-                saved['timestamps'][key] = time.time()
+            if new_json is None:
+                raise RuntimeError('Failed to get metadata for', ident)
+            saved['metadata'][key] = new_json
+            saved['timestamps'][key] = time.time()
         return saved['metadata'].get(key)
     return wrapper