spack · scheibelp · Sep 13, 2022 · Sep 7, 2022 · Jul 22, 2022 · Aug 4, 2022
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py
@@ -22,7 +22,7 @@
 from llnl.util import tty
 from llnl.util.compat import Sequence
 from llnl.util.lang import dedupe, memoized
-from llnl.util.symlink import symlink
+from llnl.util.symlink import islink, symlink
 
 from spack.util.executable import Executable
 from spack.util.path import path_to_os_path, system_path_filter
@@ -637,7 +637,11 @@ def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False):
                 if symlinks:
                     target = os.readlink(s)
                     if os.path.isabs(target):
-                        new_target = re.sub(abs_src, abs_dest, target)
+
+                        def escaped_path(path):
+                            return path.replace("\\", r"\\")
+
+                        new_target = re.sub(escaped_path(abs_src), escaped_path(abs_dest), target)
                         if new_target != target:
                             tty.debug("Redirecting link {0} to {1}".format(target, new_target))
                             target = new_target
@@ -1903,7 +1907,11 @@ def names(self):
                 name = x[3:]
 
             # Valid extensions include: ['.dylib', '.so', '.a']
-            for ext in [".dylib", ".so", ".a"]:
+            # on non Windows platform
+            # Windows valid library extensions are:
+            # ['.dll', '.lib']
+            valid_exts = [".dll", ".lib"] if is_windows else [".dylib", ".so", ".a"]
+            for ext in valid_exts:
                 i = name.rfind(ext)
                 if i != -1:
                     names.append(name[:i])
@@ -2046,15 +2054,23 @@ def find_libraries(libraries, root, shared=True, recursive=False):
         message = message.format(find_libraries.__name__, type(libraries))
         raise TypeError(message)
 
+    if is_windows:
+        static = "lib"
+        shared = "dll"
+    else:
+        # Used on both Linux and macOS
+        static = "a"
+        shared = "so"
+
     # Construct the right suffix for the library
     if shared:
         # Used on both Linux and macOS
-        suffixes = ["so"]
+        suffixes = [shared]
         if sys.platform == "darwin":
             # Only used on macOS
             suffixes.append("dylib")
     else:
-        suffixes = ["a"]
+        suffixes = [static]
 
     # List of libraries we are searching with suffixes
     libraries = ["{0}.{1}".format(lib, suffix) for lib in libraries for suffix in suffixes]
@@ -2067,7 +2083,11 @@ def find_libraries(libraries, root, shared=True, recursive=False):
     # perform first non-recursive search in root/lib then in root/lib64 and
     # finally search all of root recursively. The search stops when the first
     # match is found.
-    for subdir in ("lib", "lib64"):
+    common_lib_dirs = ["lib", "lib64"]
+    if is_windows:
+        common_lib_dirs.extend(["bin", "Lib"])
+
+    for subdir in common_lib_dirs:
         dirname = join_path(root, subdir)
         if not os.path.isdir(dirname):
             continue
@@ -2080,6 +2100,155 @@ def find_libraries(libraries, root, shared=True, recursive=False):
     return LibraryList(found_libs)
 
 
+def find_all_shared_libraries(root, recursive=False):
+    """Convenience function that returns the list of all shared libraries found
+    in the directory passed as argument.
+
+    See documentation for `llnl.util.filesystem.find_libraries` for more information
+    """
+    return find_libraries("*", root=root, shared=True, recursive=recursive)
+
+
+def find_all_static_libraries(root, recursive=False):
+    """Convenience function that returns the list of all static libraries found
+    in the directory passed as argument.
+
+    See documentation for `llnl.util.filesystem.find_libraries` for more information
+    """
+    return find_libraries("*", root=root, shared=False, recursive=recursive)
+
+
+def find_all_libraries(root, recursive=False):
+    """Convenience function that returns the list of all libraries found
+    in the directory passed as argument.
+
+    See documentation for `llnl.util.filesystem.find_libraries` for more information
+    """
+
+    return find_all_shared_libraries(root, recursive=recursive) + find_all_static_libraries(
+        root, recursive=recursive
+    )
+
+
+class WindowsSimulatedRPath(object):
+    """Class representing Windows filesystem rpath analog
+
+    One instance of this class is associated with a package (only on Windows)
+    For each lib/binary directory in an associated package, this class introduces
+    a symlink to any/all dependent libraries/binaries. This includes the packages
+    own bin/lib directories, meaning the libraries are linked to the bianry directory
+    and vis versa.
+    """
+
+    def __init__(self, package, link_install_prefix=True):
+        """
+        Args:
+            package (spack.package_base.PackageBase): Package requiring links
+            link_install_prefix (bool): Link against package's own install or stage root.
+                Packages that run their own executables during build and require rpaths to
+                the build directory during build time require this option. Default: install
+                root
+        """
+        self.pkg = package
+        self._addl_rpaths = set()
+        self.link_install_prefix = link_install_prefix
+        self._internal_links = set()
+
+    @property
+    def link_dest(self):
+        """
+        Set of directories where package binaries/libraries are located.
+        """
+        if hasattr(self.pkg, "libs") and self.pkg.libs:
+            pkg_libs = set(self.pkg.libs.directories)
+        else:
+            pkg_libs = set((self.pkg.prefix.lib, self.pkg.prefix.lib64))
+
+        return pkg_libs | set([self.pkg.prefix.bin]) | self.internal_links
+
+    @property
+    def internal_links(self):
+        """
+        linking that would need to be established within the package itself. Useful for links
+        against extension modules/build time executables/internal linkage
+        """
+        return self._internal_links
+
+    def add_internal_links(self, *dest):
+        """
+        Incorporate additional paths into the rpath (sym)linking scheme.
+
+        Paths provided to this method are linked against by a package's libraries
+        and libraries found at these paths are linked against a package's binaries.
+        (i.e. /site-packages -> /bin and /bin -> /site-packages)
+
+        Specified paths should be outside of a package's lib, lib64, and bin
+        directories.
+        """
+        self._internal_links = self._internal_links | set(*dest)
+
+    @property
+    def link_targets(self):
+        """
+        Set of libraries this package needs to link against during runtime
+        These packages will each be symlinked into the packages lib and binary dir
+        """
+
+        dependent_libs = []
+        for path in self.pkg.rpath:
+            dependent_libs.extend(list(find_all_shared_libraries(path, recursive=True)))
+        for extra_path in self._addl_rpaths:
+            dependent_libs.extend(list(find_all_shared_libraries(extra_path, recursive=True)))
+        return set(dependent_libs)
+
+    def include_additional_link_paths(self, *paths):
+        """
+        Add libraries found at the root of provided paths to runtime linking
+
+        These are libraries found outside of the typical scope of rpath linking
+        that require manual inclusion in a runtime linking scheme
+
+        Args:
+            *paths (str): arbitrary number of paths to be added to runtime linking
+        """
+        self._addl_rpaths = self._addl_rpaths | set(paths)
+
+    def establish_link(self):
+        """
+        (sym)link packages to runtime dependencies based on RPath configuration for
+        Windows heuristics
+        """
+        # from build_environment.py:463
+        # The top-level package is always RPATHed. It hasn't been installed yet
+        # so the RPATHs are added unconditionally
+
+        # for each binary install dir in self.pkg (i.e. pkg.prefix.bin, pkg.prefix.lib)
+        # install a symlink to each dependent library
+        for library, lib_dir in itertools.product(self.link_targets, self.link_dest):
+            if not path_contains_subdirectory(library, lib_dir):
+                file_name = os.path.basename(library)
+                dest_file = os.path.join(lib_dir, file_name)
+                if os.path.exists(lib_dir):
+                    try:
+                        symlink(library, dest_file)
+                    # For py2 compatibility, we have to catch the specific Windows error code
+                    # associate with trying to create a file that already exists (winerror 183)
+                    except OSError as e:
+                        if e.winerror == 183:
+                            # We have either already symlinked or we are encoutering a naming clash
+                            # either way, we don't want to overwrite existing libraries
+                            already_linked = islink(dest_file)
+                            tty.debug(
+                                "Linking library %s to %s failed, " % (library, dest_file)
+                                + "already linked."
+                                if already_linked
+                                else "library with name %s already exists." % file_name
+                            )
+                            pass
+                        else:
+                            raise e
+
+
 @system_path_filter
 @memoized
 def can_access_dir(path):

diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py
@@ -84,6 +84,9 @@
 #: queue invariants).
 STATUS_REMOVED = "removed"
 
+is_windows = sys.platform == "win32"
+is_osx = sys.platform == "darwin"
+
 
 class InstallAction(object):
     #: Don't perform an install
@@ -165,7 +168,9 @@ def _do_fake_install(pkg):
     if not pkg.name.startswith("lib"):
         library = "lib" + library
 
-    dso_suffix = ".dylib" if sys.platform == "darwin" else ".so"
+    plat_shared = ".dll" if is_windows else ".so"
+    plat_static = ".lib" if is_windows else ".a"
+    dso_suffix = ".dylib" if is_osx else plat_shared
 
     # Install fake command
     fs.mkdirp(pkg.prefix.bin)
@@ -180,7 +185,7 @@ def _do_fake_install(pkg):
 
     # Install fake shared and static libraries
     fs.mkdirp(pkg.prefix.lib)
-    for suffix in [dso_suffix, ".a"]:
+    for suffix in [dso_suffix, plat_static]:
         fs.touch(os.path.join(pkg.prefix.lib, library + suffix))
 
     # Install fake man page
@@ -1214,7 +1219,10 @@ def _install_task(self, task):
             spack.package_base.PackageBase._verbose = spack.build_environment.start_build_process(
                 pkg, build_process, install_args
             )
-
+            # Currently this is how RPATH-like behavior is achieved on Windows, after install
+            # establish runtime linkage via Windows Runtime link object
+            # Note: this is a no-op on non Windows platforms
+            pkg.windows_establish_runtime_linkage()
             # Note: PARENT of the build process adds the new package to
             # the database, so that we don't need to re-read from file.
             spack.store.db.add(pkg.spec, spack.store.layout, explicit=explicit)

diff --git a/lib/spack/spack/package_base.py b/lib/spack/spack/package_base.py
@@ -97,6 +97,9 @@
 _spack_configure_argsfile = "spack-configure-args.txt"
 
 
+is_windows = sys.platform == "win32"
+
+
 def preferred_version(pkg):
     """
     Returns a sorted list of the preferred versions of the package.
@@ -182,6 +185,30 @@ def copy(self):
             return other
 
 
+class WindowsRPathMeta(object):
+    """Collection of functionality surrounding Windows RPATH specific features
+
+    This is essentially meaningless for all other platforms
+    due to their use of RPATH. All methods within this class are no-ops on
+    non Windows. Packages can customize and manipulate this class as
+    they would a genuine RPATH, i.e. adding directories that contain
+    runtime library dependencies"""
+
+    def add_search_paths(self, *path):
+        """Add additional rpaths that are not implicitly included in the search
+        scheme
+        """
+        self.win_rpath.include_additional_link_paths(*path)
+
+    def windows_establish_runtime_linkage(self):
+        """Establish RPATH on Windows
+
+        Performs symlinking to incorporate rpath dependencies to Windows runtime search paths
+        """
+        if is_windows:
+            self.win_rpath.establish_link()
+
+
 #: Registers which are the detectable packages, by repo and package name
 #: Need a pass of package repositories to be filled.
 detectable_packages = collections.defaultdict(list)
@@ -221,7 +248,7 @@ def to_windows_exe(exe):
                 plat_exe = []
                 if hasattr(cls, "executables"):
                     for exe in cls.executables:
-                        if sys.platform == "win32":
+                        if is_windows:
                             exe = to_windows_exe(exe)
                         plat_exe.append(exe)
                 return plat_exe
@@ -513,7 +540,7 @@ def test_log_pathname(test_stage, spec):
     return os.path.join(test_stage, "test-{0}-out.txt".format(TestSuite.test_pkg_id(spec)))
 
 
-class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)):
+class PackageBase(six.with_metaclass(PackageMeta, WindowsRPathMeta, PackageViewMixin, object)):
     """This is the superclass for all spack packages.
 
     ***The Package class***
@@ -753,6 +780,8 @@ def __init__(self, spec):
         # Set up timing variables
         self._fetch_time = 0.0
 
+        self.win_rpath = fsys.WindowsSimulatedRPath(self)
+
         if self.is_extension:
             pkg_cls = spack.repo.path.get_pkg_class(self.extendee_spec.name)
             pkg_cls(self.extendee_spec)._check_extendable()
@@ -2754,6 +2783,8 @@ def rpath(self):
         deps = self.spec.dependencies(deptype="link")
         rpaths.extend(d.prefix.lib for d in deps if os.path.isdir(d.prefix.lib))
         rpaths.extend(d.prefix.lib64 for d in deps if os.path.isdir(d.prefix.lib64))
+        if is_windows:
+            rpaths.extend(d.prefix.bin for d in deps if os.path.isdir(d.prefix.bin))
         return rpaths
 
     @property

diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py
@@ -27,7 +27,7 @@
 
 import llnl.util.lang
 import llnl.util.tty as tty
-from llnl.util.filesystem import mkdirp, remove_linked_tree, working_dir
+from llnl.util.filesystem import copy_tree, mkdirp, remove_linked_tree, working_dir
 
 import spack.binary_distribution
 import spack.caches
@@ -803,7 +803,7 @@ def mock_store(tmpdir_factory, mock_repo_path, mock_configuration_scopes, _store
             with spack.store.use_store(str(store_path)) as store:
                 with spack.repo.use_repositories(mock_repo_path):
                     _populate(store.db)
-        store_path.copy(store_cache, mode=True, stat=True)
+        copy_tree(str(store_path), str(store_cache))
 
     # Make the DB filesystem read-only to ensure we can't modify entries
     store_path.join(".spack-db").chmod(mode=0o555, rec=1)
@@ -844,7 +844,7 @@ def mutable_database(database_mutable_config, _store_dir_and_cache):
     # Restore the initial state by copying the content of the cache back into
     # the store and making the database read-only
     store_path.remove(rec=1)
-    store_cache.copy(store_path, mode=True, stat=True)
+    copy_tree(str(store_cache), str(store_path))
     store_path.join(".spack-db").chmod(mode=0o555, rec=1)
 
 

diff --git a/lib/spack/spack/test/data/directory_search/a/libc.dll b/lib/spack/spack/test/data/directory_search/a/libc.dll
diff --git a/lib/spack/spack/test/data/directory_search/a/libc.lib b/lib/spack/spack/test/data/directory_search/a/libc.lib
diff --git a/lib/spack/spack/test/data/directory_search/b/liba.dll b/lib/spack/spack/test/data/directory_search/b/liba.dll
diff --git a/lib/spack/spack/test/data/directory_search/b/liba.lib b/lib/spack/spack/test/data/directory_search/b/liba.lib
diff --git a/lib/spack/spack/test/data/directory_search/b/libd.dll b/lib/spack/spack/test/data/directory_search/b/libd.dll
diff --git a/lib/spack/spack/test/data/directory_search/b/libd.lib b/lib/spack/spack/test/data/directory_search/b/libd.lib
diff --git a/lib/spack/spack/test/data/directory_search/c/libb.dll b/lib/spack/spack/test/data/directory_search/c/libb.dll
diff --git a/lib/spack/spack/test/data/directory_search/c/libb.lib b/lib/spack/spack/test/data/directory_search/c/libb.lib