Skip to content

add a way to run custom scripts for environment setup#1315

Draft
Neumann-A wants to merge 17 commits intomicrosoft:mainfrom
Neumann-A:add_env_setup_script
Draft

add a way to run custom scripts for environment setup#1315
Neumann-A wants to merge 17 commits intomicrosoft:mainfrom
Neumann-A:add_env_setup_script

Conversation

@Neumann-A
Copy link
Copy Markdown
Contributor

@Neumann-A Neumann-A commented Dec 30, 2023

Introduce: VCPKG_ENVIRONMENT_SETUP_SCRIPTS to run custom scripts instead or supplementing the vcvars set up environment.
If the script is a cmake file -> call cmake with the paths vcpkg would use -> allows downloading of stuff before vcpkg runs any other stuff like compiler detection etc.

# Conflicts:
#	include/vcpkg/base/system.process.h
#	include/vcpkg/commands.build.h
#	src/vcpkg/base/system.process.cpp
#	src/vcpkg/cmakevars.cpp
#	src/vcpkg/commands.build.cpp
#	src/vcpkg/vcpkgcmdarguments.cpp
@Neumann-A Neumann-A changed the title Draft: add a way to run custom scripts for environment setup add a way to run custom scripts for environment setup May 29, 2024
@Neumann-A Neumann-A marked this pull request as ready for review May 29, 2024 04:58
@Neumann-A
Copy link
Copy Markdown
Contributor Author

Neumann-A commented May 30, 2024

A bit of story what I did hear: I want to install&setup Intel (Fortran) compilers before running vcpkg and have successfully compiled lapack-reference with it.

Triplet x64-win:

set(VCPKG_TARGET_ARCHITECTURE x64)
set(VCPKG_CRT_LINKAGE dynamic)
set(VCPKG_LIBRARY_LINKAGE dynamic)

set(VCPKG_ENVIRONMENT_SETUP_SCRIPTS 
  "${CMAKE_CURRENT_LIST_DIR}/x64-win/download-tools.cmake"
  "${CMAKE_CURRENT_LIST_DIR}/x64-win/setup.bat" # Just calls out to generated scripts
)

# This below could be in a toolchain instead: (basically copied and adapted from one of my Fortran PRs)
set(PATH_SUFFIX "bin")
find_program(Fortran_COMPILER NAMES ifort ifort.exe 
  PATHS "${DOWNLOADS}/tools/intel.2024.1/compiler/latest"
  PATH_SUFFIXES "${PATH_SUFFIX}"
  NO_DEFAULT_PATH
)
if(NOT Fortran_COMPILER AND DEFINED CURRENT_PACKAGES_DIR)
  message(FATAL_ERROR "unable to find Fortran compiler!" )
endif()

set(mach_flag "/Qm64 /QxSSE4.2")

list(APPEND VCPKG_CMAKE_CONFIGURE_OPTIONS
  "-DCMAKE_TRY_COMPILE_PLATFORM_VARIABLES=CMAKE_Fortran_FLAGS\;CMAKE_Fortran_FLAGS_RELEASE\;CMAKE_Fortran_FLAGS_DEBUG\;CMAKE_Fortran_STANDARD_LIBRARIES\;CMAKE_EXE_LINKER_FLAGS\;CMAKE_EXE_LINKER_FLAGS_RELEASE\;CMAKE_EXE_LINKER_FLAGS_DEBUG\;CMAKE_STATIC_LINKER_FLAGS\;CMAKE_STATIC_LINKER_FLAGS_DEBUG\;CMAKE_STATIC_LINKER_FLAGS_RELEASE\;CMAKE_SHARED_LINKER_FLAGS\;CMAKE_SHARED_LINKER_FLAGS_RELEASE\;CMAKE_SHARED_LINKER_FLAGS_DEBUG\;CMAKE_REQUIRED_LINK_OPTIONS"
  "-DCMAKE_Fortran_COMPILER=${Fortran_COMPILER}"
  "-DCMAKE_Fortran_FLAGS_INIT:STRING=/Z7 /names:lowercase /assume:underscore /assume:protect_parens /fp:strict ${mach_flag} /Qopenmp-"
  "-DCMAKE_Fortran_FLAGS_DEBUG_INIT:STRING=/Od /Ob0"
  "-DCMAKE_Fortran_FLAGS_RELEASE_INIT:STRING=/O2 /Ot"
)

x64-win/download-tools.cmake:

# Available variables: VCPKG_ROOT_DIR PACKAGES_DIR BUILDTREES_DIR _VCPKG_INSTALLED_DIR DOWNLOADS

# vcpkg is not passing the scripts folder down to cmake. It just invokes ports.cmake directly. However, it internally knows where the scripts are. 

set(SCRIPTS "${VCPKG_ROOT_DIR}/scripts" CACHE PATH "Location to stored scripts")
list(APPEND CMAKE_MODULE_PATH "${SCRIPTS}/cmake") # This is necessary since some script is using `inlude(vcpkg_*)` instead of full paths

include("${SCRIPTS}/cmake/execute_process.cmake")
include("${SCRIPTS}/cmake/vcpkg_acquire_msys.cmake")
include("${SCRIPTS}/cmake/vcpkg_add_to_path.cmake")
include("${SCRIPTS}/cmake/vcpkg_apply_patches.cmake")
include("${SCRIPTS}/cmake/vcpkg_download_distfile.cmake")
include("${SCRIPTS}/cmake/vcpkg_download_sourceforge.cmake")
include("${SCRIPTS}/cmake/vcpkg_execute_required_process.cmake")
include("${SCRIPTS}/cmake/vcpkg_execute_required_process_repeat.cmake")
include("${SCRIPTS}/cmake/vcpkg_extract_archive.cmake")
include("${SCRIPTS}/cmake/vcpkg_extract_source_archive.cmake")
include("${SCRIPTS}/cmake/vcpkg_extract_source_archive_ex.cmake")
include("${SCRIPTS}/cmake/vcpkg_find_acquire_program.cmake")
include("${SCRIPTS}/cmake/vcpkg_from_bitbucket.cmake")
include("${SCRIPTS}/cmake/vcpkg_from_git.cmake")
include("${SCRIPTS}/cmake/vcpkg_from_github.cmake")
include("${SCRIPTS}/cmake/vcpkg_from_gitlab.cmake")
include("${SCRIPTS}/cmake/vcpkg_from_sourceforge.cmake")
include("${SCRIPTS}/cmake/vcpkg_get_program_files_platform_bitness.cmake")
include("${SCRIPTS}/cmake/vcpkg_host_path_list.cmake")
include("${SCRIPTS}/cmake/vcpkg_list.cmake")

include("${SCRIPTS}/cmake/z_vcpkg_apply_patches.cmake")
include("${SCRIPTS}/cmake/z_vcpkg_forward_output_variable.cmake")
include("${SCRIPTS}/cmake/z_vcpkg_function_arguments.cmake")
include("${SCRIPTS}/cmake/z_vcpkg_prettify_command_line.cmake")

message(STATUS "***** Running env setup cmake script *****")

#vcpkg_find_acquire_program(PYTHON3)
vcpkg_find_acquire_program(7Z)

# Setup Intel Compiler for Fortran

# Download link from https://github.com/oneapi-src/oneapi-ci/blob/master/.azure-pipelines.yml
block()
  find_program(Fortran_COMPILER NAMES ifort ifort.exe 
    PATHS "${DOWNLOADS}/tools/intel.2024.1/compiler/latest"
    PATH_SUFFIXES "${PATH_SUFFIX}"
    NO_DEFAULT_PATH
  )
  if(NOT Fortran_COMPILER)
    vcpkg_download_distfile(archive_path
        URLS "https://registrationcenter-download.intel.com/akdlm/IRC_NAS/c95a3b26-fc45-496c-833b-df08b10297b9/w_HPCKit_p_2024.1.0.561_offline.exe"
        FILENAME "w_HPCKit_p_2024.1.0.561_offline.exe"
        SHA512 84d94b3b6e240325e8d8b5018ed57df888653b912e3132ec91b7d89164f6d36b411975d65f981e366012fa05a582c0451d55dd81409a312baa3f1c6b80cae87b
    )

    set(extract_path "${DOWNLOADS}/tools/intel-extract")
    file(MAKE_DIRECTORY "${DOWNLOADS}/tools/intel-extract")
    vcpkg_execute_in_download_mode(
                            COMMAND "${7Z}" x "${archive_path}" "-o${extract_path}" "-y" "-bso0" "-bsp0"
                            WORKING_DIRECTORY "${extract_path}"
                        )

    set(packages
            "intel.oneapi.win.openmp,v=2024.1.0+964"
            "intel.oneapi.win.ifort-compiler,v=2024.1.0+964"
            "intel.oneapi.win.compilers-common,v=2024.1.0+964"
            "intel.oneapi.win.compilers-common-runtime,v=2024.1.0+964"
            "intel.oneapi.win.oneapi-common.vars,v=2024.1.0+582"
        )

    foreach(package IN LISTS packages)
      vcpkg_execute_required_process(
          COMMAND "${CMAKE_COMMAND}" "-E" "tar" "-xf" "${extract_path}/packages/${package}/cupPayload.cup"
          WORKING_DIRECTORY "${extract_path}"
          LOGNAME "extract-${package}"
      )
    endforeach()

    file(REMOVE_RECURSE "${DOWNLOADS}/tools/intel.2024.1")
    file(RENAME "${extract_path}/_installdir/" "${DOWNLOADS}/tools/intel.2024.1")
    # The env setup script assumes that the "latest" folder exists, so rename the version folder (alternnative would be to create a link)
    file(RENAME "${DOWNLOADS}/tools/intel.2024.1/compiler/2024.1"  "${DOWNLOADS}/tools/intel.2024.1/compiler/latest")
  endif()
  
  # Generate setup script
  file(MAKE_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}/gen") # Here goes the generated scripts
  file(WRITE "${CMAKE_CURRENT_LIST_DIR}/gen/setup-intel-env.bat" 
  "
  set \"ONEAPI_ROOT=${DOWNLOADS}/tools/intel.2024.1\"
  \"${DOWNLOADS}/tools/intel.2024.1/setvars.bat\" \"compiler\"
  ")
endblock()
# Setup LLVM Compiler

message(STATUS "***** ENV setup cmake script finished *****")

What do I get here:

  • every download performed in download-tools.cmake runs through vcpkg and is thus asset cached.
  • The scripts run before the compiler detection, so the environment is correctly setup
  • The script is only run once instead for every port (don't know if per port customization is possible?)

Unwanted behavior:

  • If CMake errors vcpkg just continues execution as normal? (Need to test if that also applies to other scripts?) (fixed)

Extensions:

  • Use this to obtain MSBuild itself?
  • Use this to obtain EMSCRIPTEN toolchain? (already have that working.)
  • Use this to obtain a MSYS environment?
  • Use this to install system deps automatically?

@Neumann-A
Copy link
Copy Markdown
Contributor Author

Neumann-A commented May 30, 2024

Note: I now also have a env setup script which can setup emscripten on windows

@ras0219-msft
Copy link
Copy Markdown
Collaborator

Could this be done inline in the triplet file by defining an "IN_PASS_0" variable? Then the user could execute_process() whatever they'd like.

@JavierMatosD JavierMatosD added the requires:vcpkg-team-review This PR or issue requires someone from the vcpkg team to take a further look. label Oct 9, 2024
@JavierMatosD JavierMatosD marked this pull request as draft January 9, 2025 16:06
@BillyONeal BillyONeal removed the requires:vcpkg-team-review This PR or issue requires someone from the vcpkg team to take a further look. label Jan 8, 2026
@BillyONeal
Copy link
Copy Markdown
Member

@Neumann-A What do you think of @ras0219-msft 's suggestion above?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

4 participants