From 75620554411cf1ea58ed4ada7b496c9c48b06e49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Golonka?= Date: Wed, 23 Dec 2020 13:46:14 +0100 Subject: [PATCH 1/9] Remove workaround for issue #9762 Latest version of Py2exe no longer invalidates signature of Python's dll so this workaround is no longer needed --- sconstruct | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/sconstruct b/sconstruct index 334351dc61e..cbc6243626f 100755 --- a/sconstruct +++ b/sconstruct @@ -1,5 +1,6 @@ # A part of NonVisual Desktop Access (NVDA) -# Copyright (C) 2010-2020 NV Access Limited, James Teh, Michael Curran, Peter Vágner, Joseph Lee, Reef Turner, Babbage B.V., Leonard de Ruijter, Łukasz Golonka, Accessolutions, Julien Cochuyt # noqa: E501 +# Copyright (C) 2010-2021 NV Access Limited, James Teh, Michael Curran, Peter Vágner, Joseph Lee, +# Reef Turner, Babbage B.V., Leonard de Ruijter, Łukasz Golonka, Accessolutions, Julien Cochuyt # This file may be used under the terms of the GNU General Public License, version 2 or later. # For more details see: https://www.gnu.org/licenses/gpl-2.0.html @@ -44,7 +45,6 @@ import sourceEnv sys.path.remove(sourceEnvPath) import time from glob import glob -from py2exe.dllfinder import pydll import importlib.util import winreg @@ -299,11 +299,6 @@ def NVDADistGenerator(target, source, env, for_signature): action.append(buildCmd) - # Python3 has started signing its main python dll. - # However, Py2exe currently tries to add a string resource to it, invalidating the signature and possibly currupting the certificate. - # Therefore, copy a fresh version of the dll one more time once py2exe has completed. - action.append(Copy(target[0],pydll)) - # #10031: Apps written in Python 3 require Universal CRT to be installed. We cannot assume users have it on their systems. # Therefore , copy required libraries from Windows 10 SDK. try: From 8ea46a7fb088cb76f82bee6fc895502ca76e7c4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Golonka?= Date: Wed, 23 Dec 2020 13:52:58 +0100 Subject: [PATCH 2/9] Stop bundling Py2exe as a git submodule --- .gitmodules | 3 --- source/sourceEnv.py | 1 - 2 files changed, 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index 41835aa2324..35adfbce31b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -34,9 +34,6 @@ [submodule "include/configobj"] path = include/configobj url = https://github.com/DiffSK/configobj.git -[submodule "include/py2exe"] - path = include/py2exe - url = https://github.com/nvaccess/py2exe-bin [submodule "include/javaAccessBridge32"] path = include/javaAccessBridge32 url = https://github.com/nvaccess/javaAccessBridge32-bin.git diff --git a/source/sourceEnv.py b/source/sourceEnv.py index dfd7a6b9297..b0ce5eb4875 100644 --- a/source/sourceEnv.py +++ b/source/sourceEnv.py @@ -17,7 +17,6 @@ os.path.join(TOP_DIR, "include", "comtypes"), os.path.join(TOP_DIR, "include", "configobj", "src"), os.path.join(TOP_DIR, "include", "wxPython"), - os.path.join(TOP_DIR, "include", "py2exe"), os.path.join(TOP_DIR, "miscDeps", "python"), ) From 489e2574ce9f596e471caaa9ea5de2aeae500def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Golonka?= Date: Wed, 23 Dec 2020 13:55:09 +0100 Subject: [PATCH 3/9] Add a generic function which installs package with a given name using pip igf not installed --- buildFunctions.py | 31 +++++++++++++++++++++++++++++++ include/py2exe | 1 - 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 buildFunctions.py delete mode 160000 include/py2exe diff --git a/buildFunctions.py b/buildFunctions.py new file mode 100644 index 00000000000..e1313e1e558 --- /dev/null +++ b/buildFunctions.py @@ -0,0 +1,31 @@ +# A part of NonVisual Desktop Access (NVDA) +# Copyright (C) 2020 NV Access Limited, Łukasz Golonka +# This file may be used under the terms of the GNU General Public License, version 2 or later. +# For more details see: https://www.gnu.org/licenses/gpl-2.0.html + +"""Functions useful when building NVDA""" + + +def installCallback(package): + import sys + import subprocess + subprocess.check_call( + [sys.executable, "-m", "pip", "install", str(package)] + ) + import importlib + import pkg_resources + # Evenn though this is quite ugly there is no other way to refresh list of available packages. + importlib.reload(pkg_resources) + return pkg_resources.working_set.find( + # `resolve` considers installation to be succesfull only when installed package is + # returned from the callback. + list(pkg_resources.parse_requirements(str(package)))[0] + ) + + +def requestPackage(requirementsString): + import pkg_resources + pkg_resources.working_set.resolve( + pkg_resources.parse_requirements(requirementsString), + installer=installCallback + ) diff --git a/include/py2exe b/include/py2exe deleted file mode 160000 index c496ae65e4c..00000000000 --- a/include/py2exe +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c496ae65e4cb67fb5a1c17087ec33e4fd69be859 From 3cfe28cdef4bc38a0c4d821187596fe84b320d8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Golonka?= Date: Wed, 23 Dec 2020 18:46:09 +0100 Subject: [PATCH 4/9] Install Py2exe when executing SCons --- sconstruct | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sconstruct b/sconstruct index cbc6243626f..2cf5fc9d5ff 100755 --- a/sconstruct +++ b/sconstruct @@ -265,8 +265,13 @@ for t2tFile in env.Glob(os.path.join(userDocsDir.path,'*','*.t2t')): # Build unicode CLDR dictionaries env.SConscript('cldrDict_sconscript',exports=['env', 'sourceDir']) + +REQUIRED_PY2EXE_VER = "py2exe==0.10.1.0" + # A builder to generate an NVDA distribution. def NVDADistGenerator(target, source, env, for_signature): + import buildFunctions + buildFunctions.requestPackage(REQUIRED_PY2EXE_VER) buildVersionFn = os.path.join(str(source[0]), "_buildVersion.py") # Make the NVDA build use the specified version. # We don't do this using normal scons mechanisms because we want it to be cleaned up immediately after this builder From bb2aa828da35773926862cc95fe75248af59f9ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Golonka?= Date: Wed, 23 Dec 2020 18:51:42 +0100 Subject: [PATCH 5/9] Update readme to reflect the new situation --- readme.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 4ee3e02e918..cd53bfb6732 100644 --- a/readme.md +++ b/readme.md @@ -103,7 +103,6 @@ For reference, the following run time dependencies are included in Git submodule Additionally, the following build time dependencies are included in Git submodules: -* [Py2Exe](https://github.com/albertosottile/py2exe/), version 0.9.3.2 commit b372a8e * [Python Windows Extensions](https://sourceforge.net/projects/pywin32/ ), build 224, required by py2exe * [txt2tags](https://txt2tags.org/), version 2.5 * [SCons](https://www.scons.org/), version 4.0.1 @@ -114,8 +113,12 @@ Additionally, the following build time dependencies are included in Git submodul * [Boost Optional (stand-alone header)](https://github.com/akrzemi1/Optional), from commit [3922965](https://github.com/akrzemi1/Optional/commit/3922965396fc455c6b1770374b9b4111799588a9) ### Other Dependencies +Some dependencies are installed automatically via pip as needed. +Developers may wish to first configure a Python Virtual Environment to ensure their general install is not affected. + +To build binary version of NVDA py2exe version 0.10.1.0 is installed automatically when executing `scons dist` To lint using Flake 8 locally using our SCons integration, some dependencies are installed (automatically) via pip. -Although this [must be run manually](#linting-your-changes), developers may wish to first configure a Python Virtual Environment to ensure their general install is not affected. +This [must be run manually](#linting-your-changes). * Flake8 * Flake8-tabs From d55470e5131877473b480b9d4b20908accac14ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Golonka?= Date: Tue, 12 Jan 2021 12:40:54 +0100 Subject: [PATCH 6/9] WIP --- .gitignore | 1 + buildFunctions.py | 22 +++++++++++++++++----- devDocs/conf.py | 4 +++- sconstruct | 1 + source/nvda.pyw | 1 + source/sourceEnv.py | 31 +++++++++++++++++++++++-------- tests/unit/__init__.py | 7 +++---- 7 files changed, 49 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 1f9fbdeb99a..d4381946e94 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ source/userConfig build build_debug dist +installed_packages *.dll *.exe *.manifest diff --git a/buildFunctions.py b/buildFunctions.py index e1313e1e558..d6da747d804 100644 --- a/buildFunctions.py +++ b/buildFunctions.py @@ -9,13 +9,21 @@ def installCallback(package): import sys import subprocess + import sourceEnv subprocess.check_call( - [sys.executable, "-m", "pip", "install", str(package)] + [sys.executable, "-m", "pip", "install", "-t", sourceEnv.pythonPackagesDir(), str(package)] ) import importlib import pkg_resources # Evenn though this is quite ugly there is no other way to refresh list of available packages. importlib.reload(pkg_resources) + pkg_resources._initialize_master_working_set() + print(type(pkg_resources.working_set.find( + # `resolve` considers installation to be succesfull only when installed package is + # returned from the callback. + list(pkg_resources.parse_requirements(str(package)))[0] + )) + ) return pkg_resources.working_set.find( # `resolve` considers installation to be succesfull only when installed package is # returned from the callback. @@ -25,7 +33,11 @@ def installCallback(package): def requestPackage(requirementsString): import pkg_resources - pkg_resources.working_set.resolve( - pkg_resources.parse_requirements(requirementsString), - installer=installCallback - ) + import traceback + try: + pkg_resources.working_set.resolve( + pkg_resources.parse_requirements(requirementsString), + installer=installCallback + ) + except Exception as e: + print(f"during install {traceback.format_exc()}") diff --git a/devDocs/conf.py b/devDocs/conf.py index b01ee4fd6f2..d16d3328c11 100644 --- a/devDocs/conf.py +++ b/devDocs/conf.py @@ -10,7 +10,9 @@ import os import sys sys.path.insert(0, os.path.abspath('../source')) -import sourceEnv # noqa: F401, E402 +import sourceEnv # noqa: E402 + +sourceEnv.expandPythonPath() # Initialize languageHandler so that sphinx is able to deal with translatable strings. import languageHandler # noqa: E402 diff --git a/sconstruct b/sconstruct index 2cf5fc9d5ff..9cbf72d1097 100755 --- a/sconstruct +++ b/sconstruct @@ -42,6 +42,7 @@ if sys.version_info.micro == 6: sourceEnvPath = os.path.abspath(os.path.join(Dir('.').srcnode().path, "source")) sys.path.append(sourceEnvPath) import sourceEnv +sourceEnv.expandPythonPath() sys.path.remove(sourceEnvPath) import time from glob import glob diff --git a/source/nvda.pyw b/source/nvda.pyw index f2e07dfdc38..f4e0b43d55d 100755 --- a/source/nvda.pyw +++ b/source/nvda.pyw @@ -19,6 +19,7 @@ if getattr(sys, "frozen", None): appDir = sys.prefix else: import sourceEnv + sourceEnv.expandPythonPath() #We should always change directory to the location of this module (nvda.pyw), don't rely on sys.path[0] appDir = os.path.normpath(os.path.dirname(__file__)) appDir = os.path.abspath(appDir) diff --git a/source/sourceEnv.py b/source/sourceEnv.py index b0ce5eb4875..5d4f19b229a 100644 --- a/source/sourceEnv.py +++ b/source/sourceEnv.py @@ -1,5 +1,5 @@ # A part of NonVisual Desktop Access (NVDA) -# Copyright (C) 2013-2020 NV Access Limited, Leonard de Ruijter +# Copyright (C) 2013-2021 NV Access Limited, Leonard de Ruijter # This file is covered by the GNU General Public License. # See the file COPYING for more details. @@ -11,6 +11,17 @@ # Get the path to the top of the repo; i.e. where include and miscDeps are. TOP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + + +def pythonPackagesDir() -> str: + """Returns path to the directory in which packages installed using pip should be placed. + The directory would be created if it does not exist. + """ + PACKAGES_DIR = os.path.abspath(os.path.join(TOP_DIR, "installed_packages")) + os.makedirs(PACKAGES_DIR, exist_ok=True) + return PACKAGES_DIR + + # Directories containing Python modules included in git submodules. PYTHON_DIRS = ( os.path.join(TOP_DIR, "include", "pyserial"), @@ -18,13 +29,17 @@ os.path.join(TOP_DIR, "include", "configobj", "src"), os.path.join(TOP_DIR, "include", "wxPython"), os.path.join(TOP_DIR, "miscDeps", "python"), + pythonPackagesDir(), ) -#Check for existance of each Python dir -for path in PYTHON_DIRS: - if not os.path.exists(path): - raise OSError("Path %s does not exist. Perhaps try running git submodule update --init"%path) -# sys.path[0] will always be the current dir, which should take precedence. -# Insert our include paths after that. -sys.path[1:1] = PYTHON_DIRS +def expandPythonPath() -> None: + """Adds `PYTHON_DIRS` to the `PythonPath` of the current interpreter + raising appropriate exceptions if any of the dirs does not exist.""" + # Check for existance of each Python dir + for path in PYTHON_DIRS: + if not os.path.exists(path): + raise OSError("Path %s does not exist. Perhaps try running git submodule update --init" % path) + # sys.path[0] will always be the current dir, which should take precedence. + # Insert our include paths after that. + sys.path[1:1] = PYTHON_DIRS diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index 4e4d26c05b4..ef2884adbfd 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -1,4 +1,3 @@ -# tests/unit/__init__.py # A part of NonVisual Desktop Access (NVDA) # This file is covered by the GNU General Public License. # See the file COPYING for more details. @@ -31,9 +30,9 @@ SOURCE_DIR = os.path.join(TOP_DIR, "source") # Let us import modules from the NVDA source. sys.path.insert(1, SOURCE_DIR) -# Suppress Flake8 warning F401 (module imported but unused) -# as this module is imported to expand the system path. -import sourceEnv # noqa: F401 + +import sourceEnv +sourceEnv.expandPythonPath() import globalVars From 5d4130ef99ed8482c35619aa40fd23485b362281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Golonka?= Date: Tue, 12 Jan 2021 14:34:38 +0100 Subject: [PATCH 7/9] WIP2 --- buildFunctions.py | 18 ++++-------------- tests/unit/__init__.py | 2 +- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/buildFunctions.py b/buildFunctions.py index d6da747d804..ad668740f0f 100644 --- a/buildFunctions.py +++ b/buildFunctions.py @@ -18,12 +18,6 @@ def installCallback(package): # Evenn though this is quite ugly there is no other way to refresh list of available packages. importlib.reload(pkg_resources) pkg_resources._initialize_master_working_set() - print(type(pkg_resources.working_set.find( - # `resolve` considers installation to be succesfull only when installed package is - # returned from the callback. - list(pkg_resources.parse_requirements(str(package)))[0] - )) - ) return pkg_resources.working_set.find( # `resolve` considers installation to be succesfull only when installed package is # returned from the callback. @@ -33,11 +27,7 @@ def installCallback(package): def requestPackage(requirementsString): import pkg_resources - import traceback - try: - pkg_resources.working_set.resolve( - pkg_resources.parse_requirements(requirementsString), - installer=installCallback - ) - except Exception as e: - print(f"during install {traceback.format_exc()}") + pkg_resources.working_set.resolve( + pkg_resources.parse_requirements(requirementsString), + installer=installCallback + ) diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index ef2884adbfd..24897d8dd9e 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -31,7 +31,7 @@ # Let us import modules from the NVDA source. sys.path.insert(1, SOURCE_DIR) -import sourceEnv +import sourceEnv # noqa: E402 sourceEnv.expandPythonPath() import globalVars From 49e71c585d7067ad1ee374075ae3a404b0f2efcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Golonka?= Date: Tue, 12 Jan 2021 22:57:08 +0100 Subject: [PATCH 8/9] WIP3 --- buildFunctions.py | 26 +++++++++++++++++++++----- sconstruct | 10 ++++++---- source/sourceEnv.py | 17 +++++++++-------- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/buildFunctions.py b/buildFunctions.py index ad668740f0f..ac66fc4bc90 100644 --- a/buildFunctions.py +++ b/buildFunctions.py @@ -1,23 +1,39 @@ # A part of NonVisual Desktop Access (NVDA) -# Copyright (C) 2020 NV Access Limited, Łukasz Golonka +# Copyright (C) 2021 NV Access Limited, Łukasz Golonka # This file may be used under the terms of the GNU General Public License, version 2 or later. # For more details see: https://www.gnu.org/licenses/gpl-2.0.html """Functions useful when building NVDA""" +import os + + +def pythonPackagesDir() -> str: + """Returns path to the directory in which packages installed using pip should be placed. + The directory would be created if it does not exist. + """ + PACKAGES_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "installed_packages")) + os.makedirs(PACKAGES_DIR, exist_ok=True) + return PACKAGES_DIR + def installCallback(package): + import importlib + import pkg_resources + importlib.reload(pkg_resources) + requestedPackage = pkg_resources.working_set.find( + list(pkg_resources.parse_requirements(str(package)))[0] + ) + if requestedPackage: + return requestedPackage import sys import subprocess - import sourceEnv subprocess.check_call( - [sys.executable, "-m", "pip", "install", "-t", sourceEnv.pythonPackagesDir(), str(package)] + [sys.executable, "-m", "pip", "install", "-t", pythonPackagesDir(), str(package)] ) - import importlib import pkg_resources # Evenn though this is quite ugly there is no other way to refresh list of available packages. importlib.reload(pkg_resources) - pkg_resources._initialize_master_working_set() return pkg_resources.working_set.find( # `resolve` considers installation to be succesfull only when installed package is # returned from the callback. diff --git a/sconstruct b/sconstruct index 9cbf72d1097..8c03f251107 100755 --- a/sconstruct +++ b/sconstruct @@ -271,15 +271,17 @@ REQUIRED_PY2EXE_VER = "py2exe==0.10.1.0" # A builder to generate an NVDA distribution. def NVDADistGenerator(target, source, env, for_signature): - import buildFunctions - buildFunctions.requestPackage(REQUIRED_PY2EXE_VER) + action = [] + action.append( + [sys.executable, "-c", f"import buildFunctions; buildFunctions.requestPackage('{REQUIRED_PY2EXE_VER}');"] + ) buildVersionFn = os.path.join(str(source[0]), "_buildVersion.py") # Make the NVDA build use the specified version. # We don't do this using normal scons mechanisms because we want it to be cleaned up immediately after this builder # and py2exe will cause bytecode files to be created for it which scons doesn't know about. updateVersionType = env["updateVersionType"] or None # Any '\n' characters written are translated to the system default line separator, os.linesep. - action = [lambda target, source, env: open(buildVersionFn, "w", encoding="utf-8").write( + action.append([lambda target, source, env: open(buildVersionFn, "w", encoding="utf-8").write( 'version = {version!r}\n' 'publisher = {publisher!r}\n' 'updateVersionType = {updateVersionType!r}\n' @@ -288,7 +290,7 @@ def NVDADistGenerator(target, source, env, for_signature): ) # In Python 3 write returns the number of characters written, # which scons treats as an error code. - and None] + and None][0]) buildCmd = ["cd", source[0].path, "&&", sys.executable] diff --git a/source/sourceEnv.py b/source/sourceEnv.py index 5d4f19b229a..e5b970001cb 100644 --- a/source/sourceEnv.py +++ b/source/sourceEnv.py @@ -13,13 +13,14 @@ TOP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) -def pythonPackagesDir() -> str: - """Returns path to the directory in which packages installed using pip should be placed. - The directory would be created if it does not exist. - """ - PACKAGES_DIR = os.path.abspath(os.path.join(TOP_DIR, "installed_packages")) - os.makedirs(PACKAGES_DIR, exist_ok=True) - return PACKAGES_DIR +def getBuildFunctions(): + try: + import buildFunctions + except ImportError: + sys.path.append(TOP_DIR) + import buildFunctions + sys.path.remove(TOP_DIR) + return buildFunctions # Directories containing Python modules included in git submodules. @@ -29,7 +30,7 @@ def pythonPackagesDir() -> str: os.path.join(TOP_DIR, "include", "configobj", "src"), os.path.join(TOP_DIR, "include", "wxPython"), os.path.join(TOP_DIR, "miscDeps", "python"), - pythonPackagesDir(), + getBuildFunctions().pythonPackagesDir(), ) From 6eacc81c380e159a52e70b79499d0306668d2ee2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Golonka?= Date: Sun, 7 Feb 2021 20:48:36 +0100 Subject: [PATCH 9/9] Upgrade pip during build rather than just before tests as now it is used way earlier. --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 364c5d87591..53b95423cdd 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -95,6 +95,7 @@ build_script: Set-AppveyorBuildVariable "sconsOutTargets" $sconsOutTargets Set-AppveyorBuildVariable "sconsArgs" $sconsArgs - 'echo scons args: %sconsArgs%' + - py -m pip install --upgrade pip - py scons.py source %sconsArgs% # We don't need launcher to run checkPot, so run the checkPot before launcher. - py scons.py checkPot %sconsArgs% @@ -119,7 +120,6 @@ build_script: before_test: # install required packages - - py -m pip install --upgrade pip - py -m pip install -r tests/system/requirements.txt -r tests/lint/lintInstall/requirements.txt - mkdir testOutput - mkdir testOutput\unit