diff --git a/.gitignore b/.gitignore index cd3f1c1eb73..26e1305456b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ source/userConfig build build_debug dist +installed_packages *.dll *.exe *.manifest diff --git a/.gitmodules b/.gitmodules index ad949f1e294..6cf421d6451 100644 --- a/.gitmodules +++ b/.gitmodules @@ -34,9 +34,6 @@ [submodule "include/configobj"] path = include/configobj url = https://github.com/DiffSK/configobj.git -[submodule "include/py2exe"] - path = include/py2exe - url = https://github.com/nvaccess/py2exe-bin [submodule "include/javaAccessBridge32"] path = include/javaAccessBridge32 url = https://github.com/nvaccess/javaAccessBridge32-bin.git diff --git a/appveyor.yml b/appveyor.yml index 364c5d87591..53b95423cdd 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -95,6 +95,7 @@ build_script: Set-AppveyorBuildVariable "sconsOutTargets" $sconsOutTargets Set-AppveyorBuildVariable "sconsArgs" $sconsArgs - 'echo scons args: %sconsArgs%' + - py -m pip install --upgrade pip - py scons.py source %sconsArgs% # We don't need launcher to run checkPot, so run the checkPot before launcher. - py scons.py checkPot %sconsArgs% @@ -119,7 +120,6 @@ build_script: before_test: # install required packages - - py -m pip install --upgrade pip - py -m pip install -r tests/system/requirements.txt -r tests/lint/lintInstall/requirements.txt - mkdir testOutput - mkdir testOutput\unit diff --git a/buildFunctions.py b/buildFunctions.py new file mode 100644 index 00000000000..ac66fc4bc90 --- /dev/null +++ b/buildFunctions.py @@ -0,0 +1,49 @@ +# A part of NonVisual Desktop Access (NVDA) +# Copyright (C) 2021 NV Access Limited, Łukasz Golonka +# This file may be used under the terms of the GNU General Public License, version 2 or later. +# For more details see: https://www.gnu.org/licenses/gpl-2.0.html + +"""Functions useful when building NVDA""" + +import os + + +def pythonPackagesDir() -> str: + """Returns path to the directory in which packages installed using pip should be placed. + The directory would be created if it does not exist. + """ + PACKAGES_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "installed_packages")) + os.makedirs(PACKAGES_DIR, exist_ok=True) + return PACKAGES_DIR + + +def installCallback(package): + import importlib + import pkg_resources + importlib.reload(pkg_resources) + requestedPackage = pkg_resources.working_set.find( + list(pkg_resources.parse_requirements(str(package)))[0] + ) + if requestedPackage: + return requestedPackage + import sys + import subprocess + subprocess.check_call( + [sys.executable, "-m", "pip", "install", "-t", pythonPackagesDir(), str(package)] + ) + import pkg_resources + # Evenn though this is quite ugly there is no other way to refresh list of available packages. + importlib.reload(pkg_resources) + return pkg_resources.working_set.find( + # `resolve` considers installation to be succesfull only when installed package is + # returned from the callback. + list(pkg_resources.parse_requirements(str(package)))[0] + ) + + +def requestPackage(requirementsString): + import pkg_resources + pkg_resources.working_set.resolve( + pkg_resources.parse_requirements(requirementsString), + installer=installCallback + ) diff --git a/devDocs/conf.py b/devDocs/conf.py index b01ee4fd6f2..d16d3328c11 100644 --- a/devDocs/conf.py +++ b/devDocs/conf.py @@ -10,7 +10,9 @@ import os import sys sys.path.insert(0, os.path.abspath('../source')) -import sourceEnv # noqa: F401, E402 +import sourceEnv # noqa: E402 + +sourceEnv.expandPythonPath() # Initialize languageHandler so that sphinx is able to deal with translatable strings. import languageHandler # noqa: E402 diff --git a/include/py2exe b/include/py2exe deleted file mode 160000 index c496ae65e4c..00000000000 --- a/include/py2exe +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c496ae65e4cb67fb5a1c17087ec33e4fd69be859 diff --git a/readme.md b/readme.md index 61910f12878..ab78e209bd4 100644 --- a/readme.md +++ b/readme.md @@ -103,7 +103,6 @@ For reference, the following run time dependencies are included in Git submodule Additionally, the following build time dependencies are included in Git submodules: -* [Py2Exe](https://github.com/albertosottile/py2exe/), version 0.9.3.2 commit b372a8e * [Python Windows Extensions](https://sourceforge.net/projects/pywin32/ ), build 224, required by py2exe * [txt2tags](https://txt2tags.org/), version 2.5 * [SCons](https://www.scons.org/), version 4.0.1 @@ -113,8 +112,12 @@ Additionally, the following build time dependencies are included in Git submodul * [Boost Optional (stand-alone header)](https://github.com/akrzemi1/Optional), from commit [3922965](https://github.com/akrzemi1/Optional/commit/3922965396fc455c6b1770374b9b4111799588a9) ### Other Dependencies +Some dependencies are installed automatically via pip as needed. +Developers may wish to first configure a Python Virtual Environment to ensure their general install is not affected. + +To build binary version of NVDA py2exe version 0.10.1.0 is installed automatically when executing `scons dist` To lint using Flake 8 locally using our SCons integration, some dependencies are installed (automatically) via pip. -Although this [must be run manually](#linting-your-changes), developers may wish to first configure a Python Virtual Environment to ensure their general install is not affected. +This [must be run manually](#linting-your-changes). * Flake8 * Flake8-tabs diff --git a/sconstruct b/sconstruct index 2824c19d335..4be166a24f8 100755 --- a/sconstruct +++ b/sconstruct @@ -1,5 +1,6 @@ # A part of NonVisual Desktop Access (NVDA) -# Copyright (C) 2010-2020 NV Access Limited, James Teh, Michael Curran, Peter Vágner, Joseph Lee, Reef Turner, Babbage B.V., Leonard de Ruijter, Łukasz Golonka, Accessolutions, Julien Cochuyt # noqa: E501 +# Copyright (C) 2010-2021 NV Access Limited, James Teh, Michael Curran, Peter Vágner, Joseph Lee, +# Reef Turner, Babbage B.V., Leonard de Ruijter, Łukasz Golonka, Accessolutions, Julien Cochuyt # This file may be used under the terms of the GNU General Public License, version 2 or later. # For more details see: https://www.gnu.org/licenses/gpl-2.0.html @@ -41,10 +42,10 @@ if sys.version_info.micro == 6: sourceEnvPath = os.path.abspath(os.path.join(Dir('.').srcnode().path, "source")) sys.path.append(sourceEnvPath) import sourceEnv +sourceEnv.expandPythonPath() sys.path.remove(sourceEnvPath) import time from glob import glob -from py2exe.dllfinder import pydll import importlib.util import winreg @@ -281,15 +282,22 @@ for t2tFile in env.Glob(os.path.join(userDocsDir.path,'*','*.t2t')): # Build unicode CLDR dictionaries env.SConscript('cldrDict_sconscript',exports=['env', 'sourceDir']) + +REQUIRED_PY2EXE_VER = "py2exe==0.10.1.0" + # A builder to generate an NVDA distribution. def NVDADistGenerator(target, source, env, for_signature): + action = [] + action.append( + [sys.executable, "-c", f"import buildFunctions; buildFunctions.requestPackage('{REQUIRED_PY2EXE_VER}');"] + ) buildVersionFn = os.path.join(str(source[0]), "_buildVersion.py") # Make the NVDA build use the specified version. # We don't do this using normal scons mechanisms because we want it to be cleaned up immediately after this builder # and py2exe will cause bytecode files to be created for it which scons doesn't know about. updateVersionType = env["updateVersionType"] or None # Any '\n' characters written are translated to the system default line separator, os.linesep. - action = [lambda target, source, env: open(buildVersionFn, "w", encoding="utf-8").write( + action.append([lambda target, source, env: open(buildVersionFn, "w", encoding="utf-8").write( 'version = {version!r}\n' 'publisher = {publisher!r}\n' 'updateVersionType = {updateVersionType!r}\n' @@ -298,7 +306,7 @@ def NVDADistGenerator(target, source, env, for_signature): ) # In Python 3 write returns the number of characters written, # which scons treats as an error code. - and None] + and None][0]) buildCmd = ["cd", source[0].path, "&&", sys.executable] @@ -315,11 +323,6 @@ def NVDADistGenerator(target, source, env, for_signature): action.append(buildCmd) - # Python3 has started signing its main python dll. - # However, Py2exe currently tries to add a string resource to it, invalidating the signature and possibly currupting the certificate. - # Therefore, copy a fresh version of the dll one more time once py2exe has completed. - action.append(Copy(target[0],pydll)) - # #10031: Apps written in Python 3 require Universal CRT to be installed. We cannot assume users have it on their systems. # Therefore , copy required libraries from Windows 10 SDK. try: diff --git a/source/nvda.pyw b/source/nvda.pyw index f2e07dfdc38..f4e0b43d55d 100755 --- a/source/nvda.pyw +++ b/source/nvda.pyw @@ -19,6 +19,7 @@ if getattr(sys, "frozen", None): appDir = sys.prefix else: import sourceEnv + sourceEnv.expandPythonPath() #We should always change directory to the location of this module (nvda.pyw), don't rely on sys.path[0] appDir = os.path.normpath(os.path.dirname(__file__)) appDir = os.path.abspath(appDir) diff --git a/source/sourceEnv.py b/source/sourceEnv.py index dfd7a6b9297..e5b970001cb 100644 --- a/source/sourceEnv.py +++ b/source/sourceEnv.py @@ -1,5 +1,5 @@ # A part of NonVisual Desktop Access (NVDA) -# Copyright (C) 2013-2020 NV Access Limited, Leonard de Ruijter +# Copyright (C) 2013-2021 NV Access Limited, Leonard de Ruijter # This file is covered by the GNU General Public License. # See the file COPYING for more details. @@ -11,21 +11,36 @@ # Get the path to the top of the repo; i.e. where include and miscDeps are. TOP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + + +def getBuildFunctions(): + try: + import buildFunctions + except ImportError: + sys.path.append(TOP_DIR) + import buildFunctions + sys.path.remove(TOP_DIR) + return buildFunctions + + # Directories containing Python modules included in git submodules. PYTHON_DIRS = ( os.path.join(TOP_DIR, "include", "pyserial"), os.path.join(TOP_DIR, "include", "comtypes"), os.path.join(TOP_DIR, "include", "configobj", "src"), os.path.join(TOP_DIR, "include", "wxPython"), - os.path.join(TOP_DIR, "include", "py2exe"), os.path.join(TOP_DIR, "miscDeps", "python"), + getBuildFunctions().pythonPackagesDir(), ) -#Check for existance of each Python dir -for path in PYTHON_DIRS: - if not os.path.exists(path): - raise OSError("Path %s does not exist. Perhaps try running git submodule update --init"%path) -# sys.path[0] will always be the current dir, which should take precedence. -# Insert our include paths after that. -sys.path[1:1] = PYTHON_DIRS +def expandPythonPath() -> None: + """Adds `PYTHON_DIRS` to the `PythonPath` of the current interpreter + raising appropriate exceptions if any of the dirs does not exist.""" + # Check for existance of each Python dir + for path in PYTHON_DIRS: + if not os.path.exists(path): + raise OSError("Path %s does not exist. Perhaps try running git submodule update --init" % path) + # sys.path[0] will always be the current dir, which should take precedence. + # Insert our include paths after that. + sys.path[1:1] = PYTHON_DIRS diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index 4e4d26c05b4..24897d8dd9e 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -1,4 +1,3 @@ -# tests/unit/__init__.py # A part of NonVisual Desktop Access (NVDA) # This file is covered by the GNU General Public License. # See the file COPYING for more details. @@ -31,9 +30,9 @@ SOURCE_DIR = os.path.join(TOP_DIR, "source") # Let us import modules from the NVDA source. sys.path.insert(1, SOURCE_DIR) -# Suppress Flake8 warning F401 (module imported but unused) -# as this module is imported to expand the system path. -import sourceEnv # noqa: F401 + +import sourceEnv # noqa: E402 +sourceEnv.expandPythonPath() import globalVars