Skip to content

Commit 54cd2b5

Browse files
trexfeatherstrexfeathers
authored
Merge 2e1a53b into e30e623 
2 parents e30e623 + 2e1a53b commit 54cd2b5

File tree

4 files changed

+186
-191
lines changed

4 files changed

+186
-191
lines changed

‎benchmarks/asv.conf.json‎

Lines changed: 5 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -3,57 +3,20 @@
33
"project": "scitools-iris",
44
"project_url": "https://github.com/SciTools/iris",
55
"repo": "..",
6-
"environment_type": "delegated",
6+
"environment_type": "delegated-iris",
77
"show_commit_url": "https://github.com/scitools/iris/commit/",
88
"branches": ["upstream/main"],
99

1010
"benchmark_dir": "./benchmarks",
1111
"env_dir": ".asv/env",
1212
"results_dir": ".asv/results",
1313
"html_dir": ".asv/html",
14-
"plugins": [".asv_delegated"],
15-
16-
"delegated_env_commands_comment": [
17-
"The command(s) that create/update an environment correctly for the",
18-
"checked-out commit. Command(s) format follows `build_command`:",
19-
" https://asv.readthedocs.io/en/stable/asv.conf.json.html#build-command-install-command-uninstall-command",
20-
21-
"The commit key indicates the earliest commit where the command(s)",
22-
"will work.",
23-
24-
"Differences from `build_command`:",
25-
" * See: https://asv.readthedocs.io/en/stable/asv.conf.json.html#build-command-install-command-uninstall-command",
26-
" * Env vars limited to those set outside build time.",
27-
" (e.g. `{conf_dir}` available but `{build_dir}` not)",
28-
" * Run in the same environment as the ASV install itself.",
29-
30-
"Mandatory format for the first 'command' within each commit:",
31-
" * `ENV_PARENT=path/to/parent/directory/of/env-directory`",
32-
" * Can contain env vars (e.g. `{conf_dir}`)",
33-
" * `ENV_PARENT` available as `{env_parent}` in subsequent commands",
34-
" * The environment will be detected as the most recently updated",
35-
" environment in `{env_parent}`."
36-
37-
],
38-
"delegated_env_commands": {
39-
"c8a663a0": [
40-
"ENV_PARENT={conf_dir}/.asv/env/nox312",
41-
"PY_VER=3.12 nox --envdir={env_parent} --session=tests --install-only --no-error-on-external-run --verbose"
42-
],
43-
"d58fca7e": [
44-
"ENV_PARENT={conf_dir}/.asv/env/nox311",
45-
"PY_VER=3.11 nox --envdir={env_parent} --session=tests --install-only --no-error-on-external-run --verbose"
46-
],
47-
"44fae030": [
48-
"ENV_PARENT={conf_dir}/.asv/env/nox310",
49-
"PY_VER=3.10 nox --envdir={env_parent} --session=tests --install-only --no-error-on-external-run --verbose"
50-
]
51-
},
14+
"plugins": [".asv_delegated_iris"],
5215

5316
"command_comment": [
54-
"We know that the Nox command takes care of installation in each",
55-
"environment, and in the case of Iris no specialised uninstall or",
56-
"build commands are needed to get it working.",
17+
"The inherited setup of the Iris test environment takes care of ",
18+
"Iris-installation too, and in the case of Iris no specialised ",
19+
"uninstall or build commands are needed to get it working either.",
5720

5821
"We do however need to install the custom benchmarks for them to be",
5922
"usable."

‎benchmarks/asv_delegated.py‎

Lines changed: 46 additions & 149 deletions
Original file line numberDiff line numberDiff line change
@@ -7,116 +7,48 @@
77
Preps an environment via custom user scripts, then uses that as the
88
benchmarking environment.
99
10+
This module is intended as the generic code that can be shared between
11+
repositories. Providing a functional benchmarking environment relies on correct
12+
subclassing of the :class:`Delegated` class to specialise it for the repo in
13+
question.
14+
1015
"""
1116

17+
from abc import ABC, abstractmethod
1218
from contextlib import contextmanager, suppress
1319
from os import environ
14-
from os.path import getmtime
1520
from pathlib import Path
1621
import sys
1722

18-
from asv import util as asv_util
1923
from asv.console import log
2024
from asv.environment import Environment, EnvironmentUnavailable
2125
from asv.repo import Repo
22-
from asv.util import ProcessError
23-
24-
25-
class EnvPrepCommands:
26-
"""A container for the environment preparation commands for a given commit.
27-
28-
Designed to read a value from the `delegated_env_commands` in the ASV
29-
config, and validate that the command(s) are structured correctly.
30-
"""
31-
32-
ENV_PARENT_VAR = "ENV_PARENT"
33-
env_parent: Path
34-
commands: list[str]
35-
36-
def __init__(self, environment: Environment, raw_commands: tuple[str]):
37-
env_var = self.ENV_PARENT_VAR
38-
raw_commands_list = list(raw_commands)
3926

40-
(first_command,) = environment._interpolate_commands(raw_commands_list[0])
41-
env: dict
42-
command, env, return_codes, cwd = first_command
4327

44-
valid = command == []
45-
valid = valid and return_codes == {0}
46-
valid = valid and cwd is None
47-
valid = valid and list(env.keys()) == [env_var]
48-
if not valid:
49-
message = (
50-
"First command MUST ONLY "
51-
f"define the {env_var} env var, with no command e.g: "
52-
f"`{env_var}=foo/`. Got: \n {raw_commands_list[0]}"
53-
)
54-
raise ValueError(message)
55-
56-
self.env_parent = Path(env[env_var]).resolve()
57-
self.commands = raw_commands_list[1:]
58-
59-
60-
class CommitFinder(dict[str, EnvPrepCommands]):
61-
"""A specialised dict for finding the appropriate env prep script for a commit."""
62-
63-
def __call__(self, repo: Repo, commit_hash: str):
64-
"""Return the latest env prep script that is earlier than the given commit."""
65-
66-
def validate_commit(commit: str, is_lookup: bool) -> None:
67-
try:
68-
_ = repo.get_date(commit)
69-
except ProcessError:
70-
if is_lookup:
71-
message_start = "Lookup commit"
72-
else:
73-
message_start = "Requested commit"
74-
repo_path = getattr(repo, "_path", "unknown")
75-
message = f"{message_start}: {commit} not found in repo: {repo_path}"
76-
raise KeyError(message)
77-
78-
for lookup in self.keys():
79-
validate_commit(lookup, is_lookup=True)
80-
validate_commit(commit_hash, is_lookup=False)
81-
82-
def parent_distance(parent_hash: str) -> int:
83-
range_spec = repo.get_range_spec(parent_hash, commit_hash)
84-
parents = repo.get_hashes_from_range(range_spec)
85-
86-
if parent_hash[:8] == commit_hash[:8]:
87-
distance = 0
88-
elif len(parents) == 0:
89-
distance = -1
90-
else:
91-
distance = len(parents)
92-
return distance
93-
94-
parentage = {commit: parent_distance(commit) for commit in self.keys()}
95-
parentage = {k: v for k, v in parentage.items() if v >= 0}
96-
if len(parentage) == 0:
97-
message = f"No env prep script available for commit: {commit_hash} ."
98-
raise KeyError(message)
99-
else:
100-
parentage = dict(sorted(parentage.items(), key=lambda item: item[1]))
101-
commit = next(iter(parentage))
102-
content = self[commit]
103-
return content
104-
105-
106-
class Delegated(Environment):
28+
class Delegated(Environment, ABC):
10729
"""Manage a benchmark environment using custom user scripts, run at each commit.
10830
10931
Ignores user input variations - ``matrix`` / ``pythons`` /
11032
``exclude``, since environment is being managed outside ASV.
11133
11234
A vanilla :class:`asv.environment.Environment` is created for containing
11335
the expected ASV configuration files and checked-out project. The actual
114-
'functional' environment is created/updated using the command(s) specified
115-
in the config ``delegated_env_commands``, then the location is recorded via
36+
'functional' environment is created/updated using
37+
:meth:`_prep_env_override`, then the location is recorded via
11638
a symlink within the ASV environment. The symlink is used as the
11739
environment path used for any executable calls (e.g.
11840
``python my_script.py``).
11941
42+
Intended as the generic parent class that can be shared between
43+
repositories. Providing a functional benchmarking environment relies on
44+
correct subclassing of this class to specialise it for the repo in question.
45+
46+
Warnings
47+
--------
48+
:class:`Delegated` is an abstract base class. It MUST ONLY be used via
49+
subclasses implementing their own :meth:`_prep_env_override`, and also
50+
:attr:`tool_name`, which must be unique.
51+
12052
"""
12153

12254
tool_name = "delegated"
@@ -180,20 +112,6 @@ def __init__(self, conf, python, requirements, tagged_env_vars):
180112
"""Preserves the 'true' path of the environment so that self._path can
181113
be safely modified and restored."""
182114

183-
env_commands = getattr(conf, "delegated_env_commands")
184-
try:
185-
env_prep_commands = {
186-
commit: EnvPrepCommands(self, commands)
187-
for commit, commands in env_commands.items()
188-
}
189-
except ValueError as err:
190-
message = f"Problem handling `delegated_env_commands`:\n{err}"
191-
log.error(message)
192-
raise EnvironmentUnavailable(message)
193-
self._env_prep_lookup = CommitFinder(**env_prep_commands)
194-
"""An object that can be called downstream to get the appropriate
195-
env prep script for a given repo and commit."""
196-
197115
@property
198116
def _path_delegated(self) -> Path:
199117
"""The path of the symlink to the delegated environment."""
@@ -241,63 +159,42 @@ def _setup(self):
241159
message += "Correct environment will be set up at the first commit checkout."
242160
log.warning(message)
243161

244-
def _prep_env(self, repo: Repo, commit_hash: str) -> None:
162+
@abstractmethod
163+
def _prep_env_override(self, env_parent_dir: Path) -> Path:
164+
"""Run aspects of :meth:`_prep_env` that vary between repos.
165+
166+
This is the method that is expected to do the preparing
167+
(:meth:`_prep_env` only performs pre- and post- steps). MUST be
168+
overridden in any subclass environments before they will work.
169+
170+
Parameters
171+
----------
172+
env_parent_dir : Path
173+
The directory that the prepared environment should be placed in.
174+
175+
Returns
176+
-------
177+
Path
178+
The path to the prepared environment.
179+
"""
180+
pass
181+
182+
def _prep_env(self, commit_hash: str) -> None:
245183
"""Prepare the delegated environment for the given commit hash."""
246184
message = (
247185
f"Running delegated environment management for: {self.name} "
248186
f"at commit: {commit_hash[:8]}"
249187
)
250188
log.info(message)
251189

252-
env_prep: EnvPrepCommands
253-
try:
254-
env_prep = self._env_prep_lookup(repo, commit_hash)
255-
except KeyError as err:
256-
message = f"Problem finding env prep commands: {err}"
257-
log.error(message)
258-
raise EnvironmentUnavailable(message)
259-
190+
env_parent = Path(self._env_dir).resolve()
260191
new_env_per_commit = self.COMMIT_ENVS_VAR in environ
261192
if new_env_per_commit:
262-
env_parent = env_prep.env_parent / commit_hash[:8]
263-
else:
264-
env_parent = env_prep.env_parent
265-
266-
# See :meth:`Environment._interpolate_commands`.
267-
# All ASV-namespaced env vars are available in the below format when
268-
# interpolating commands:
269-
# ASV_FOO_BAR = {foo_bar}
270-
# We want the env parent path to be one of those available.
271-
global_key = f"ASV_{EnvPrepCommands.ENV_PARENT_VAR}"
272-
self._global_env_vars[global_key] = str(env_parent)
273-
274-
# The project checkout.
275-
build_dir = Path(self._build_root) / self._repo_subdir
276-
277-
# Run the script(s) for delegated environment creation/updating.
278-
# (An adaptation of :meth:`Environment._interpolate_and_run_commands`).
279-
for command, env, return_codes, cwd in self._interpolate_commands(
280-
env_prep.commands
281-
):
282-
local_envs = dict(environ)
283-
local_envs.update(env)
284-
if cwd is None:
285-
cwd = str(build_dir)
286-
_ = asv_util.check_output(
287-
command,
288-
timeout=self._install_timeout,
289-
cwd=cwd,
290-
env=local_envs,
291-
valid_return_codes=return_codes,
292-
)
193+
env_parent = env_parent / commit_hash[:8]
194+
195+
delegated_env_path = self._prep_env_override(env_parent)
196+
assert delegated_env_path.is_relative_to(env_parent)
293197

294-
# Find the environment created/updated by running env_prep.commands.
295-
# The most recently updated directory in env_parent.
296-
delegated_env_path = sorted(
297-
env_parent.glob("*"),
298-
key=getmtime,
299-
reverse=True,
300-
)[0]
301198
# Record the environment's path via a symlink within this environment.
302199
self._symlink_to_delegated(delegated_env_path)
303200

@@ -307,7 +204,7 @@ def _prep_env(self, repo: Repo, commit_hash: str) -> None:
307204
def checkout_project(self, repo: Repo, commit_hash: str) -> None:
308205
"""Check out the working tree of the project at given commit hash."""
309206
super().checkout_project(repo, commit_hash)
310-
self._prep_env(repo, commit_hash)
207+
self._prep_env(commit_hash)
311208

312209
@contextmanager
313210
def _delegate_path(self):

0 commit comments

Comments
 (0)