Skip to content

[Archery][CI][Integration] Integration workflow is no longer running Rust integration tests #41481

Closed
Closed
[Archery][CI][Integration] Integration workflow is no longer running Rust integration tests#41481
Assignees
paleolimbot
Labels
Component: ArcheryComponent: Continuous IntegrationComponent: IntegrationType: bug
Milestone
18.0.0
@paleolimbot

Description

@paleolimbot
paleolimbot
opened on May 1, 2024

Describe the bug, including details regarding any error messages, version, and platform.

Discovered while trying to modify the integration workflow to get nanoarrow's integration tests to run (#39302). Despite issuing the archery docker run invocation:

arrow/.github/workflows/integration.yml

Lines 98 to 100 in 22f88fa

archery docker run \
-e ARCHERY_DEFAULT_BRANCH=${{ github.event.repository.default_branch }} \
-e ARCHERY_INTEGRATION_WITH_RUST=1 \

We get:

DEBUG:archery:Executing `['docker', 'run', '--rm', '-e', 'ARCHERY_DEFAULT_BRANCH=main', '-e', 'ARCHERY_INTEGRATION_WITH_RUST=1', '-e', 'ARCHERY_INTEGRATION_WITH_RUST=0', '-e', 'ARROW_CPP_EXE_PATH=/build/cpp/debug', '-e', 'ARROW_RUST_EXE_PATH=/build/rust/debug', '-e', 'CCACHE_COMPILERCHECK=content', '-e', 'CCACHE_COMPRESS=1', '-e', 'CCACHE_COMPRESSLEVEL=6', '-e', 'CCACHE_DIR=/ccache', '-e', 'CCACHE_MAXSIZE=1G', '-e', 'GITHUB_ACTIONS=true', '-v', '/home/runner/work/arrow/arrow:/arrow', '-v', '/home/runner/work/arrow/arrow/.docker/conda-ccache:/ccache', 'apache/arrow-dev:amd64-conda-integration', '/arrow/ci/scripts/integration_arrow_build.sh /arrow /build && /arrow/ci/scripts/integration_arrow.sh /arrow /build']`

https://github.com/apache/arrow/actions/runs/8907059610/job/24460263757#step:8:345

...which results in

 + /arrow/ci/scripts/rust_build.sh /arrow /build
=====================================================================
Not building the Rust implementation.
=====================================================================

The place where this command appears to be calculated is below. It seems that it's doubling the parameters that are included in the docker-compose.yml file (overriding the values in docker-compose.yml seems to be the only way I was able to get Rust to build).

arrow/dev/archery/archery/docker/core.py

Lines 59 to 169 in 22f88fa

class ComposeConfig:
def __init__(self, config_path, dotenv_path, compose_bin,
using_docker=False, using_buildx=False,
params=None, debug=False):
self.using_docker = using_docker
self.using_buildx = using_buildx
self.debug = debug
config_path = _ensure_path(config_path)
if dotenv_path:
dotenv_path = _ensure_path(dotenv_path)
else:
dotenv_path = config_path.parent / '.env'
if self.debug:
# Log docker version
Docker().run('version')
self._read_env(dotenv_path, params)
self._read_config(config_path, compose_bin)
def _read_env(self, dotenv_path, params):
"""
Read .env and merge it with explicitly passed parameters.
"""
self.dotenv = dotenv_values(str(dotenv_path))
if params is None:
self.params = {}
else:
self.params = {k: v for k, v in params.items() if k in self.dotenv}
# forward the process' environment variables
self.env = os.environ.copy()
# set the defaults from the dotenv files
self.env.update(self.dotenv)
# override the defaults passed as parameters
self.env.update(self.params)
# translate docker's architecture notation to a more widely used one
arch = self.env.get('ARCH', 'amd64')
self.env['ARCH_ALIAS'] = _arch_alias_mapping.get(arch, arch)
self.env['ARCH_SHORT'] = _arch_short_mapping.get(arch, arch)
def _read_config(self, config_path, compose_bin):
"""
Validate and read the docker-compose.yml
"""
yaml = YAML()
with config_path.open() as fp:
self.raw_config = yaml.load(fp)
services = self.raw_config['services'].keys()
self.hierarchy = dict(flatten(self.raw_config.get('x-hierarchy', {})))
self.limit_presets = self.raw_config.get('x-limit-presets', {})
self.with_gpus = self.raw_config.get('x-with-gpus', [])
nodes = self.hierarchy.keys()
errors = []
for name in self.with_gpus:
if name not in services:
errors.append(
'Service `{}` defined in `x-with-gpus` bot not in '
'`services`'.format(name)
)
for name in nodes - services:
errors.append(
'Service `{}` is defined in `x-hierarchy` bot not in '
'`services`'.format(name)
)
for name in services - nodes:
errors.append(
'Service `{}` is defined in `services` but not in '
'`x-hierarchy`'.format(name)
)
# trigger docker-compose's own validation
if self.using_docker:
compose = Docker()
args = ['compose']
else:
compose = Command('docker-compose')
args = []
args += ['--file', str(config_path), 'config']
result = compose.run(*args, env=self.env, check=False,
stderr=subprocess.PIPE, stdout=subprocess.PIPE)
if result.returncode != 0:
# strip the intro line of docker-compose errors
errors += result.stderr.decode().splitlines()
if errors:
msg = '\n'.join([' - {}'.format(msg) for msg in errors])
raise ValueError(
'Found errors with docker-compose:\n{}'.format(msg)
)
rendered_config = StringIO(result.stdout.decode())
self.path = config_path
self.config = yaml.load(rendered_config)
def get(self, service_name):
try:
service = self.config['services'][service_name]
except KeyError:
raise UndefinedImage(service_name)
service['name'] = service_name
service['need_gpu'] = service_name in self.with_gpus
service['ancestors'] = self.hierarchy[service_name]
return service
def __getitem__(self, service_name):
return self.get(service_name)

Component(s)

Archery, Continuous Integration, Integration

Metadata

Metadata

Assignees

Labels

Component: ArcheryComponent: Continuous IntegrationComponent: IntegrationType: bug

Type

No type

Fields

No fields configured for issues without a type.

Projects

No projects

Milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions