This repository was archived by the owner on Apr 1, 2026. It is now read-only.
This repository was archived by the owner on Apr 1, 2026. It is now read-only.
Hang in MutationsBatcher.mutate() #949
Description
Environment details
- OS type and version: Mac OS X 14.1.2, Ubuntu 22.04
- Python version: 3.10.13
- pip version: N/A
google-cloud-bigtableversion: 2.23.0
Steps to reproduce
Recently, we started seeing this usage of the MutationsBatcher hang indefinitely in mutate(). This issue seems to be reproducible both in my local development environment (Mac) as well as in our deployed dev environment (Linux).
Based on the stack traces I included below, I'm guessing there's some problem with the creation of the gRPC channel to perform the flush but it's unclear to me how to get any insight into what might be going on there.
Code example
def _on_batch_completed(self, statuses: Sequence[Status]) -> None:
logger.debug(f"Completed write batch of size {len(statuses)}: {statuses}")
def store_features(self, entity_type: str, entities: pd.DataFrame) -> None:
table = "ml_features_cases"
packer = msgpack.Packer()
with MutationsBatcher(table, batch_completed_callback=self._on_batch_completed) as batch:
for index, *values in entities.itertuples(name=None):
row = table.direct_row(index.encode("utf-8"))
for column_name, value in zip(entities.columns.values, values):
# ... encode data into variable named encoded_value
row.set_cell("features", column_name.encode(), encoded_value)
batch.mutate(row)Stack trace
Thread 0x000000029d037000 (most recent call first):
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/grpc/_channel.py", line 1972 in __init__
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/grpc/__init__.py", line 2107 in secure_channel
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/google/api_core/grpc_helpers.py", line 386 in create_channel
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/google/cloud/bigtable_v2/services/bigtable/transports/grpc.py", line 217 in create_channel
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/google/cloud/bigtable/client.py", line 285 in _create_gapic_client_channel
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/google/cloud/bigtable/client.py", line 332 in table_data_client
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/google/cloud/bigtable/table.py", line 160 in name
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/google/cloud/bigtable/table.py", line 724 in mutate_rows
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/google/cloud/bigtable/batcher.py", line 385 in _flush_rows
File "/Users/patrick/.pyenv/versions/3.10.13/lib/python3.10/concurrent/futures/thread.py", line 58 in run
File "/Users/patrick/.pyenv/versions/3.10.13/lib/python3.10/concurrent/futures/thread.py", line 83 in _worker
File "/Users/patrick/.pyenv/versions/3.10.13/lib/python3.10/threading.py", line 953 in run
File "/Users/patrick/.pyenv/versions/3.10.13/lib/python3.10/threading.py", line 1016 in _bootstrap_inner
File "/Users/patrick/.pyenv/versions/3.10.13/lib/python3.10/threading.py", line 973 in _bootstrap
Thread 0x000000029c02b000 (most recent call first):
File "", line 1074 in get_data
File "/Users/patrick/.pyenv/versions/3.10.13/lib/python3.10/pkgutil.py", line 639 in get_data
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/grpc/_channel.py", line 1972 in __init__
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/grpc/__init__.py", line 2107 in secure_channel
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/google/api_core/grpc_helpers.py", line 386 in create_channel
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/google/cloud/bigtable_v2/services/bigtable/transports/grpc.py", line 217 in create_channel
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/google/cloud/bigtable/client.py", line 285 in _create_gapic_client_channel
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/google/cloud/bigtable/client.py", line 332 in table_data_client
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/google/cloud/bigtable/table.py", line 160 in name
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/google/cloud/bigtable/table.py", line 724 in mutate_rows
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/google/cloud/bigtable/batcher.py", line 385 in _flush_rows
File "/Users/patrick/.pyenv/versions/3.10.13/lib/python3.10/concurrent/futures/thread.py", line 58 in run
File "/Users/patrick/.pyenv/versions/3.10.13/lib/python3.10/concurrent/futures/thread.py", line 83 in _worker
File "/Users/patrick/.pyenv/versions/3.10.13/lib/python3.10/threading.py", line 953 in run
File "/Users/patrick/.pyenv/versions/3.10.13/lib/python3.10/threading.py", line 1016 in _bootstrap_inner
File "/Users/patrick/.pyenv/versions/3.10.13/lib/python3.10/threading.py", line 973 in _bootstrap
Current thread 0x00000001d6c01ec0 (most recent call first):
File "/Users/patrick/.pyenv/versions/3.10.13/lib/python3.10/threading.py", line 320 in wait
File "/Users/patrick/.pyenv/versions/3.10.13/lib/python3.10/threading.py", line 607 in wait
File "/Users/patrick/.pyenv/versions/3.10.13/lib/python3.10/threading.py", line 940 in start
File "/Users/patrick/.pyenv/versions/3.10.13/lib/python3.10/concurrent/futures/thread.py", line 199 in _adjust_thread_count
File "/Users/patrick/.pyenv/versions/3.10.13/lib/python3.10/concurrent/futures/thread.py", line 176 in submit
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/google/cloud/bigtable/batcher.py", line 340 in _flush_async
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/google/cloud/bigtable/batcher.py", line 266 in mutate
File "/Users/patrick/src/Apella-Technology/realtime-dags/case_forecasting_baseline/feature_store/feature_store.py", line 138 in store_features
File "/Users/patrick/src/Apella-Technology/realtime-dags/case_forecasting_baseline/dags/ops/feature_calculation.py", line 137 in store_features
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/dagster/_core/execution/plan/compute_generator.py", line 125 in invoke_compute_fn
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/dagster/_core/execution/plan/compute_generator.py", line 131 in _coerce_op_compute_fn_to_iterator
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/dagster/_utils/__init__.py", line 465 in iterate_with_context
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/dagster/_core/execution/plan/compute.py", line 181 in _yield_compute_results
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/dagster/_core/execution/plan/compute.py", line 212 in execute_core_compute
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/dagster/_core/execution/plan/execute_step.py", line 95 in _process_asset_results_to_events
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/dagster/_core/execution/plan/execute_step.py", line 160 in _step_output_error_checked_user_event_sequence
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/dagster/_core/execution/plan/execute_step.py", line 478 in core_dagster_event_sequence_for_step
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/dagster/_core/execution/plan/execute_plan.py", line 286 in dagster_event_sequence_for_step
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/dagster/_core/execution/plan/execute_plan.py", line 121 in inner_plan_execution_iterator
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/dagster/_core/executor/in_process.py", line 26 in inprocess_execution_iterator
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/dagster/_core/execution/api.py", line 875 in __iter__
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/dagster/_core/executor/in_process.py", line 55 in execute
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/dagster/_core/execution/api.py", line 764 in job_execution_iterator
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/dagster/_core/execution/api.py", line 875 in __iter__
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/dagster/_grpc/impl.py", line 152 in core_execute_run
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/dagster/_grpc/impl.py", line 242 in _run_in_subprocess
File "/Users/patrick/Library/Caches/pypoetry/virtualenvs/realtime-dagster-wbtued84-py3.10/lib/python3.10/site-packages/dagster/_grpc/impl.py", line 266 in start_run_in_subprocess
File "/Users/patrick/.pyenv/versions/3.10.13/lib/python3.10/multiprocessing/process.py", line 108 in run
File "/Users/patrick/.pyenv/versions/3.10.13/lib/python3.10/multiprocessing/process.py", line 314 in _bootstrap
File "/Users/patrick/.pyenv/versions/3.10.13/lib/python3.10/multiprocessing/spawn.py", line 129 in _main
File "/Users/patrick/.pyenv/versions/3.10.13/lib/python3.10/multiprocessing/spawn.py", line 116 in spawn_main
File "", line 1 in