Skip to content

perf: use _DummyListBuffer in test_reads.py to avoid GIL contention #16860

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zhixiangli wants to merge 2 commits into
base: main
Choose a base branch
from
+23 −4
Open

perf: use _DummyListBuffer in test_reads.py to avoid GIL contention #16860

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7fc3100
perf: use _DummyListBuffer in test_reads.py to avoid GIL contention
zhixiangli Apr 29, 2026
918ac98
test(perf): return number of bytes written in _DummyListBuffer.write
zhixiangli Apr 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 23 additions & 4 deletions packages/google-cloud-storage/tests/perf/microbenchmarks/time_based/reads/test_reads.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import os
import random
import time
from io import BytesIO

import pytest

Expand Down Expand Up @@ -114,6 +113,26 @@ def _download_time_based_json(client, filename, params):
return total_bytes_downloaded


# _DummyListBuffer is used instead of io.BytesIO to avoid GIL contention
# during profiling. io.BytesIO.write() holds the GIL while copying data,
# which introduces significant noise and bottlenecks in performance tests
# with high concurrency or large data transfers.
# This buffer simply collects chunks in a list and tracks the total size.
class _DummyListBuffer:
def __init__(self):
self.chunks = []
self.size = 0

def write(self, data):
self.chunks.append(data)
Comment thread
zhixiangli marked this conversation as resolved.
nbytes = len(data)
self.size += nbytes
return nbytes

def getvalue(self):
return b"".join(self.chunks)


async def _download_time_based_async(client, filename, params):
mrd = AsyncMultiRangeDownloader(client, params.bucket_name, filename)
await mrd.open()
Expand All @@ -138,17 +157,17 @@ async def _worker_coro():
offset = random.randint(
0, params.file_size_bytes - params.chunk_size_bytes
)
ranges.append((offset, params.chunk_size_bytes, BytesIO()))
ranges.append((offset, params.chunk_size_bytes, _DummyListBuffer()))
else: # seq
for _ in range(params.num_ranges):
ranges.append((offset, params.chunk_size_bytes, BytesIO()))
ranges.append((offset, params.chunk_size_bytes, _DummyListBuffer()))
offset += params.chunk_size_bytes
if offset + params.chunk_size_bytes > params.file_size_bytes:
offset = 0 # Reset offset if end of file is reached

await mrd.download_ranges(ranges)

bytes_in_buffers = sum(r[2].getbuffer().nbytes for r in ranges)
bytes_in_buffers = sum(r[2].size for r in ranges)
assert bytes_in_buffers == params.chunk_size_bytes * params.num_ranges

if not is_warming_up:
Expand Down
Loading