Skip to content

Commit

Permalink
PYTHON-4879 Use shrub.py for compressor tests (#1944)
Browse files Browse the repository at this point in the history
  • Loading branch information
blink1073 authored Oct 17, 2024
1 parent 257aa24 commit 317a539
Show file tree
Hide file tree
Showing 2 changed files with 136 additions and 50 deletions.
135 changes: 86 additions & 49 deletions .evergreen/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2112,23 +2112,6 @@ axes:
AUTH: "noauth"
SSL: "nossl"

# Choice of wire protocol compression support
- id: compression
display_name: Compression
values:
- id: snappy
display_name: snappy compression
variables:
COMPRESSORS: "snappy"
- id: zlib
display_name: zlib compression
variables:
COMPRESSORS: "zlib"
- id: zstd
display_name: zstd compression
variables:
COMPRESSORS: "zstd"

# Choice of MongoDB server version
- id: mongodb-version
display_name: "MongoDB"
Expand Down Expand Up @@ -3125,6 +3108,92 @@ buildvariants:
PYTHON_BINARY: C:/python/Python313/python.exe
tags: [encryption_tag]

# Compressor tests.
- name: snappy-compression-rhel8-py3.9-no-c
tasks:
- name: .standalone
display_name: snappy compression RHEL8 py3.9 No C
run_on:
- rhel87-small
expansions:
COMPRESSORS: snappy
NO_EXT: "1"
PYTHON_BINARY: /opt/python/3.9/bin/python3
- name: snappy-compression-rhel8-py3.10
tasks:
- name: .standalone
display_name: snappy compression RHEL8 py3.10
run_on:
- rhel87-small
expansions:
COMPRESSORS: snappy
PYTHON_BINARY: /opt/python/3.10/bin/python3
- name: zlib-compression-rhel8-py3.11-no-c
tasks:
- name: .standalone
display_name: zlib compression RHEL8 py3.11 No C
run_on:
- rhel87-small
expansions:
COMPRESSORS: zlib
NO_EXT: "1"
PYTHON_BINARY: /opt/python/3.11/bin/python3
- name: zlib-compression-rhel8-py3.12
tasks:
- name: .standalone
display_name: zlib compression RHEL8 py3.12
run_on:
- rhel87-small
expansions:
COMPRESSORS: zlib
PYTHON_BINARY: /opt/python/3.12/bin/python3
- name: zstd-compression-rhel8-py3.13-no-c
tasks:
- name: .standalone !.4.0
display_name: zstd compression RHEL8 py3.13 No C
run_on:
- rhel87-small
expansions:
COMPRESSORS: zstd
NO_EXT: "1"
PYTHON_BINARY: /opt/python/3.13/bin/python3
- name: zstd-compression-rhel8-py3.9
tasks:
- name: .standalone !.4.0
display_name: zstd compression RHEL8 py3.9
run_on:
- rhel87-small
expansions:
COMPRESSORS: zstd
PYTHON_BINARY: /opt/python/3.9/bin/python3
- name: snappy-compression-rhel8-pypy3.9
tasks:
- name: .standalone
display_name: snappy compression RHEL8 pypy3.9
run_on:
- rhel87-small
expansions:
COMPRESSORS: snappy
PYTHON_BINARY: /opt/python/pypy3.9/bin/python3
- name: zlib-compression-rhel8-pypy3.10
tasks:
- name: .standalone
display_name: zlib compression RHEL8 pypy3.10
run_on:
- rhel87-small
expansions:
COMPRESSORS: zlib
PYTHON_BINARY: /opt/python/pypy3.10/bin/python3
- name: zstd-compression-rhel8-pypy3.9
tasks:
- name: .standalone !.4.0
display_name: zstd compression RHEL8 pypy3.9
run_on:
- rhel87-small
expansions:
COMPRESSORS: zstd
PYTHON_BINARY: /opt/python/pypy3.9/bin/python3

- matrix_name: "tests-fips"
matrix_spec:
platform:
Expand Down Expand Up @@ -3214,38 +3283,6 @@ buildvariants:
- ".4.2"
- ".4.0"

- matrix_name: "tests-python-version-rhel8-compression"
matrix_spec:
platform: rhel8
python-version: "*"
c-extensions: "*"
compression: "*"
exclude_spec:
# These interpreters are always tested without extensions.
- platform: rhel8
python-version: ["pypy3.9", "pypy3.10"]
c-extensions: "with-c-extensions"
compression: "*"
display_name: "${compression} ${c-extensions} ${python-version} ${platform}"
tasks:
- "test-latest-standalone"
- "test-8.0-standalone"
- "test-7.0-standalone"
- "test-6.0-standalone"
- "test-5.0-standalone"
- "test-4.4-standalone"
- "test-4.2-standalone"
- "test-4.0-standalone"
rules:
# Server version 4.0 supports snappy and zlib but not zstd.
- if:
python-version: "*"
c-extensions: "*"
compression: ["zstd"]
then:
remove_tasks:
- "test-4.0-standalone"

- matrix_name: "tests-python-version-green-framework-rhel8"
matrix_spec:
platform: rhel8
Expand Down
51 changes: 50 additions & 1 deletion .evergreen/scripts/generate_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,14 @@
BATCHTIME_WEEK = 10080
AUTH_SSLS = [("auth", "ssl"), ("noauth", "ssl"), ("noauth", "nossl")]
TOPOLOGIES = ["standalone", "replica_set", "sharded_cluster"]
C_EXTS = ["with_ext", "without_ext"]
SYNCS = ["sync", "async"]
DISPLAY_LOOKUP = dict(
ssl=dict(ssl="SSL", nossl="NoSSL"),
auth=dict(auth="Auth", noauth="NoAuth"),
test_suites=dict(default="Sync", default_async="Async"),
coverage=dict(coverage="cov"),
no_ext={"1": "No C"},
)
HOSTS = dict()

Expand Down Expand Up @@ -137,6 +139,12 @@ def zip_cycle(*iterables, empty_default=None):
yield tuple(next(i, empty_default) for i in cycles)


def handle_c_ext(c_ext, expansions):
"""Handle c extension option."""
if c_ext == C_EXTS[0]:
expansions["NO_EXT"] = "1"


def generate_yaml(tasks=None, variants=None):
"""Generate the yaml for a given set of tasks and variants."""
project = EvgProject(tasks=tasks, buildvariants=variants)
Expand Down Expand Up @@ -337,8 +345,49 @@ def create_load_balancer_variants():
return variants


def create_compression_variants():
# Compression tests - standalone versions of each server, across python versions, with and without c extensions.
# PyPy interpreters are always tested without extensions.
host = "rhel8"
task_names = dict(snappy=[".standalone"], zlib=[".standalone"], zstd=[".standalone !.4.0"])
variants = []
for ind, (compressor, c_ext) in enumerate(product(["snappy", "zlib", "zstd"], C_EXTS)):
expansions = dict(COMPRESSORS=compressor)
handle_c_ext(c_ext, expansions)
base_name = f"{compressor} compression"
python = CPYTHONS[ind % len(CPYTHONS)]
display_name = get_display_name(base_name, host, python=python, **expansions)
variant = create_variant(
task_names[compressor],
display_name,
python=python,
host=host,
expansions=expansions,
)
variants.append(variant)

other_pythons = PYPYS + CPYTHONS[ind:]
for compressor, python in zip_cycle(["snappy", "zlib", "zstd"], other_pythons):
expansions = dict(COMPRESSORS=compressor)
handle_c_ext(c_ext, expansions)
base_name = f"{compressor} compression"
display_name = get_display_name(base_name, host, python=python, **expansions)
variant = create_variant(
task_names[compressor],
display_name,
python=python,
host=host,
expansions=expansions,
)
variants.append(variant)

return variants


##################
# Generate Config
##################

generate_yaml(variants=create_load_balancer_variants())
variants = create_compression_variants()
# print(len(variants))
generate_yaml(variants=variants)

0 comments on commit 317a539

Please sign in to comment.