From 3d103d5553df0d4eb5b06c261692ef1770a2f630 Mon Sep 17 00:00:00 2001 From: Reinier Maas Date: Wed, 24 Dec 2025 16:00:56 +0100 Subject: [PATCH 1/6] Refactor `opsqueue_service` to use `wait_for_server` for `port` assignment and add utility function for server wait logic --- libs/opsqueue_python/tests/conftest.py | 10 ++--- libs/opsqueue_python/tests/util.py | 57 ++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 5 deletions(-) create mode 100644 libs/opsqueue_python/tests/util.py diff --git a/libs/opsqueue_python/tests/conftest.py b/libs/opsqueue_python/tests/conftest.py index b4b10bf..625f722 100644 --- a/libs/opsqueue_python/tests/conftest.py +++ b/libs/opsqueue_python/tests/conftest.py @@ -6,6 +6,8 @@ import subprocess import uuid import os +from libs.opsqueue_python.tests.util import wait_for_server +import psutil import pytest from dataclasses import dataclass from pathlib import Path @@ -52,13 +54,10 @@ def opsqueue() -> Generator[OpsqueueProcess, None, None]: @contextmanager def opsqueue_service( - *, port: int | None = None + *, port: int = 0, ) -> Generator[OpsqueueProcess, None, None]: global test_opsqueue_port_offset - if port is None: - port = random_free_port() - temp_dbname = f"/tmp/opsqueue_tests-{uuid.uuid4()}.db" command = [ @@ -72,7 +71,8 @@ def opsqueue_service( if env.get("RUST_LOG") is None: env["RUST_LOG"] = "off" - with subprocess.Popen(command, cwd=PROJECT_ROOT, env=env) as process: + with psutil.Popen(command, cwd=PROJECT_ROOT, env=env) as process: + _host, port = wait_for_server(process) try: wrapper = OpsqueueProcess(port=port, process=process) yield wrapper diff --git a/libs/opsqueue_python/tests/util.py b/libs/opsqueue_python/tests/util.py new file mode 100644 index 0000000..1b2d76f --- /dev/null +++ b/libs/opsqueue_python/tests/util.py @@ -0,0 +1,57 @@ +import logging + +import psutil +from opnieuw import retry +from psutil._common import pconn + +LOGGER = logging.getLogger(__name__) + + +@retry( + retry_on_exceptions=ValueError, + max_calls_total=5, + retry_window_after_first_call_in_seconds=5, +) +def wait_for_server(proc: psutil.Popen) -> tuple[str, int]: + """ + Wait for a process to be listening on a single port. + If the process is listening on no ports, a ValueError is thrown and this is retried. + If multiple ports are listening, a RuntimeError is thrown. + """ + if not proc.is_running(): + raise ValueError(f"Process {proc} is not running") + + try: + # Try to get the connections of the main process first, if that fails try the children. + # Processes wrapped with `timeout` do not have connections themselves. + connections: list[pconn] = ( + proc.net_connections() + or [ + child_conn + for child in proc.children(recursive=False) + for child_conn in child.net_connections() + ] + or [ + child_conn + for child in proc.children(recursive=True) + for child_conn in child.net_connections() + ] + ) + except psutil.AccessDenied as e: + match proc.status(): + case psutil.STATUS_ZOMBIE | psutil.STATUS_DEAD | psutil.STATUS_STOPPED: + raise RuntimeError(f"Process {proc} has exited unexpectedly") from e + case _: + raise RuntimeError( + f"Could not get `net_connections` for process {proc}, access denied " + ) from e + + ports = [x for x in connections if x.status == psutil.CONN_LISTEN] + listen_count = len(ports) + + if listen_count == 0: + raise ValueError(f"Process {proc} is not listening on any ports") + if listen_count == 1: + return ports[0].laddr + + raise RuntimeError(f"Process {proc} is listening on multiple ports") From cf605e5725b5f3780a9986b303fbc3e98d7b409a Mon Sep 17 00:00:00 2001 From: Reinier Maas Date: Tue, 30 Dec 2025 11:37:28 +0100 Subject: [PATCH 2/6] fixup! Refactor `opsqueue_service` to use `wait_for_server` for `port` assignment and add utility function for server wait logic --- libs/opsqueue_python/tests/__init__.py | 0 libs/opsqueue_python/tests/conftest.py | 10 +++++----- libs/opsqueue_python/tests/util.py | 9 ++++++--- 3 files changed, 11 insertions(+), 8 deletions(-) create mode 100644 libs/opsqueue_python/tests/__init__.py diff --git a/libs/opsqueue_python/tests/__init__.py b/libs/opsqueue_python/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/libs/opsqueue_python/tests/conftest.py b/libs/opsqueue_python/tests/conftest.py index 625f722..bf6c1d9 100644 --- a/libs/opsqueue_python/tests/conftest.py +++ b/libs/opsqueue_python/tests/conftest.py @@ -6,7 +6,6 @@ import subprocess import uuid import os -from libs.opsqueue_python.tests.util import wait_for_server import psutil import pytest from dataclasses import dataclass @@ -16,6 +15,8 @@ from opsqueue.common import SerializationFormat, json_as_bytes from opsqueue.consumer import Strategy +from tests.util import wait_for_server + # @pytest.hookimpl(tryfirst=True) # def pytest_configure(config: pytest.Config) -> None: # print("A") @@ -27,7 +28,7 @@ @dataclass class OpsqueueProcess: port: int - process: subprocess.Popen[bytes] + process: psutil.Popen # subprocess.Popen[bytes] @functools.cache @@ -54,10 +55,9 @@ def opsqueue() -> Generator[OpsqueueProcess, None, None]: @contextmanager def opsqueue_service( - *, port: int = 0, + *, + port: int = 0, ) -> Generator[OpsqueueProcess, None, None]: - global test_opsqueue_port_offset - temp_dbname = f"/tmp/opsqueue_tests-{uuid.uuid4()}.db" command = [ diff --git a/libs/opsqueue_python/tests/util.py b/libs/opsqueue_python/tests/util.py index 1b2d76f..4815b2d 100644 --- a/libs/opsqueue_python/tests/util.py +++ b/libs/opsqueue_python/tests/util.py @@ -14,9 +14,12 @@ ) def wait_for_server(proc: psutil.Popen) -> tuple[str, int]: """ - Wait for a process to be listening on a single port. - If the process is listening on no ports, a ValueError is thrown and this is retried. - If multiple ports are listening, a RuntimeError is thrown. + Wait for a process to be listening on exactly one port and return that address. + + This function expects the given process to listen on a single port only. If the + process is listening on no ports, a ValueError is raised and the check is retried. + If multiple ports are listening, a RuntimeError is raised as this indicates an + unexpected server configuration. """ if not proc.is_running(): raise ValueError(f"Process {proc} is not running") From 8f2fa09aa6cbfd30e9a912853e7a058572bc00ef Mon Sep 17 00:00:00 2001 From: Reinier Maas Date: Tue, 30 Dec 2025 11:38:19 +0100 Subject: [PATCH 3/6] Nix: import `opnieuw` and `psutil` --- .envrc | 2 +- default.nix | 7 +++++-- nix/python-overlay.nix | 16 ++++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/.envrc b/.envrc index 6fce732..087c34f 100644 --- a/.envrc +++ b/.envrc @@ -11,6 +11,6 @@ if ! has nix_direnv_version || ! nix_direnv_version 3.0.4; then source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/3.0.4/direnvrc" "sha256-DzlYZ33mWF/Gs8DDeyjr8mnVmQGx7ASYqA5WlxwvBG4=" fi -watch_file ./nix/ ./**/*.nix +watch_file ./nix/ ./**/*.nix default.nix use nix default.nix --argstr environment shell diff --git a/default.nix b/default.nix index aa5cedd..fb4cb6d 100644 --- a/default.nix +++ b/default.nix @@ -8,11 +8,14 @@ let p: with p; [ click mypy - uv + opnieuw + psutil pytest - pytest-random-order pytest-parallel + pytest-random-order pytest-timeout + types-psutil + uv # Repeated here so MyPy sees them: cbor2 diff --git a/nix/python-overlay.nix b/nix/python-overlay.nix index ad4aba1..d00f6b1 100644 --- a/nix/python-overlay.nix +++ b/nix/python-overlay.nix @@ -1,3 +1,19 @@ final: prev: { + opnieuw = final.buildPythonPackage rec { + pname = "opnieuw"; + version = "3.1.0"; + pyproject = true; + + propagatedBuildInputs = with final; [ + setuptools + setuptools-scm + ]; + + src = final.fetchPypi { + inherit pname version; + sha256 = "sha256-4aLFPQYqnCOBsxdXr41gr5sr/cT9HcN9PLEc+AWwLrY="; + }; + }; + opsqueue_python = final.callPackage ../libs/opsqueue_python/opsqueue_python.nix { }; } From 01b1ffa43a345bfc84195425f6ebb1d21dec5b98 Mon Sep 17 00:00:00 2001 From: Reinier Maas Date: Wed, 31 Dec 2025 10:27:39 +0100 Subject: [PATCH 4/6] fixup! Refactor `opsqueue_service` to use `wait_for_server` for `port` assignment and add utility function for server wait logic --- libs/opsqueue_python/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/opsqueue_python/tests/conftest.py b/libs/opsqueue_python/tests/conftest.py index bf6c1d9..c609a03 100644 --- a/libs/opsqueue_python/tests/conftest.py +++ b/libs/opsqueue_python/tests/conftest.py @@ -28,7 +28,7 @@ @dataclass class OpsqueueProcess: port: int - process: psutil.Popen # subprocess.Popen[bytes] + process: psutil.Popen @functools.cache From ba32ff4bb6a1d3b61afcb7ebadff0f26fdc3c583 Mon Sep 17 00:00:00 2001 From: Reinier Maas Date: Wed, 31 Dec 2025 10:29:38 +0100 Subject: [PATCH 5/6] fixup! Nix: import `opnieuw` and `psutil` --- libs/opsqueue_python/opsqueue_python.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libs/opsqueue_python/opsqueue_python.nix b/libs/opsqueue_python/opsqueue_python.nix index d392480..442c806 100644 --- a/libs/opsqueue_python/opsqueue_python.nix +++ b/libs/opsqueue_python/opsqueue_python.nix @@ -9,6 +9,8 @@ opentelemetry-api, opentelemetry-exporter-otlp, opentelemetry-sdk, + opnieuw, + psutil, }: let root = ../../.; @@ -63,5 +65,7 @@ buildPythonPackage rec { opentelemetry-api opentelemetry-exporter-otlp opentelemetry-sdk + opnieuw + psutil ]; } From 990fe9aa29e41fbda7a2dda9763ea8f2b0de0ae8 Mon Sep 17 00:00:00 2001 From: Reinier Maas Date: Wed, 31 Dec 2025 11:19:35 +0100 Subject: [PATCH 6/6] fixup! fixup! Nix: import `opnieuw` and `psutil` --- libs/opsqueue_python/pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libs/opsqueue_python/pyproject.toml b/libs/opsqueue_python/pyproject.toml index 097f0ee..961de96 100644 --- a/libs/opsqueue_python/pyproject.toml +++ b/libs/opsqueue_python/pyproject.toml @@ -24,6 +24,9 @@ dependencies = [ "opentelemetry-api", "opentelemetry-sdk", "opentelemetry-exporter-otlp", + # Testing: + "opnieuw", + "psutil", ] [project.urls]