From 571ed98f20bddc7d1bbce208b21eca44ce172161 Mon Sep 17 00:00:00 2001 From: Peter Su Date: Tue, 2 Dec 2025 23:38:08 +0000 Subject: [PATCH 1/7] updated test_setup.sh to respect network, subnetwork, service_account, and location environment variables --- test/integration/test_setup.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/integration/test_setup.sh b/test/integration/test_setup.sh index e60f78a..2ff4c4c 100644 --- a/test/integration/test_setup.sh +++ b/test/integration/test_setup.sh @@ -89,16 +89,23 @@ function run_dsub() { } function dsub_google-batch() { - local location="${LOCATION:-}" + # Use REGIONS env var if set, otherwise fall back to LOCATION + local location="${LOCATION:-${REGIONS:-}}" + + # Use environment variables for VPC-SC configuration if set + local network="${GPU_NETWORK:-global/networks/default}" + local subnetwork="${GPU_SUBNETWORK:-regions/us-central1/subnetworks/default}" + local service_account="${PET_SA_EMAIL:-}" dsub \ --provider google-batch \ --project "${PROJECT_ID}" \ ${location:+--location "${location}"} \ --logging "${LOGGING_OVERRIDE:-${LOGGING}}" \ - --network "global/networks/default" \ - --subnetwork "regions/us-central1/subnetworks/default" \ + --network "${network}" \ + --subnetwork "${subnetwork}" \ --use-private-address \ + ${service_account:+--service-account "${service_account}"} \ "${@}" } From 4a5a8f64ea133bc868b3e324a7bdb743ce4add5a Mon Sep 17 00:00:00 2001 From: Peter Su Date: Tue, 16 Dec 2025 16:57:07 +0000 Subject: [PATCH 2/7] updating a few integration tests to explicitly set the USER variable to jupyter if unset. test_setup.sh updated to use env variables rather than hardcoded network settings --- test/integration/e2e_logging_paths.sh | 2 +- test/integration/e2e_logging_paths_pattern_tasks.sh | 2 +- test/integration/io_setup.sh | 2 +- test/integration/io_tasks_setup.sh | 2 +- test/integration/test_setup.sh | 3 +++ 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/test/integration/e2e_logging_paths.sh b/test/integration/e2e_logging_paths.sh index 7cac80c..36be744 100755 --- a/test/integration/e2e_logging_paths.sh +++ b/test/integration/e2e_logging_paths.sh @@ -27,7 +27,7 @@ readonly LOGGING_BASE="$(dirname "${LOGGING}")" declare LOGGING_OVERRIDE readonly JOB_NAME="log-test" -readonly JOB_USER="${USER}" +readonly JOB_USER="${USER:-jupyter}" # Test a basic job with base logging path echo "Subtest #1: Basic logging path" diff --git a/test/integration/e2e_logging_paths_pattern_tasks.sh b/test/integration/e2e_logging_paths_pattern_tasks.sh index 0150fc0..8d41c70 100755 --- a/test/integration/e2e_logging_paths_pattern_tasks.sh +++ b/test/integration/e2e_logging_paths_pattern_tasks.sh @@ -31,7 +31,7 @@ readonly LOGGING_BASE="$(dirname "${LOGGING}")" declare LOGGING_OVERRIDE readonly JOB_NAME=$(logging_paths_tasks_setup::get_job_name) -readonly JOB_USER="${USER}" +readonly JOB_USER="${USER:-jupyter}" # Set up the tasks file logging_paths_tasks_setup::write_tasks_file diff --git a/test/integration/io_setup.sh b/test/integration/io_setup.sh index 79d2a24..191a948 100644 --- a/test/integration/io_setup.sh +++ b/test/integration/io_setup.sh @@ -230,7 +230,7 @@ function io_setup::check_dstat() { local dstat_output=$(run_dstat --status '*' --jobs "${job_id}" --full) echo " Checking user-id" - util::dstat_yaml_assert_field_equal "${dstat_output}" "[0].user-id" "${USER}" + util::dstat_yaml_assert_field_equal "${dstat_output}" "[0].user-id" "${USER:-jupyter}" echo " Checking logging" util::dstat_yaml_assert_field_equal "${dstat_output}" "[0].logging" "${LOGGING}" diff --git a/test/integration/io_tasks_setup.sh b/test/integration/io_tasks_setup.sh index f44f867..36336be 100644 --- a/test/integration/io_tasks_setup.sh +++ b/test/integration/io_tasks_setup.sh @@ -119,7 +119,7 @@ function io_tasks_setup::check_dstat() { echo " Check task ${task_id}" echo " Checking user-id" - util::dstat_yaml_assert_field_equal "${dstat_output}" "[0].user-id" "${USER}" + util::dstat_yaml_assert_field_equal "${dstat_output}" "[0].user-id" "${USER:-jupyter}" echo " Checking logging" util::dstat_yaml_assert_field_equal "${dstat_output}" "[0].logging" "${LOGGING}/${job_id}.${task_id}.log" diff --git a/test/integration/test_setup.sh b/test/integration/test_setup.sh index 2ff4c4c..6f1e302 100644 --- a/test/integration/test_setup.sh +++ b/test/integration/test_setup.sh @@ -30,6 +30,9 @@ # * Provide functions run_dsub, run_dstat, run_ddel which will call a function # with DSUB_PROVIDER-specific default parameters set. +# Set default USER if not already set (needed for Jupyterlab/Docker environments) +export USER="${USER:-jupyter}" + # If the DSUB_PROVIDER is not set, figure it out from the name of the script. # If the script name is ..sh, pull out the provider. # If the script name is .sh, use "local". From df7ad924473415b2a5d28aa9d2a12407c57e61d5 Mon Sep 17 00:00:00 2001 From: Peter Su Date: Tue, 16 Dec 2025 22:51:40 +0000 Subject: [PATCH 3/7] updating test_setup_e2e.py to use env variables instead of hardcoded network and SA values --- test/integration/test_setup_e2e.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/test/integration/test_setup_e2e.py b/test/integration/test_setup_e2e.py index 8f01a5f..898d878 100644 --- a/test/integration/test_setup_e2e.py +++ b/test/integration/test_setup_e2e.py @@ -171,16 +171,27 @@ def dsub_google_batch(dsub_args): if val: opt_args.append(var[1], val) - # pyformat: disable - return dsub_command.call([ + # Use environment variables for VPC-SC configuration if set + network = os.environ.get("GPU_NETWORK", "global/networks/default") + subnetwork = os.environ.get("GPU_SUBNETWORK", + "regions/us-central1/subnetworks/default") + location = os.environ.get("LOCATION", os.environ.get("REGIONS", "us-central1")) + service_account = os.environ.get("PET_SA_EMAIL", "") + + args = [ "--provider", "google-batch", "--project", PROJECT_ID, "--logging", LOGGING, - "--regions", "us-central1", - "--network", "global/networks/default", - "--subnetwork", "regions/us-central1/subnetworks/default", + "--regions", location, + "--network", network, + "--subnetwork", subnetwork, "--use-private-address" - ] + opt_args + dsub_args) + ] + if service_account: + args.extend(["--service-account", service_account]) + + # pyformat: disable + return dsub_command.call(args + opt_args + dsub_args) # pyformat: enable From 086f68d5bdc995439c0cc767f4ea5b23e792009f Mon Sep 17 00:00:00 2001 From: Peter Su Date: Fri, 19 Dec 2025 17:02:26 +0000 Subject: [PATCH 4/7] replaced default USER of jupyter with whoami, define JOB_USER in a few integration test files and use that directly --- test/integration/e2e_logging_paths.sh | 2 +- test/integration/e2e_logging_paths_pattern_tasks.sh | 2 +- test/integration/io_setup.sh | 4 +++- test/integration/io_tasks_setup.sh | 3 ++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/test/integration/e2e_logging_paths.sh b/test/integration/e2e_logging_paths.sh index 36be744..e6e8213 100755 --- a/test/integration/e2e_logging_paths.sh +++ b/test/integration/e2e_logging_paths.sh @@ -27,7 +27,7 @@ readonly LOGGING_BASE="$(dirname "${LOGGING}")" declare LOGGING_OVERRIDE readonly JOB_NAME="log-test" -readonly JOB_USER="${USER:-jupyter}" +readonly JOB_USER="${USER:-whoami}" # Test a basic job with base logging path echo "Subtest #1: Basic logging path" diff --git a/test/integration/e2e_logging_paths_pattern_tasks.sh b/test/integration/e2e_logging_paths_pattern_tasks.sh index 8d41c70..421f7c0 100755 --- a/test/integration/e2e_logging_paths_pattern_tasks.sh +++ b/test/integration/e2e_logging_paths_pattern_tasks.sh @@ -31,7 +31,7 @@ readonly LOGGING_BASE="$(dirname "${LOGGING}")" declare LOGGING_OVERRIDE readonly JOB_NAME=$(logging_paths_tasks_setup::get_job_name) -readonly JOB_USER="${USER:-jupyter}" +readonly JOB_USER="${USER:-whoami}" # Set up the tasks file logging_paths_tasks_setup::write_tasks_file diff --git a/test/integration/io_setup.sh b/test/integration/io_setup.sh index 191a948..82fae4f 100644 --- a/test/integration/io_setup.sh +++ b/test/integration/io_setup.sh @@ -26,6 +26,8 @@ readonly INPUT_BAM_MD5="4afb9b8908959dbd4e2d5c54bf254c93" readonly REQUESTER_PAYS_INPUT_BAM_FULL_PATH="gs://${DSUB_BUCKET_REQUESTER_PAYS}/${INPUT_BAM_FILE}" readonly REQUESTER_PAYS_POPULATION_FILE_FULL_PATH="gs://${DSUB_BUCKET_REQUESTER_PAYS}/${POPULATION_FILE}" +# Set user variable like in other tests +readonly JOB_USER="${USER:-whoami}" # This is the image we use to test the PD mount feature. # Inject the TEST_TOKEN into the name so that multiple tests can run # concurrently. Since the image test can be run multiple times for one @@ -230,7 +232,7 @@ function io_setup::check_dstat() { local dstat_output=$(run_dstat --status '*' --jobs "${job_id}" --full) echo " Checking user-id" - util::dstat_yaml_assert_field_equal "${dstat_output}" "[0].user-id" "${USER:-jupyter}" + util::dstat_yaml_assert_field_equal "${dstat_output}" "[0].user-id" "${JOB_USER}" echo " Checking logging" util::dstat_yaml_assert_field_equal "${dstat_output}" "[0].logging" "${LOGGING}" diff --git a/test/integration/io_tasks_setup.sh b/test/integration/io_tasks_setup.sh index 36336be..1a82399 100644 --- a/test/integration/io_tasks_setup.sh +++ b/test/integration/io_tasks_setup.sh @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +readonly JOB_USER="${USER:-whoami}" readonly POPULATION_FILE="gs://genomics-public-data/ftp-trace.ncbi.nih.gov/1000genomes/ftp/20131219.superpopulations.tsv" readonly POPULATION_MD5="68a73f849b82071afe11888bac1aa8a7" @@ -119,7 +120,7 @@ function io_tasks_setup::check_dstat() { echo " Check task ${task_id}" echo " Checking user-id" - util::dstat_yaml_assert_field_equal "${dstat_output}" "[0].user-id" "${USER:-jupyter}" + util::dstat_yaml_assert_field_equal "${dstat_output}" "[0].user-id" "${JOB_USER}" echo " Checking logging" util::dstat_yaml_assert_field_equal "${dstat_output}" "[0].logging" "${LOGGING}/${job_id}.${task_id}.log" From 34651dd3b27fabf185ad6cc5199e22c40e13004f Mon Sep 17 00:00:00 2001 From: Peter Su Date: Mon, 22 Dec 2025 22:04:36 +0000 Subject: [PATCH 5/7] upping version to 0.5.2 to prepare for merge to main --- dsub/_dsub_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsub/_dsub_version.py b/dsub/_dsub_version.py index 71bbde4..050f045 100644 --- a/dsub/_dsub_version.py +++ b/dsub/_dsub_version.py @@ -27,5 +27,5 @@ """ -DSUB_VERSION = '0.5.2.dev0' +DSUB_VERSION = '0.5.2' From 60c98bf00f30c7d15719c7b2f75220b42a58af23 Mon Sep 17 00:00:00 2001 From: Peter Su Date: Fri, 2 Jan 2026 20:08:43 +0000 Subject: [PATCH 6/7] replaced whoami string with user to get the actual logged in user --- test/integration/e2e_logging_paths.sh | 2 +- test/integration/e2e_logging_paths_pattern_tasks.sh | 2 +- test/integration/io_setup.sh | 2 +- test/integration/io_tasks_setup.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/integration/e2e_logging_paths.sh b/test/integration/e2e_logging_paths.sh index e6e8213..314cfd5 100755 --- a/test/integration/e2e_logging_paths.sh +++ b/test/integration/e2e_logging_paths.sh @@ -27,7 +27,7 @@ readonly LOGGING_BASE="$(dirname "${LOGGING}")" declare LOGGING_OVERRIDE readonly JOB_NAME="log-test" -readonly JOB_USER="${USER:-whoami}" +readonly JOB_USER="${USER:-$(whoami)}" # Test a basic job with base logging path echo "Subtest #1: Basic logging path" diff --git a/test/integration/e2e_logging_paths_pattern_tasks.sh b/test/integration/e2e_logging_paths_pattern_tasks.sh index 421f7c0..5524c01 100755 --- a/test/integration/e2e_logging_paths_pattern_tasks.sh +++ b/test/integration/e2e_logging_paths_pattern_tasks.sh @@ -31,7 +31,7 @@ readonly LOGGING_BASE="$(dirname "${LOGGING}")" declare LOGGING_OVERRIDE readonly JOB_NAME=$(logging_paths_tasks_setup::get_job_name) -readonly JOB_USER="${USER:-whoami}" +readonly JOB_USER="${USER:-$(whoami)}" # Set up the tasks file logging_paths_tasks_setup::write_tasks_file diff --git a/test/integration/io_setup.sh b/test/integration/io_setup.sh index 82fae4f..6cf7caf 100644 --- a/test/integration/io_setup.sh +++ b/test/integration/io_setup.sh @@ -27,7 +27,7 @@ readonly REQUESTER_PAYS_INPUT_BAM_FULL_PATH="gs://${DSUB_BUCKET_REQUESTER_PAYS}/ readonly REQUESTER_PAYS_POPULATION_FILE_FULL_PATH="gs://${DSUB_BUCKET_REQUESTER_PAYS}/${POPULATION_FILE}" # Set user variable like in other tests -readonly JOB_USER="${USER:-whoami}" +readonly JOB_USER="${USER:-$(whoami)}" # This is the image we use to test the PD mount feature. # Inject the TEST_TOKEN into the name so that multiple tests can run # concurrently. Since the image test can be run multiple times for one diff --git a/test/integration/io_tasks_setup.sh b/test/integration/io_tasks_setup.sh index 1a82399..9b93627 100644 --- a/test/integration/io_tasks_setup.sh +++ b/test/integration/io_tasks_setup.sh @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -readonly JOB_USER="${USER:-whoami}" +readonly JOB_USER="${USER:-$(whoami)}" readonly POPULATION_FILE="gs://genomics-public-data/ftp-trace.ncbi.nih.gov/1000genomes/ftp/20131219.superpopulations.tsv" readonly POPULATION_MD5="68a73f849b82071afe11888bac1aa8a7" From b101d8a04fe188b91937691a9fbbb12381254b89 Mon Sep 17 00:00:00 2001 From: Peter Su Date: Tue, 6 Jan 2026 17:56:05 +0000 Subject: [PATCH 7/7] updating user from jupyter to user in test/integration/test_setup.sh --- test/integration/test_setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/test_setup.sh b/test/integration/test_setup.sh index 6f1e302..769e0f6 100644 --- a/test/integration/test_setup.sh +++ b/test/integration/test_setup.sh @@ -31,7 +31,7 @@ # with DSUB_PROVIDER-specific default parameters set. # Set default USER if not already set (needed for Jupyterlab/Docker environments) -export USER="${USER:-jupyter}" +export USER="${USER:-$(whoami)}" # If the DSUB_PROVIDER is not set, figure it out from the name of the script. # If the script name is ..sh, pull out the provider.