From b7e6c250f64821cd97948ef7872570df0a6432cf Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Tue, 13 Jan 2026 08:35:28 +0000 Subject: [PATCH 1/6] migrate to Checks API for integration tests Use GitHub Checks API instead of Statuses API to report integration test results. This enables the use of GitHub App authentication and eliminates the need for monthly PAT rotation. Changes: - Generate a second GitHub App token for check creation - Create check run before triggering tests in eng-dev-ecosystem - Pass check_run_id to the workflow for status updates - Update get_status() to query Checks API instead of Statuses API --- .github/workflows/start-integration-tests.yml | 11 ++- tools/start_integration_tests.py | 92 ++++++++++++++----- 2 files changed, 78 insertions(+), 25 deletions(-) diff --git a/.github/workflows/start-integration-tests.yml b/.github/workflows/start-integration-tests.yml index 66ecfde2d4..ce3649a25e 100644 --- a/.github/workflows/start-integration-tests.yml +++ b/.github/workflows/start-integration-tests.yml @@ -22,7 +22,7 @@ jobs: if: "${{ !github.event.pull_request.head.repo.fork }}" steps: - - name: Generate GitHub App Token + - name: Generate GitHub App Token for Workflow Trigger id: generate-token uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6 with: @@ -31,11 +31,20 @@ jobs: owner: ${{ secrets.ORG_NAME }} repositories: ${{secrets.REPO_NAME}} + - name: Generate GitHub App Token for Check Updates + id: generate-check-token + uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6 + with: + app-id: ${{ secrets.DECO_TEST_APPROVAL_APP_ID }} + private-key: ${{ secrets.DECO_TEST_APPROVAL_PRIVATE_KEY }} + owner: databricks + - name: Fetch start_integration_tests.py run: wget https://raw.githubusercontent.com/databricks/cli/refs/heads/main/tools/start_integration_tests.py - name: Run start_integration_tests.py env: GH_TOKEN: ${{ steps.generate-token.outputs.token }} + GH_CHECK_TOKEN: ${{ steps.generate-check-token.outputs.token }} run: |- python3 ./start_integration_tests.py -R ${{ secrets.ORG_NAME }}/${{secrets.REPO_NAME}} --yes diff --git a/tools/start_integration_tests.py b/tools/start_integration_tests.py index 088c2a1711..8d353b9d38 100755 --- a/tools/start_integration_tests.py +++ b/tools/start_integration_tests.py @@ -8,10 +8,9 @@ import argparse import json +import os import subprocess import sys -from pathlib import Path -import re CLI_REPO = "databricks/cli" @@ -20,14 +19,17 @@ ALLOWED_HEAD_OWNER = {"id": "MDEyOk9yZ2FuaXphdGlvbjQ5OTgwNTI=", "login": "databricks"} -def run(cmd): +def run(cmd, env=None): sys.stderr.write("+ " + " ".join(cmd) + "\n") - return subprocess.run(cmd, check=True) + return subprocess.run(cmd, check=True, env=env) -def run_json(cmd): +def run_json(cmd, env=None): sys.stderr.write("+ " + " ".join(cmd) + "\n") - result = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="utf-8", check=True) + run_env = os.environ.copy() + if env: + run_env.update(env) + result = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="utf-8", check=True, env=run_env) try: return json.loads(result.stdout) @@ -36,6 +38,38 @@ def run_json(cmd): raise +def create_check(commit_sha): + """Create a check run for the given commit and return the check_run_id.""" + check_token = os.environ.get("GH_CHECK_TOKEN") + if not check_token: + print("Warning: GH_CHECK_TOKEN not set, skipping check creation") + return None + + response = run_json( + [ + "gh", + "api", + "-X", + "POST", + f"/repos/{CLI_REPO}/check-runs", + "-f", + "name=Integration Tests", + "-f", + f"head_sha={commit_sha}", + "-f", + "status=queued", + "-f", + "output[title]=Integration Tests", + "-f", + "output[summary]=Tests queued and will be triggered shortly...", + ], + env={"GH_TOKEN": check_token}, + ) + check_run_id = response.get("id") + print(f"Created check run: {check_run_id}") + return check_run_id + + def get_approved_prs_by_non_team(): prs = run_json( [ @@ -108,30 +142,40 @@ def start_job(pr_number, commit_sha, author, approved_by, workflow, repo, force= response = input("Start integration tests? (y/n): ") if response.lower() == "y": - result = run( - [ - "gh", - "workflow", - "run", - workflow, - "-R", - repo, - "-F", - f"pull_request_number={pr_number}", - "-F", - f"commit_sha={commit_sha}", - ], - ) + check_run_id = create_check(commit_sha) + + cmd = [ + "gh", + "workflow", + "run", + workflow, + "-R", + repo, + "-F", + f"pull_request_number={pr_number}", + "-F", + f"commit_sha={commit_sha}", + ] + if check_run_id: + cmd.extend(["-F", f"check_run_id={check_run_id}"]) + + run(cmd) print(f"Started integration tests for PR #{pr_number}") def get_status(commit_sha): - statuses = run_json(["gh", "api", f"repos/{CLI_REPO}/commits/{commit_sha}/statuses"]) + response = run_json(["gh", "api", f"repos/{CLI_REPO}/commits/{commit_sha}/check-runs"]) result = [] - for st in statuses: - if st["context"] != "Integration Tests Check": + for check in response.get("check_runs", []): + if check["name"] != "Integration Tests": continue - result.append(f"{st['state']} {st['target_url']}") + status = check["status"] + conclusion = check.get("conclusion", "") + details_url = check.get("details_url", "") + if conclusion: + result.append(f"{conclusion} {details_url}") + else: + result.append(f"{status} {details_url}") return result From 96835b07f120c0002a50131b92b3758fe8942dd1 Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Mon, 19 Jan 2026 11:28:07 +0000 Subject: [PATCH 2/6] add --summary-file flag to gh_report and gh_parse allows ecosystem workflows to extract just the summary line without grep parsing --- tools/gh_parse.py | 17 ++++++++++++++++- tools/gh_report.py | 3 +++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/tools/gh_parse.py b/tools/gh_parse.py index 3425aa3e79..2a57c16c80 100755 --- a/tools/gh_parse.py +++ b/tools/gh_parse.py @@ -353,7 +353,7 @@ def mark_known_failures(results, known_failures_config): return marked_results -def print_report(filenames, filter, filter_env, show_output, markdown=False, omit_repl=False): +def print_report(filenames, filter, filter_env, show_output, markdown=False, omit_repl=False, summary_file=None): known_failures_config = load_known_failures() outputs = {} # test_key -> env -> [output] per_test_per_env_stats = {} # test_key -> env -> action -> count @@ -536,12 +536,25 @@ def key(column): } ) table_txt = format_table(table, markdown=markdown) + summary_msg = None if len(table) > 5: summary_msg = make_summary_message(table, summary) table_txt = wrap_in_details(table_txt, summary_msg) if table_txt: print(table_txt) + # Write summary to file if requested + if summary_file: + if summary_msg: + Path(summary_file).write_text(summary_msg) + elif len(table) > 0: + # If we have tests but not enough for full summary, create a simple one + summary_msg = make_summary_message(table, summary) + Path(summary_file).write_text(summary_msg) + else: + # No interesting tests found + Path(summary_file).write_text("All tests passed") + # Generate slowest tests table (tests slower than 10 minutes) all_durations = [] # [(env, package, testname, duration), ...] for env, env_durations in durations_by_env.items(): @@ -691,6 +704,7 @@ def main(): help="Omit lines starting with 'REPL' and containing 'Available replacements:'", action="store_true", ) + parser.add_argument("--summary-file", help="Write summary line to this file") args = parser.parse_args() print_report( args.filenames, @@ -699,6 +713,7 @@ def main(): show_output=args.output, markdown=args.markdown, omit_repl=args.omit_repl, + summary_file=args.summary_file, ) diff --git a/tools/gh_report.py b/tools/gh_report.py index ff1b69f416..21c0bea4ca 100755 --- a/tools/gh_report.py +++ b/tools/gh_report.py @@ -175,6 +175,7 @@ def main(): help="Omit lines starting with 'REPL' and containing 'Available replacements:'", action="store_true", ) + parser.add_argument("--summary-file", help="Write summary line to this file") # This does not work because we don't store artifacts for unit tests. We could download logs instead but that requires different parsing method: # ~/work/cli % gh api -H "Accept: application/vnd.github+json" /repos/databricks/cli/actions/runs/15827411452/logs > logs.zip @@ -217,6 +218,8 @@ def main(): cmd.append("--markdown") if args.omit_repl: cmd.append("--omit-repl") + if args.summary_file: + cmd.extend(["--summary-file", args.summary_file]) cmd.append(f"{target_dir}") run(cmd, shell=True) From 30c2ee0ceb8960ac30daed7a5cc47b6772f02844 Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Mon, 19 Jan 2026 11:34:52 +0000 Subject: [PATCH 3/6] temp: point to ecosystem test branch for testing temporarily use omer-lachish_data/deco-26173-cli-report-in-checks branch to test new check report functionality before merging --- tools/start_integration_tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/start_integration_tests.py b/tools/start_integration_tests.py index 8d353b9d38..d8c8734aa5 100755 --- a/tools/start_integration_tests.py +++ b/tools/start_integration_tests.py @@ -151,6 +151,8 @@ def start_job(pr_number, commit_sha, author, approved_by, workflow, repo, force= workflow, "-R", repo, + "--ref", + "omer-lachish_data/deco-26173-cli-report-in-checks", "-F", f"pull_request_number={pr_number}", "-F", From efaf099019e86a7c969f39f9aa2903bf80614312 Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Mon, 19 Jan 2026 15:24:24 +0000 Subject: [PATCH 4/6] remove temporary testing artifacts --- .github/workflows/start-integration-tests.yml | 2 -- tools/start_integration_tests.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/start-integration-tests.yml b/.github/workflows/start-integration-tests.yml index ce3649a25e..94f5b88586 100644 --- a/.github/workflows/start-integration-tests.yml +++ b/.github/workflows/start-integration-tests.yml @@ -1,8 +1,6 @@ name: start-integration-tests on: - #schedule: - # - cron: '*/10 * * * *' workflow_dispatch: jobs: diff --git a/tools/start_integration_tests.py b/tools/start_integration_tests.py index d8c8734aa5..0f9b7a77e3 100755 --- a/tools/start_integration_tests.py +++ b/tools/start_integration_tests.py @@ -152,7 +152,7 @@ def start_job(pr_number, commit_sha, author, approved_by, workflow, repo, force= "-R", repo, "--ref", - "omer-lachish_data/deco-26173-cli-report-in-checks", + "main", "-F", f"pull_request_number={pr_number}", "-F", From 6f78fff781dbf06207596558659f52f82a5ff92e Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Mon, 19 Jan 2026 15:51:13 +0000 Subject: [PATCH 5/6] remove --summary-file changes (will be in separate PR) --- tools/gh_parse.py | 17 +---------------- tools/gh_report.py | 3 --- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/tools/gh_parse.py b/tools/gh_parse.py index 2a57c16c80..3425aa3e79 100755 --- a/tools/gh_parse.py +++ b/tools/gh_parse.py @@ -353,7 +353,7 @@ def mark_known_failures(results, known_failures_config): return marked_results -def print_report(filenames, filter, filter_env, show_output, markdown=False, omit_repl=False, summary_file=None): +def print_report(filenames, filter, filter_env, show_output, markdown=False, omit_repl=False): known_failures_config = load_known_failures() outputs = {} # test_key -> env -> [output] per_test_per_env_stats = {} # test_key -> env -> action -> count @@ -536,25 +536,12 @@ def key(column): } ) table_txt = format_table(table, markdown=markdown) - summary_msg = None if len(table) > 5: summary_msg = make_summary_message(table, summary) table_txt = wrap_in_details(table_txt, summary_msg) if table_txt: print(table_txt) - # Write summary to file if requested - if summary_file: - if summary_msg: - Path(summary_file).write_text(summary_msg) - elif len(table) > 0: - # If we have tests but not enough for full summary, create a simple one - summary_msg = make_summary_message(table, summary) - Path(summary_file).write_text(summary_msg) - else: - # No interesting tests found - Path(summary_file).write_text("All tests passed") - # Generate slowest tests table (tests slower than 10 minutes) all_durations = [] # [(env, package, testname, duration), ...] for env, env_durations in durations_by_env.items(): @@ -704,7 +691,6 @@ def main(): help="Omit lines starting with 'REPL' and containing 'Available replacements:'", action="store_true", ) - parser.add_argument("--summary-file", help="Write summary line to this file") args = parser.parse_args() print_report( args.filenames, @@ -713,7 +699,6 @@ def main(): show_output=args.output, markdown=args.markdown, omit_repl=args.omit_repl, - summary_file=args.summary_file, ) diff --git a/tools/gh_report.py b/tools/gh_report.py index 21c0bea4ca..ff1b69f416 100755 --- a/tools/gh_report.py +++ b/tools/gh_report.py @@ -175,7 +175,6 @@ def main(): help="Omit lines starting with 'REPL' and containing 'Available replacements:'", action="store_true", ) - parser.add_argument("--summary-file", help="Write summary line to this file") # This does not work because we don't store artifacts for unit tests. We could download logs instead but that requires different parsing method: # ~/work/cli % gh api -H "Accept: application/vnd.github+json" /repos/databricks/cli/actions/runs/15827411452/logs > logs.zip @@ -218,8 +217,6 @@ def main(): cmd.append("--markdown") if args.omit_repl: cmd.append("--omit-repl") - if args.summary_file: - cmd.extend(["--summary-file", args.summary_file]) cmd.append(f"{target_dir}") run(cmd, shell=True) From 6760de5386cf1755444c50cc917fdd49714bac66 Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Tue, 20 Jan 2026 08:49:50 +0000 Subject: [PATCH 6/6] auto-approve integration tests for merge queue --- .github/workflows/start-integration-tests.yml | 41 ++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/.github/workflows/start-integration-tests.yml b/.github/workflows/start-integration-tests.yml index 94f5b88586..99919beab9 100644 --- a/.github/workflows/start-integration-tests.yml +++ b/.github/workflows/start-integration-tests.yml @@ -2,23 +2,52 @@ name: start-integration-tests on: workflow_dispatch: + merge_group: jobs: - # Trigger for pull requests. - # + # Auto-approve integration tests for merge queue. + # The tests already passed on the PR, so we don't need to run them again. + auto-approve: + if: github.event_name == 'merge_group' + + runs-on: + group: databricks-deco-testing-runner-group + labels: ubuntu-latest-deco + + permissions: + checks: write + contents: read + + steps: + - name: Auto-approve Check for Merge Queue + uses: actions/github-script@v7 + with: + script: | + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + name: 'Integration Tests', + head_sha: context.sha, + status: 'completed', + conclusion: 'success', + output: { + title: 'Integration Tests', + summary: 'Auto-approved for merge queue (tests already passed on PR)' + } + }); + + # Trigger integration tests for PRs. # This workflow triggers the integration test workflow in a different repository. # It requires secrets from the "test-trigger-is" environment, which are only available to authorized users. trigger: + if: github.event_name == 'workflow_dispatch' + runs-on: group: databricks-deco-testing-runner-group labels: ubuntu-latest-deco environment: "test-trigger-is" - # Only run this job for PRs from branches on the main repository and not from forks. - # Workflows triggered by PRs from forks don't have access to the "test-trigger-is" environment. - if: "${{ !github.event.pull_request.head.repo.fork }}" - steps: - name: Generate GitHub App Token for Workflow Trigger id: generate-token