From 123569e9e1af11f6dba19f228ed90e14eccb370f Mon Sep 17 00:00:00 2001 From: Ads Dawson <104169244+GangGreenTemperTatum@users.noreply.github.com> Date: Sun, 9 Feb 2025 20:30:12 -0500 Subject: [PATCH 1/2] feat: new improved ci workflows and rigging decorator --- .github/PULL_REQUEST_TEMPLATE.md | 31 ++- .github/labeler.yaml | 76 ++++++ .github/labels.yaml | 134 ++++++++++ .github/scripts/rigging_pr_decorator.py | 242 +++++++++---------- .github/workflows/rigging_pr_description.yml | 56 ++--- .hooks/linters/mdstyle.rb | 55 +++++ .hooks/linters/yamllint.yaml | 12 + 7 files changed, 440 insertions(+), 166 deletions(-) create mode 100644 .github/labeler.yaml create mode 100644 .github/labels.yaml create mode 100644 .hooks/linters/mdstyle.rb create mode 100644 .hooks/linters/yamllint.yaml diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 69c314b..19e12b1 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,5 +1,30 @@ -🏴‍☠️ Burpference +# [Title of Your PR] -## Ahoy, Mateys! +**Key Changes:** -Ahoy, ye scurvy dogs, and welcome aboard Burpference! For non-forks, leave that there pull request description blank, and let [rigging](https://github.com/dreadnode/rigging) work its sorcery like a true sea wizard. Arrr! \ No newline at end of file +- [ ] List major changes and core updates +- [ ] Keep each line under 80 characters +- [ ] Focus on the "what" and "why" + +**Added:** + +- [ ] New features/functionality +- [ ] New files/configurations +- [ ] New dependencies + +**Changed:** + +- [ ] Updates to existing code +- [ ] Configuration changes +- [ ] Dependency updates + +**Removed:** + +- [ ] Deleted files/code +- [ ] Removed dependencies +- [ ] Cleaned up configurations + +--- + + + diff --git a/.github/labeler.yaml b/.github/labeler.yaml new file mode 100644 index 0000000..a764319 --- /dev/null +++ b/.github/labeler.yaml @@ -0,0 +1,76 @@ +--- +# Area Labels +area/docs: + - changed-files: + - any-glob-to-any-file: "docs/**/*" + +area/examples: + - changed-files: + - any-glob-to-any-file: "examples/**/*" + +area/github: + - changed-files: + - any-glob-to-any-file: ".github/**/*" + +area/pre-commit: + - changed-files: + - any-glob-to-any-file: ".pre-commit-config.yaml" + - any-glob-to-any-file: ".hooks/**/*" + +area/python: + - changed-files: + - any-glob-to-any-file: "pyproject.toml" + - any-glob-to-any-file: "requirements.txt" + - any-glob-to-any-file: "*.py" + +area/security: + - changed-files: + - any-glob-to-any-file: "SECURITY.md" + - any-glob-to-any-file: "secrets.baseline" + +area/taskfiles: + - changed-files: + - any-glob-to-any-file: "Taskfile.yaml" + +area/tests: + - changed-files: + - any-glob-to-any-file: "tests/**/*" + +area/workspace: + - changed-files: + - any-glob-to-any-file: "python.code-workspace" + +# Development Labels +area/dev: + - changed-files: + - any-glob-to-any-file: "dev/**/*" + +# Semantic Type Labels +type/digest: + - head-branch: ["^renovate/"] + - head-branch: ["^deps/"] + +type/patch: + - any: ["title:/^(?:Fix|Patch|Update)/"] + +type/minor: + - any: ["title:/^(?:Add|Feature|Improve)/"] + +type/major: + - any: ["title:/^(?:BREAKING)/"] + +type/break: + - any: ["body:/BREAKING CHANGE:/"] + +# Documentation Labels +type/docs: + - changed-files: + - any-glob-to-any-file: "docs/**/*" + - any-glob-to-any-file: "*.md" + +# Core Files Labels +type/core: + - changed-files: + - any-glob-to-any-file: "CODEOWNERS" + - any-glob-to-any-file: "LICENSE" + - any-glob-to-any-file: "README.md" diff --git a/.github/labels.yaml b/.github/labels.yaml new file mode 100644 index 0000000..7e6a6c6 --- /dev/null +++ b/.github/labels.yaml @@ -0,0 +1,134 @@ +--- +# Area Labels +- name: area/docs + color: "72CCF3" # Light Blue + description: >- + Changes to documentation and guides + +- name: area/examples + color: "BC9BE3" # Lavender + description: >- + Changes to example code and demonstrations + +- name: area/github + color: "F4D1B7" # Peach + description: >- + Changes made to GitHub Actions + +- name: area/pre-commit + color: "84B6EB" # Steel Blue + description: >- + Changes made to pre-commit hooks + +- name: area/python + color: "7BD7E0" # Turquoise + description: >- + Changes to Python package configuration and dependencies + +- name: area/security + color: "FF6600" # Orange + description: >- + Changes to security policies and configurations + +- name: area/taskfiles + color: "66CCFF" # Sky Blue + description: >- + Changes made to Taskfiles + +- name: area/tests + color: "99CC00" # Lime Green + description: >- + Changes to test files and testing infrastructure + +- name: area/workspace + color: "FF99CC" # Pink + description: >- + Changes to VSCode workspace configuration + +- name: area/assets + color: "FFA07A" # Light Salmon + description: >- + Changes to asset files + +- name: area/templates + color: "DA70D6" # Orchid + description: >- + Changes to templates + +- name: area/scripts + color: "40E0D0" # Turquoise + description: >- + Changes to script files + +- name: area/src + color: "4682B4" # Steel Blue + description: >- + Changes to source code + +- name: area/ci + color: "FF4500" # Orange Red + description: >- + Changes related to CI/CD configurations + +- name: area/shell + color: "556B2F" # Dark Olive Green + description: >- + Changes to shell scripts + +- name: area/dev + color: "CC6699" # Dusty Rose + description: >- + Changes to development tools and assets + +# Renovate Labels +- name: renovate/container + color: "9933CC" # Purple + description: >- + Docker container updates via Renovate + +- name: renovate/github-action + color: "FF3366" # Hot Pink + description: >- + GitHub Action updates via Renovate + +- name: renovate/github-release + color: "3399FF" # Bright Blue + description: >- + GitHub Release updates via Renovate + +# Semantic Type Labels +- name: type/digest + color: "FF66CC" # Bright Pink + description: >- + Dependency digest updates + +- name: type/patch + color: "FFC300" # Golden Yellow + description: >- + Patch changes (fixes, updates) + +- name: type/minor + color: "FFD700" # Gold + description: >- + Minor changes (features, improvements) + +- name: type/major + color: "F6412D" # Red Orange + description: >- + Major changes + +- name: type/break + color: "FF0000" # Bright Red + description: >- + Breaking changes + +# Documentation Labels +- name: type/docs + color: "0075CA" # Documentation Blue + description: >- + Documentation updates and improvements + +- name: type/core + color: "A2EEEF" # Light Blue + description: >- + Changes to core repository files and configurations diff --git a/.github/scripts/rigging_pr_decorator.py b/.github/scripts/rigging_pr_decorator.py index feb4bd6..16d6682 100644 --- a/.github/scripts/rigging_pr_decorator.py +++ b/.github/scripts/rigging_pr_decorator.py @@ -1,142 +1,120 @@ +#!/usr/bin/env python +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "rigging", +# "typer", +# ] +# /// import asyncio -import base64 import os import typing as t -from pydantic import ConfigDict, StringConstraints - import rigging as rg -from rigging import logger -from rigging.generator import GenerateParams, Generator, register_generator - -logger.enable("rigging") - -MAX_TOKENS = 8000 -TRUNCATION_WARNING = "\n\n**Note**: Due to the large size of this diff, some content has been truncated." -str_strip = t.Annotated[str, StringConstraints(strip_whitespace=True)] - - -class PRDiffData(rg.Model): - """XML model for PR diff data""" - - content: str_strip = rg.element() - - @classmethod - def xml_example(cls) -> str: - return """example diff content""" - - -class PRDecorator(Generator): - """Generator for creating PR descriptions""" - - model_config = ConfigDict(arbitrary_types_allowed=True, validate_assignment=True) - - api_key: str = "" - max_tokens: int = MAX_TOKENS - - def __init__(self, model: str, params: rg.GenerateParams) -> None: - api_key = params.extra.get("api_key") - if not api_key: - raise ValueError("api_key is required in params.extra") - - super().__init__(model=model, params=params, api_key=api_key) - self.api_key = api_key - self.max_tokens = params.max_tokens or MAX_TOKENS - - async def generate_messages( - self, - messages: t.Sequence[t.Sequence[rg.Message]], - params: t.Sequence[GenerateParams], - ) -> t.Sequence[rg.GeneratedMessage]: - responses = [] - for message_seq, p in zip(messages, params): - base_generator = rg.get_generator(self.model, params=p) - llm_response = await base_generator.generate_messages([message_seq], [p]) - responses.extend(llm_response) - return responses - - -register_generator("pr_decorator", PRDecorator) - - -async def generate_pr_description(diff_text: str) -> str: - """Generate a PR description from the diff text""" - diff_tokens = len(diff_text) // 4 - if diff_tokens >= MAX_TOKENS: - char_limit = (MAX_TOKENS * 4) - len(TRUNCATION_WARNING) - diff_text = diff_text[:char_limit] + TRUNCATION_WARNING - - diff_data = PRDiffData(content=diff_text) - params = rg.GenerateParams( - extra={ - "api_key": os.environ["OPENAI_API_KEY"], - "diff_text": diff_text, - }, - temperature=0.1, - max_tokens=500, - ) - - generator = rg.get_generator("pr_decorator!gpt-4-turbo-preview", params=params) - prompt = f"""You are a helpful AI that generates clear and concise PR descriptions with some pirate tongue. - Analyze the provided git diff and create a summary, specifically focusing on the elements of the code that - has changed, high severity functions etc using exactly this format: - - ### PR Summary - - #### Overview of Changes - - - #### Key Modifications - 1. ****: - (continue as needed) - - #### Potential Impact - - - (continue as needed) - - Here is the PR diff to analyze: - {diff_data.to_xml()}""" - - chat = await generator.chat(prompt).run() - return chat.last.content.strip() - - -async def main(): - """Main function for CI environment""" - if not os.environ.get("OPENAI_API_KEY"): - raise ValueError("OPENAI_API_KEY environment variable must be set") - +import typer + +TRUNCATION_WARNING = "\n---\n**Note**: Due to the large size of this diff, some content has been truncated." + + +@rg.prompt +def generate_pr_description(diff: str) -> t.Annotated[str, rg.Ctx("markdown")]: # type: ignore[empty-body] + """ + Analyze the provided git diff and create a PR description in markdown format. + + - Keep the summary concise and informative. + - Use bullet points to structure important statements. + - Focus on key modifications and potential impact - if any. + - Do not add in general advice or best-practice information. + - Write like a developer who authored the changes. + - Prefer flat bullet lists over nested. + - Do not include any title structure. + - If there are no changes, just provide "No relevant changes." + - Order your bullet points by importance. + + """ + + +async def _run_git_command(args: list[str]) -> str: + """ + Safely run a git command with validated input. + """ + # Validate git exists in PATH + git_path = "git" # Could use shutil.which("git") for more security + if not any( + os.path.isfile(os.path.join(path, "git")) + for path in os.environ["PATH"].split(os.pathsep) + ): + raise ValueError("Git executable not found in PATH") + + # Validate input parameters + if not all(isinstance(arg, str) for arg in args): + raise ValueError("All command arguments must be strings") + + # Use os.execv for more secure command execution try: - diff_text = os.environ.get("GIT_DIFF", "") - if not diff_text: - raise ValueError("No diff found in GIT_DIFF environment variable") - - try: - diff_text = base64.b64decode(diff_text).decode("utf-8") - except Exception: - padding = 4 - (len(diff_text) % 4) - if padding != 4: - diff_text += "=" * padding - diff_text = base64.b64decode(diff_text).decode("utf-8") - - logger.debug(f"Processing diff of length: {len(diff_text)}") - description = await generate_pr_description(diff_text) - - with open(os.environ["GITHUB_OUTPUT"], "a") as f: - f.write("content< str: + """ + Get the git diff between two branches. + """ + # Validate refs + for ref in (base_ref, source_ref): + if not isinstance(ref, str) or not ref.strip(): + raise ValueError("Invalid git reference") + + # Get merge base + merge_base = await _run_git_command(["merge-base", source_ref, base_ref]) + + # Prepare diff command + diff_command = ["diff", "--no-color", merge_base, source_ref] + if exclude: + validated_excludes = [] + for path in exclude: + # Validate path + if not isinstance(path, str) or ".." in path: + raise ValueError(f"Invalid exclude path: {path}") + validated_excludes.append(f":(exclude){path}") + diff_command.extend(["--", ".", *validated_excludes]) + + # Get diff + return await _run_git_command(diff_command) + + +def main( + base_ref: str = "origin/main", + source_ref: str = "HEAD", + generator_id: str = "openai/gpt-4o-mini", + max_diff_lines: int = 1000, + exclude: list[str] | None = None, +) -> None: + """ + Use rigging to generate a PR description from a git diff. + """ + diff = asyncio.run(get_diff(base_ref, source_ref, exclude=exclude)) + diff_lines = diff.split("\n") + if len(diff_lines) > max_diff_lines: + diff = "\n".join(diff_lines[:max_diff_lines]) + TRUNCATION_WARNING + description = asyncio.run(generate_pr_description.bind(generator_id)(diff)) + print(description) if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file + typer.run(main) diff --git a/.github/workflows/rigging_pr_description.yml b/.github/workflows/rigging_pr_description.yml index 5bb8b8f..47d83f1 100644 --- a/.github/workflows/rigging_pr_description.yml +++ b/.github/workflows/rigging_pr_description.yml @@ -1,60 +1,54 @@ +--- name: Update PR Description with Rigging - on: pull_request: types: [opened] jobs: update-description: + name: Update PR Description with Rigging runs-on: ubuntu-latest permissions: pull-requests: write contents: read steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - fetch-depth: 0 + fetch-depth: 0 # full history for proper diffing - # Get the diff first - - name: Get Diff - id: diff - # shellcheck disable=SC2102 - run: | - git fetch origin "${{ github.base_ref }}" - MERGE_BASE=$(git merge-base HEAD "origin/${{ github.base_ref }}") - # Use separate diff arguments instead of range notation - DIFF=$(git diff "$MERGE_BASE" HEAD | base64 --wrap=0) - echo "diff=${DIFF}" >> "$GITHUB_OUTPUT" - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b #v5.0.3 + - name: Set up Python + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: - python-version: "3.11" + python-version: "3.10" - - name: Install dependencies + - name: Install uv run: | python -m pip install --upgrade pip - pip cache purge - pip install pydantic - pip install rigging[all] - # Generate the description using the diff + pip install uv + - name: Generate PR Description id: description env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - PR_NUMBER: ${{ github.event.pull_request.number }} - GIT_DIFF: ${{ steps.diff.outputs.diff }} run: | - python .github/scripts/rigging_pr_decorator.py - # Update the PR description + DESCRIPTION="$(uv run --no-project .github/scripts/rigging_pr_decorator.py --base-ref "origin/${{ github.base_ref }}" --exclude "./*.lock")" + { + echo "description<> "$GITHUB_OUTPUT" + - name: Update PR Description - uses: nefrob/pr-description@4dcc9f3ad5ec06b2a197c5f8f93db5e69d2fdca7 #v1.2.0 + uses: nefrob/pr-description@4dcc9f3ad5ec06b2a197c5f8f93db5e69d2fdca7 # v1.2.0 with: + token: ${{ secrets.GITHUB_TOKEN }} content: | - ## AI-Generated Summary - ${{ steps.description.outputs.content }} + --- + + ## Generated Summary: + + ${{ steps.description.outputs.description }} + This summary was generated with ❤️ by [rigging](https://rigging.dreadnode.io/) - regex: ".*" - regexFlags: s - token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.hooks/linters/mdstyle.rb b/.hooks/linters/mdstyle.rb new file mode 100644 index 0000000..6c7c530 --- /dev/null +++ b/.hooks/linters/mdstyle.rb @@ -0,0 +1,55 @@ +################################################################################ +# Style file for markdownlint. +# +# https://github.com/markdownlint/markdownlint/blob/master/docs/configuration.md +# +# This file is referenced by the project `.mdlrc`. +################################################################################ + +#=============================================================================== +# Start with all built-in rules. +# https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md +all + +#=============================================================================== +# Override default parameters for some built-in rules. +# https://github.com/markdownlint/markdownlint/blob/master/docs/creating_styles.md#parameters + +# Ignore line length for specific files +rule 'MD013', + ignore_code_blocks: true, + files: ['CHANGELOG.md', 'RENOVATE_TESTING.md'], + line_length: 99999 # Very high number to effectively disable for specified files + +# Allow duplicate headers in changelog files +rule 'MD024', + allow_different_nesting: true, + files: ['CHANGELOG.md'] + +#=============================================================================== +# Exclude the rules I disagree with. + +# IMHO it's easier to read lists like: +# * outmost indent +# - one indent +# - second indent +# * Another major bullet +exclude_rule 'MD004' + +# Inconsistent indentation for list items is not a problem. +exclude_rule 'MD005' + +# Ordered lists are fine. +exclude_rule 'MD029' + +# The first line doesn't always need to be a top level header. +exclude_rule 'MD041' + +# I find it necessary to use '
' to force line breaks. +exclude_rule 'MD033' # Inline HTML + +# Using bare URLs is fine. +exclude_rule 'MD034' + +# Allow emphasis to be used as headings (e.g., **Section Title**) +exclude_rule 'MD036' diff --git a/.hooks/linters/yamllint.yaml b/.hooks/linters/yamllint.yaml new file mode 100644 index 0000000..32312e4 --- /dev/null +++ b/.hooks/linters/yamllint.yaml @@ -0,0 +1,12 @@ +--- +extends: default + +rules: + line-length: + max: 400 + level: warning + truthy: false + comments: + min-spaces-from-content: 1 + braces: disable + indentation: disable From e34485a8a26c537447554bbdb8e0fbe9921644ba Mon Sep 17 00:00:00 2001 From: Ads Dawson <104169244+GangGreenTemperTatum@users.noreply.github.com> Date: Sun, 9 Feb 2025 20:38:45 -0500 Subject: [PATCH 2/2] chore: add decorator as executable --- .github/scripts/rigging_pr_decorator.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .github/scripts/rigging_pr_decorator.py diff --git a/.github/scripts/rigging_pr_decorator.py b/.github/scripts/rigging_pr_decorator.py old mode 100644 new mode 100755