From e309529116b04fbf187adee74b7165798df09483 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 11 Dec 2025 22:17:58 +0100 Subject: [PATCH 1/3] refactor(devshell): improve devshell experience The current development shell include a large number of packages which leads to a very large closure size (5 Gb) and slow startup times. To improve the situation we defer installing large packages to devshell commands which only get installed when needed. We introduce the use of devshell to define the commands in a declarative way as well as to improve the overall developer experience. Included changes: - Add organized command menu with categories (check, ami, extension, postgres) - Add watch command using watchexec + nix-fast-build for continuous checking - Expose common tasks as named commands: fmt, check, lint, watch - Add aws-vault to development tools --- flake.lock | 21 +++++++++ flake.nix | 2 + nix/checks.nix | 5 +-- nix/devShells.nix | 106 ++++++++++++++++++++++++++++++++++++++-------- nix/nixpkgs.nix | 1 + 5 files changed, 115 insertions(+), 20 deletions(-) diff --git a/flake.lock b/flake.lock index 02d9a0986..d2f121414 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,25 @@ { "nodes": { + "devshell": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1764011051, + "narHash": "sha256-M7SZyPZiqZUR/EiiBJnmyUbOi5oE/03tCeFrTiUZchI=", + "owner": "numtide", + "repo": "devshell", + "rev": "17ed8d9744ebe70424659b0ef74ad6d41fc87071", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "devshell", + "type": "github" + } + }, "flake-compat": { "flake": false, "locked": { @@ -238,6 +258,7 @@ }, "root": { "inputs": { + "devshell": "devshell", "flake-parts": "flake-parts", "flake-utils": "flake-utils", "git-hooks": "git-hooks", diff --git a/flake.nix b/flake.nix index 595cd2d96..2145185d1 100644 --- a/flake.nix +++ b/flake.nix @@ -7,6 +7,8 @@ ]; }; inputs = { + devshell.url = "github:numtide/devshell"; + devshell.inputs.nixpkgs.follows = "nixpkgs"; flake-parts.url = "github:hercules-ci/flake-parts"; flake-utils.url = "github:numtide/flake-utils"; git-hooks.inputs.nixpkgs.follows = "nixpkgs"; diff --git a/nix/checks.nix b/nix/checks.nix index 7370aee9b..31c7bfaba 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -3,7 +3,6 @@ perSystem = { self', - system, pkgs, lib, ... @@ -397,6 +396,7 @@ goss supascan ; + devShell = self'.devShells.default; } // pkgs.lib.optionalAttrs (pkgs.stdenv.isLinux) ( { @@ -413,7 +413,6 @@ inherit self; inherit pkgs; }) - ) - // pkgs.lib.optionalAttrs (system == "x86_64-linux") ({ devShell = self'.devShells.default; }); + ); }; } diff --git a/nix/devShells.nix b/nix/devShells.nix index d4fdd0804..ef464b949 100644 --- a/nix/devShells.nix +++ b/nix/devShells.nix @@ -5,6 +5,7 @@ pkgs, self', config, + lib, ... }: let @@ -34,7 +35,7 @@ in { devShells = { - default = pkgs.mkShell { + default = pkgs.devshell.mkShell { packages = with pkgs; [ @@ -45,28 +46,99 @@ shellcheck ansible ansible-lint - self'.packages.packer - - self'.packages.start-server - self'.packages.start-client - self'.packages.start-replica - self'.packages.migrate-tool - self'.packages.sync-exts-versions - self'.packages.build-test-ami - self'.packages.run-testinfra - self'.packages.cleanup-ami - self'.packages.supascan - self'.packages.goss + aws-vault + packer dbmate nushell pythonEnv config.treefmt.build.wrapper ] ++ self'.packages.docs.nativeBuildInputs; - shellHook = '' - export HISTFILE=.history - ${config.pre-commit.installationScript} - ''; + devshell.startup.pre-commit.text = config.pre-commit.installationScript; + commands = [ + { + name = "fmt"; + help = "Format code"; + command = "nix fmt"; + category = "check"; + } + { + name = "check"; + help = "Run all checks"; + command = "nix flake -L check -v"; + category = "check"; + } + { + name = "lint"; + help = "Lint code"; + command = "pre-commit run --all-files"; + category = "check"; + } + { + name = "watch"; + help = "Watch for file changes and run all checks"; + command = + let + watchExec = lib.getExe pkgs.watchexec; + nixFastBuild = '' + ${lib.getExe pkgs.nix} run github:Mic92/nix-fast-build -- \ + --skip-cached --retries=2 --no-download --option warn-dirty false \ + --option accept-flake-config true --no-link \ + --flake ".#checks.${pkgs.stdenv.hostPlatform.system}" + ''; + in + "${watchExec} --on-busy-update=queue -w . --ignore '.jj/*' --timings -- ${nixFastBuild}"; + category = "check"; + } + { + name = "cleanup-ami"; + help = "Deregister AMIs by name"; + command = "${lib.getExe self'.packages.cleanup-ami} $@"; + category = "ami"; + } + { + name = "build-test-ami"; + help = "Build AMI images for PostgreSQL testing"; + command = "${lib.getExe self'.packages.build-test-ami} $@"; + category = "ami"; + } + { + name = "sync-exts-versions"; + help = "Update extensions versions"; + command = "${lib.getExe self'.packages.sync-exts-versions}"; + category = "extension"; + } + { + name = "start-postgres-server"; + help = "Start a local Postgres server"; + command = "${lib.getExe pkgs.nix} run .#start-server -- $@"; + category = "postgres"; + } + { + name = "start-postgres-client"; + help = "Start an interactive psql with the specified Postgres version"; + command = "${lib.getExe pkgs.nix} run .#start-client -- $@"; + category = "postgres"; + } + { + name = "start-postgres-replica"; + help = "Start a local Postgres replica server"; + command = "${lib.getExe pkgs.nix} run .#start-replica -- $@"; + category = "postgres"; + } + { + name = "migrate-postgres"; + help = "Run database migrations"; + command = "${lib.getExe pkgs.nix} run .#migrate-tool -- $@"; + category = "postgres"; + } + { + name = "dbmate-tool"; + help = "Run dbmate against specified local Postgres database"; + command = "${lib.getExe pkgs.nix} run .#dbmate-tool -- $@"; + category = "postgres"; + } + ]; }; cargo-pgrx_0_11_3 = mkCargoPgrxDevShell { pgrxVersion = "0_11_3"; diff --git a/nix/nixpkgs.nix b/nix/nixpkgs.nix index 34bfae126..34ba4bd60 100644 --- a/nix/nixpkgs.nix +++ b/nix/nixpkgs.nix @@ -23,6 +23,7 @@ v8_oldstable = oldstable.v8; } ) + inputs.devshell.overlays.default ]; }; }; From 36caa9802fcf28cbd09ea24c67e756228d98d882 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 11 Dec 2025 22:17:58 +0100 Subject: [PATCH 2/3] chore: migrate scripts generated by runCommand to writeShellApplication `writeShellApplication` avoids substitute commands, provides the correct package metadata and make sure that all scripts are shellcheked. --- nix/apps.nix | 34 +-- nix/packages/build-test-ami.nix | 80 +++---- nix/packages/cleanup-ami.nix | 49 ++-- nix/packages/dbmate-tool.nix | 355 +++++++++++++++++++++++++--- nix/packages/migrate-tool.nix | 134 ++++++++++- nix/packages/start-client.nix | 118 +++++++-- nix/packages/start-replica.nix | 52 +++- nix/packages/start-replica.sh.in | 43 ---- nix/packages/sync-exts-versions.nix | 24 +- nix/tools/dbmate-tool.sh.in | 306 ------------------------ nix/tools/migrate-tool.sh.in | 123 ---------- nix/tools/run-client.sh.in | 107 --------- nix/tools/sync-exts-versions.sh.in | 119 +++++----- 13 files changed, 714 insertions(+), 830 deletions(-) delete mode 100644 nix/packages/start-replica.sh.in delete mode 100644 nix/tools/dbmate-tool.sh.in delete mode 100644 nix/tools/migrate-tool.sh.in delete mode 100644 nix/tools/run-client.sh.in diff --git a/nix/apps.nix b/nix/apps.nix index acf42910c..5bade40cf 100644 --- a/nix/apps.nix +++ b/nix/apps.nix @@ -1,11 +1,11 @@ { ... }: { perSystem = - { self', ... }: + { self', lib, ... }: let - mkApp = attrName: binName: { + mkApp = attrName: { type = "app"; - program = "${self'.packages."${attrName}"}/bin/${binName}"; + program = lib.getExe self'.packages."${attrName}"; }; in { @@ -13,20 +13,20 @@ # these are distinct from the things that can be built with 'nix build', # so they need to be listed here too. apps = { - start-server = mkApp "start-server" "start-postgres-server"; - start-client = mkApp "start-client" "start-postgres-client"; - start-replica = mkApp "start-replica" "start-postgres-replica"; - # migrate-postgres = mkApp "migrate-tool" "migrate-postgres"; - # sync-exts-versions = mkApp "sync-exts-versions" "sync-exts-versions"; - pg-restore = mkApp "pg-restore" "pg-restore"; - local-infra-bootstrap = mkApp "local-infra-bootstrap" "local-infra-bootstrap"; - dbmate-tool = mkApp "dbmate-tool" "dbmate-tool"; - update-readme = mkApp "update-readme" "update-readme"; - show-commands = mkApp "show-commands" "show-commands"; - build-test-ami = mkApp "build-test-ami" "build-test-ami"; - run-testinfra = mkApp "run-testinfra" "run-testinfra"; - cleanup-ami = mkApp "cleanup-ami" "cleanup-ami"; - trigger-nix-build = mkApp "trigger-nix-build" "trigger-nix-build"; + start-server = mkApp "start-server"; + start-client = mkApp "start-client"; + start-replica = mkApp "start-replica"; + # migrate-postgres = mkApp "migrate-tool"; + # sync-exts-versions = mkApp "sync-exts-versions"; + pg-restore = mkApp "pg-restore"; + local-infra-bootstrap = mkApp "local-infra-bootstrap"; + dbmate-tool = mkApp "dbmate-tool"; + update-readme = mkApp "update-readme"; + show-commands = mkApp "show-commands"; + build-test-ami = mkApp "build-test-ami"; + run-testinfra = mkApp "run-testinfra"; + cleanup-ami = mkApp "cleanup-ami"; + trigger-nix-build = mkApp "trigger-nix-build"; supascan = mkApp "supascan" "supascan"; }; }; diff --git a/nix/packages/build-test-ami.nix b/nix/packages/build-test-ami.nix index 9a1d4c5d6..99a16869a 100644 --- a/nix/packages/build-test-ami.nix +++ b/nix/packages/build-test-ami.nix @@ -1,23 +1,29 @@ -{ pkgs, runCommand }: -runCommand "build-test-ami" - { - buildInputs = with pkgs; [ - packer - awscli2 - yq - jq - openssl - git - coreutils - aws-vault - ]; - } - '' - mkdir -p $out/bin - cat > $out/bin/build-test-ami << 'EOL' - #!/usr/bin/env bash - set -euo pipefail - +{ + writeShellApplication, + packer, + awscli2, + yq, + jq, + openssl, + gitMinimal, + coreutils, + aws-vault, + python3, +}: +writeShellApplication { + name = "build-test-ami"; + runtimeInputs = [ + packer + awscli2 + yq + jq + openssl + gitMinimal + coreutils + aws-vault + python3 + ]; + text = '' show_help() { cat << EOF Usage: build-test-ami [--help] @@ -52,30 +58,6 @@ runCommand "build-test-ami" exit 0 fi - export PATH="${ - pkgs.lib.makeBinPath ( - with pkgs; - [ - packer - awscli2 - yq - jq - openssl - git - coreutils - aws-vault - ] - ) - }:$PATH" - - # Check for required tools - for cmd in packer aws-vault yq jq openssl; do - if ! command -v $cmd &> /dev/null; then - echo "Error: $cmd is required but not found" - exit 1 - fi - done - # Check AWS Vault profile if [ -z "''${AWS_VAULT:-}" ]; then echo "Error: AWS_VAULT environment variable must be set with the profile name" @@ -140,18 +122,18 @@ runCommand "build-test-ami" VENV_DIR=$(mktemp -d) trap 'rm -rf "$VENV_DIR"' EXIT HUP INT QUIT TERM python3 -m venv "$VENV_DIR" + # shellcheck source=/dev/null source "$VENV_DIR/bin/activate" # Install required Python packages echo "Installing required Python packages..." - pip install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest paramiko requests + pip install boto3 'boto3-stubs[essential]' docker ec2instanceconnectcli pytest paramiko requests # Run the tests with aws-vault echo "Running tests for AMI: $RANDOM_STRING using AWS Vault profile: $AWS_VAULT_PROFILE" - aws-vault exec $AWS_VAULT_PROFILE -- pytest -vv -s testinfra/test_ami_nix.py + aws-vault exec "$AWS_VAULT_PROFILE" -- pytest -vv -s testinfra/test_ami_nix.py # Deactivate virtual environment (cleanup is handled by trap) deactivate - EOL - chmod +x $out/bin/build-test-ami - '' + ''; +} diff --git a/nix/packages/cleanup-ami.nix b/nix/packages/cleanup-ami.nix index f1f9cde27..9bd670804 100644 --- a/nix/packages/cleanup-ami.nix +++ b/nix/packages/cleanup-ami.nix @@ -1,35 +1,15 @@ -{ pkgs, runCommand }: -runCommand "cleanup-ami" - { - buildInputs = with pkgs; [ - awscli2 - aws-vault - ]; - } - '' - mkdir -p $out/bin - cat > $out/bin/cleanup-ami << 'EOL' - #!/usr/bin/env bash - set -euo pipefail - - export PATH="${ - pkgs.lib.makeBinPath ( - with pkgs; - [ - awscli2 - aws-vault - ] - ) - }:$PATH" - - # Check for required tools - for cmd in aws-vault; do - if ! command -v $cmd &> /dev/null; then - echo "Error: $cmd is required but not found" - exit 1 - fi - done - +{ + writeShellApplication, + awscli2, + aws-vault, +}: +writeShellApplication { + name = "cleanup-ami"; + runtimeInputs = [ + awscli2 + aws-vault + ]; + text = '' # Check AWS Vault profile if [ -z "''${AWS_VAULT:-}" ]; then echo "Error: AWS_VAULT environment variable must be set with the profile name" @@ -56,6 +36,5 @@ runCommand "cleanup-ami" aws ec2 deregister-image --region $REGION --image-id "$ami_id" || true done done - EOL - chmod +x $out/bin/cleanup-ami - '' + ''; +} diff --git a/nix/packages/dbmate-tool.nix b/nix/packages/dbmate-tool.nix index 5eed71324..1d450f26c 100644 --- a/nix/packages/dbmate-tool.nix +++ b/nix/packages/dbmate-tool.nix @@ -1,6 +1,12 @@ { - pkgs, stdenv, + writeShellApplication, + overmind, + dbmate, + nix, + jq, + yq, + system, defaults, }: let @@ -9,37 +15,316 @@ let pgbouncerAuthSchemaSql = ../../ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql; statExtensionSql = ../../ansible/files/stat_extension.sql; in -pkgs.runCommand "dbmate-tool" - { - buildInputs = with pkgs; [ - overmind - dbmate - nix - jq - yq - ]; - nativeBuildInputs = with pkgs; [ makeWrapper ]; - } - '' - mkdir -p $out/bin $out/migrations - cp -r ${migrationsDir}/* $out - substitute ${../tools/dbmate-tool.sh.in} $out/bin/dbmate-tool \ - --subst-var-by 'PGSQL_DEFAULT_PORT' '${defaults.port}' \ - --subst-var-by 'MIGRATIONS_DIR' $out \ - --subst-var-by 'PGSQL_SUPERUSER' '${defaults.superuser}' \ - --subst-var-by 'ANSIBLE_VARS' ${ansibleVars} \ - --subst-var-by 'CURRENT_SYSTEM' '${stdenv.hostPlatform.system}' \ - --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ - --subst-var-by 'STAT_EXTENSION_SQL' '${statExtensionSql}' - chmod +x $out/bin/dbmate-tool - wrapProgram $out/bin/dbmate-tool \ - --prefix PATH : ${ - pkgs.lib.makeBinPath [ - pkgs.overmind - pkgs.dbmate - pkgs.nix - pkgs.jq - pkgs.yq - ] - } - '' +writeShellApplication { + name = "dbmate-tool"; + runtimeInputs = [ + overmind + dbmate + nix + jq + yq + ]; + text = '' + # Default values + PSQL_VERSION="ALL" + PORTNO="${defaults.port}" + PGSQL_SUPERUSER="${defaults.superuser}" + PGPASSWORD="''${PGPASSWORD:-postgres}" + PGSQL_USER="postgres" + FLAKE_URL="github:supabase/postgres" + MIGRATIONS_DIR="${migrationsDir}" + CURRENT_SYSTEM="${system}" + ANSIBLE_VARS="${ansibleVars}" + PGBOUNCER_AUTH_SCHEMA_SQL="${pgbouncerAuthSchemaSql}" + STAT_EXTENSION_SQL="${statExtensionSql}" + + # Start PostgreSQL using nix + start_postgres() { + DATDIR=$(mktemp -d) + echo "Starting PostgreSQL in directory: $DATDIR" # Create the DATDIR if it doesn't exist + nix run "$FLAKE_URL#start-server" -- "$PSQL_VERSION" --skip-migrations --daemonize --datdir "$DATDIR" + echo "PostgreSQL started." + } + + # Cleanup function + cleanup() { + echo "Cleaning up..." + + # Check if PostgreSQL processes exist + if pgrep -f "postgres" >/dev/null; then + echo "Stopping PostgreSQL gracefully..." + + # Use pg_ctl to stop PostgreSQL + pg_ctl -D "$DATDIR" stop + + # Wait a bit for graceful shutdown + sleep 5 + + # Check if processes are still running + if pgrep -f "postgres" >/dev/null; then + echo "Warning: Some PostgreSQL processes could not be stopped gracefully." + fi + else + echo "PostgreSQL is not running, skipping stop." + fi + + # Always exit successfully, log any remaining processes + if pgrep -f "postgres" >/dev/null; then + echo "Warning: Some PostgreSQL processes could not be cleaned up:" + pgrep -f "postgres" + else + echo "Cleanup completed successfully" + fi + } + + + # Function to display help + print_help() { + echo "Usage: nix run .#dbmate-tool -- [options]" + echo + echo "Options:" + echo " -v, --version [15|17|orioledb-17|all] Specify the PostgreSQL version to use (required defaults to --version all)" + echo " -p, --port PORT Specify the port number to use (default: 5435)" + echo " -h, --help Show this help message" + echo " -f, --flake-url URL Specify the flake URL to use (default: github:supabase/postgres)" + echo "Description:" + echo " Runs 'dbmate up' against a locally running the version of database you specify. Or 'all' to run against all versions." + echo " NOTE: To create a migration, you must run 'nix develop' and then 'dbmate new ' to create a new migration file." + echo + echo "Examples:" + echo " nix run .#dbmate-tool" + echo " nix run .#dbmate-tool -- --version 15" + echo " nix run .#dbmate-tool -- --version 16 --port 5433" + echo " nix run .#dbmate-tool -- --version 16 --port 5433 --flake-url github:supabase/postgres/" + } + + # Parse arguments + while [[ "$#" -gt 0 ]]; do + case "$1" in + -v|--version) + if [[ -n "$2" && ! "$2" =~ ^- ]]; then + PSQL_VERSION="$2" + shift 2 + else + echo "Error: --version requires an argument (15, 16, or orioledb-17)" + exit 1 + fi + ;; + -u|--user) + if [[ -n "$2" && ! "$2" =~ ^- ]]; then + PGSQL_USER="$2" + shift 2 + else + echo "Error: --user requires an argument" + exit 1 + fi + ;; + -f|--flake-url) + if [[ -n "$2" && ! "$2" =~ ^- ]]; then + FLAKE_URL="$2" + shift 2 + else + echo "Error: --flake-url requires an argument" + exit 1 + fi + ;; + -p|--port) + if [[ -n "$2" && ! "$2" =~ ^- ]]; then + PORTNO="$2" + shift 2 + else + echo "Error: --port requires an argument" + exit 1 + fi + ;; + -h|--help) + print_help + exit 0 + ;; + *) + echo "Unknown option: $1" + print_help + exit 1 + ;; + esac + done + + # Function to wait for PostgreSQL to be ready + wait_for_postgres() { + local max_attempts=30 # Increased significantly + local attempt=1 + + # Give PostgreSQL a moment to actually start the process + sleep 2 + + while [ $attempt -le $max_attempts ]; do + "''${PSQLBIN}/pg_isready" -h localhost -p "$PORTNO" -U "$PGSQL_SUPERUSER" -d postgres + local status=$? + + if [ $status -eq 0 ]; then + echo "PostgreSQL is ready!" + return 0 + fi + echo "Waiting for PostgreSQL to start (attempt $attempt/$max_attempts)..." + sleep 2 + attempt=$((attempt + 1)) + done + + echo "PostgreSQL failed to start after $max_attempts attempts" + return 1 + } + + check_orioledb_ready() { + local max_attempts=30 + local attempt=1 + + while [ $attempt -le $max_attempts ]; do + if "''${PSQLBIN}/psql" -v ON_ERROR_STOP=1 -U "$PGSQL_SUPERUSER" -p "$PORTNO" -h localhost -d postgres -c "SELECT * FROM pg_am WHERE amname = 'orioledb'" | grep -q orioledb; then + echo "Orioledb extension is ready!" + return 0 + fi + echo "Waiting for orioledb to be ready (attempt $attempt/$max_attempts)..." + sleep 2 + attempt=$((attempt + 1)) + done + + echo "Orioledb failed to initialize after $max_attempts attempts" + return 1 + } + + perform_dump() { + local max_attempts=3 + local attempt=1 + + while [ $attempt -le $max_attempts ]; do + echo "Attempting pg_dump (attempt $attempt/$max_attempts)" + + # Build the dump command + local dump_cmd="''${PSQLBIN}/pg_dump -h localhost -p $PORTNO -U $PGSQL_SUPERUSER -d postgres --schema-only --no-owner --no-privileges" + + # Only use --restrict-key for standard PostgreSQL 15 and 17 versions + # OrioleDB doesn't support this flag yet + if [ "$PSQL_VERSION" = "15" ] || [ "$PSQL_VERSION" = "17" ] || [ "$PSQL_VERSION" = "orioledb-17" ]; then + # Use a fixed restrict key for reproducible test dumps + # This is safe in testing contexts but should not be used in production + dump_cmd="$dump_cmd --restrict-key=SupabaseTestDumpKey123" + echo "Using --restrict-key for reproducible dumps (PostgreSQL $PSQL_VERSION)" + else + echo "Skipping --restrict-key (version: $PSQL_VERSION)" + fi + + if $dump_cmd > "./db/schema.sql"; then + return 0 + fi + + echo "Dump attempt $attempt failed, waiting before retry..." + sleep 5 + attempt=$((attempt + 1)) + done + + echo "All dump attempts failed" + return 1 + } + + migrate_version() { + echo "PSQL_VERSION: $PSQL_VERSION" + #pkill -f "postgres" || true # Ensure PostgreSQL is stopped before starting + PSQLBIN=$(nix build --no-link "$FLAKE_URL#psql_$PSQL_VERSION/bin" --json | jq -r '.[].outputs.out + "/bin"') + echo "Using PostgreSQL version $PSQL_VERSION from $PSQLBIN" + + # Start PostgreSQL + start_postgres + echo "Waiting for PostgreSQL to be ready..." + + # Wait for PostgreSQL to be ready to accept connections + if ! wait_for_postgres; then + echo "Failed to connect to PostgreSQL server" + cleanup + exit 1 + fi + + if [ "$PSQL_VERSION" = "orioledb-17" ]; then + if ! check_orioledb_ready; then + echo "Failed to initialize orioledb extension" + exit 1 + fi + fi + + echo "PostgreSQL server is ready" + + # Configure PostgreSQL roles and permissions + if ! "''${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U "$PGSQL_SUPERUSER" -p "$PORTNO" -h localhost -d postgres <<-EOSQL + create role postgres superuser login password '$PGPASSWORD'; + alter database postgres owner to postgres; + EOSQL + then + echo "Failed to configure PostgreSQL roles and permissions" + exit 1 + fi + "''${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -d postgres -f "$PGBOUNCER_AUTH_SCHEMA_SQL" + "''${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -d postgres -f "$STAT_EXTENSION_SQL" + + # Set db url to run dbmate + export DATABASE_URL="postgres://$PGSQL_USER:$PGPASSWORD@localhost:$PORTNO/postgres?sslmode=disable" + # Export path so dbmate can find correct psql and pg_dump + export PATH="$PSQLBIN:$PATH" + # Run init scripts + if ! dbmate --migrations-dir "$MIGRATIONS_DIR/init-scripts" up; then + echo "Error: Initial migration failed" + exit 1 + fi + + # Password update command + if ! "''${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -c "ALTER USER supabase_admin WITH PASSWORD '$PGPASSWORD'"; then + echo "Error: Failed to update supabase_admin password" + exit 1 + fi + + # Set up database URL + export DATABASE_URL="postgres://$PGSQL_SUPERUSER:$PGPASSWORD@localhost:$PORTNO/postgres?sslmode=disable" + # Run migrations + if ! dbmate --migrations-dir "$MIGRATIONS_DIR/migrations" up; then + echo "Error: Final migration failed" + exit 1 + fi + + echo "Running dbmate dump with $PSQLBIN" + perform_dump + + echo "CURRENT_SYSTEM: $CURRENT_SYSTEM" + if [ -f "./db/schema.sql" ]; then + cp "./db/schema.sql" "./migrations/schema-$PSQL_VERSION.sql" + echo "Schema file moved to ./migrations/schema-$PSQL_VERSION.sql" + echo "PSQLBIN is $PSQLBIN" + else + echo "Warning: schema.sql file not found in ./db directory" + exit 1 + fi + + # If we get here, all commands succeeded + echo "PostgreSQL migration completed successfully" + echo "Check migrations are idempotent" + for sql in ./migrations/db/migrations/*.sql; do + echo "$0: running $sql" + "''${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U "$PGSQL_SUPERUSER" -p "$PORTNO" -h localhost -d postgres -f "$sql" || { + echo "Failed to execute $sql" + exit 1 + } + done + } + + if [ "$PSQL_VERSION" == "all" ]; then + VERSIONS=$(yq '.postgres_major[]' "$ANSIBLE_VARS" | tr -d '"') + echo "$VERSIONS" | while read -r version; do + PSQL_VERSION="$version" + echo "Migrating to PostgreSQL version $PSQL_VERSION" + migrate_version + cleanup + done + else + echo "Migrating to PostgreSQL version $PSQL_VERSION" + migrate_version + cleanup + fi + ''; +} diff --git a/nix/packages/migrate-tool.nix b/nix/packages/migrate-tool.nix index 57f218a63..c98992110 100644 --- a/nix/packages/migrate-tool.nix +++ b/nix/packages/migrate-tool.nix @@ -1,18 +1,128 @@ -{ runCommand, psql_15 }: +{ writeShellApplication, psql_15 }: let configFile = ../tests/postgresql.conf.in; getkeyScript = ../tests/util/pgsodium_getkey.sh; primingScript = ../tests/prime.sql; migrationData = ../tests/migrations/data.sql; in -runCommand "migrate-postgres" { } '' - mkdir -p $out/bin - substitute ${../tools/migrate-tool.sh.in} $out/bin/migrate-postgres \ - --subst-var-by 'PSQL15_BINDIR' '${psql_15}' \ - --subst-var-by 'PSQL_CONF_FILE' '${configFile}' \ - --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ - --subst-var-by 'PRIMING_SCRIPT' '${primingScript}' \ - --subst-var-by 'MIGRATION_DATA' '${migrationData}' - - chmod +x $out/bin/migrate-postgres -'' +writeShellApplication { + name = "migrate-postgres"; + runtimeInputs = [ ]; + text = '' + # first argument is the old version; a path 15 or 16 + if [[ $1 == /nix/store* ]]; then + if [ ! -L "$1/receipt.json" ] || [ ! -e "$1/receipt.json" ]; then + echo "ERROR: $1 does not look like a valid Postgres install" + exit 1 + fi + OLDVER="$1" + elif [ "$1" == "15" ]; then + OLDVER="${psql_15}" + elif [ "$1" == "16" ]; then + echo "ERROR: PSQL 16 is not configured in this package" + exit 1 + else + echo "Please provide a valid Postgres version (15 or 16), or a /nix/store path" + exit 1 + fi + + # second argument is the new version; 15 or 16 + if [[ $2 == /nix/store* ]]; then + if [ ! -L "$2/receipt.json" ] || [ ! -e "$2/receipt.json" ]; then + echo "ERROR: $1 does not look like a valid Postgres install" + exit 1 + fi + NEWVER="$2" + elif [ "$2" == "15" ]; then + NEWVER="${psql_15}" + elif [ "$2" == "16" ]; then + echo "ERROR: PSQL 16 is not configured in this package" + exit 1 + else + echo "Please provide a valid Postgres version (15 or 16), or a /nix/store path" + exit 1 + fi + + # thid argument is the upgrade method: either pg_dumpall or pg_ugprade + if [ "$3" != "pg_dumpall" ] && [ "$3" != "pg_upgrade" ]; then + echo "Please provide a valid upgrade method (pg_dumpall or pg_upgrade)" + exit 1 + fi + UPGRADE_METHOD="$3" + + echo "Old server build: PSQL $1" + echo "New server build: PSQL $2" + echo "Upgrade method: $UPGRADE_METHOD" + + DATDIR=$(mktemp -d) + NEWDAT=$(mktemp -d) + mkdir -p "$DATDIR" "$NEWDAT" + + echo "NOTE: using temporary directory $DATDIR for PSQL $1 data, which will not be removed" + echo "NOTE: you are free to re-use this data directory at will" + echo + + "$OLDVER/bin/initdb" -D "$DATDIR" --locale=C --username=supabase_admin + "$NEWVER/bin/initdb" -D "$NEWDAT" --locale=C --username=supabase_admin + + # NOTE (aseipp): we need to patch postgresql.conf to have the right pgsodium_getkey script + PSQL_CONF_FILE="${configFile}" + PGSODIUM_GETKEY_SCRIPT="${getkeyScript}" + echo "NOTE: patching postgresql.conf files" + for x in "$DATDIR" "$NEWDAT"; do + sed \ + "s#@PGSODIUM_GETKEY_SCRIPT@#$PGSODIUM_GETKEY_SCRIPT#g" \ + "$PSQL_CONF_FILE" > "$x/postgresql.conf" + done + + echo "NOTE: Starting first server (v''${1}) to load data into the system" + "$OLDVER/bin/pg_ctl" start -D "$DATDIR" + + PRIMING_SCRIPT="${primingScript}" + MIGRATION_DATA="${migrationData}" + + "$OLDVER/bin/psql" -h localhost -d postgres -Xf "$PRIMING_SCRIPT" + "$OLDVER/bin/psql" -h localhost -d postgres -Xf "$MIGRATION_DATA" + + if [ "$UPGRADE_METHOD" == "pg_upgrade" ]; then + echo "NOTE: Stopping old server (v''${1}) to prepare for migration" + "$OLDVER/bin/pg_ctl" stop -D "$DATDIR" + + echo "NOTE: Migrating old data $DATDIR to $NEWDAT using pg_upgrade" + + export PGDATAOLD="$DATDIR" + export PGDATANEW="$NEWDAT" + export PGBINOLD="$OLDVER/bin" + export PGBINNEW="$NEWVER/bin" + + if ! "$NEWVER/bin/pg_upgrade" --check; then + echo "ERROR: pg_upgrade check failed" + exit 1 + fi + + echo "NOTE: pg_upgrade check passed, proceeding with migration" + "$NEWVER/bin/pg_upgrade" + rm -f delete_old_cluster.sh # we don't need this + exit 0 + fi + + if [ "$UPGRADE_METHOD" == "pg_dumpall" ]; then + SQLDAT="$DATDIR/dump.sql" + echo "NOTE: Exporting data via pg_dumpall ($SQLDAT)" + "$NEWVER/bin/pg_dumpall" -h localhost > "$SQLDAT" + + echo "NOTE: Stopping old server (v''${1}) to prepare for migration" + "$OLDVER/bin/pg_ctl" stop -D "$DATDIR" + + echo "NOTE: Starting second server (v''${2}) to load data into the system" + "$NEWVER/bin/pg_ctl" start -D "$NEWDAT" + + echo "NOTE: Loading data into new server (v''${2}) via 'cat | psql'" + cat "$SQLDAT" | "$NEWVER/bin/psql" -h localhost -d postgres + + printf "\n\n\n\n" + echo "NOTE: Done, check logs. Stopping the server; new database is located at $NEWDAT" + "$NEWVER/bin/pg_ctl" stop -D "$NEWDAT" + fi + ''; +} diff --git a/nix/packages/start-client.nix b/nix/packages/start-client.nix index 84ab06209..a8f4200d9 100644 --- a/nix/packages/start-client.nix +++ b/nix/packages/start-client.nix @@ -1,27 +1,103 @@ { - runCommand, + writeShellApplication, psql_15, psql_17, psql_orioledb-17, defaults, }: -let - migrationsDir = ../../migrations/db; - postgresqlSchemaSql = ../tools/postgresql_schema.sql; - pgbouncerAuthSchemaSql = ../../ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql; - statExtensionSql = ../../ansible/files/stat_extension.sql; -in -runCommand "start-postgres-client" { } '' - mkdir -p $out/bin - substitute ${../tools/run-client.sh.in} $out/bin/start-postgres-client \ - --subst-var-by 'PGSQL_DEFAULT_PORT' '${defaults.port}' \ - --subst-var-by 'PGSQL_SUPERUSER' '${defaults.superuser}' \ - --subst-var-by 'PSQL15_BINDIR' '${psql_15}' \ - --subst-var-by 'PSQL17_BINDIR' '${psql_17}' \ - --subst-var-by 'PSQLORIOLEDB17_BINDIR' '${psql_orioledb-17}' \ - --subst-var-by 'MIGRATIONS_DIR' '${migrationsDir}' \ - --subst-var-by 'POSTGRESQL_SCHEMA_SQL' '${postgresqlSchemaSql}' \ - --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ - --subst-var-by 'STAT_EXTENSION_SQL' '${statExtensionSql}' - chmod +x $out/bin/start-postgres-client -'' +writeShellApplication { + name = "start-postgres-client"; + runtimeInputs = [ ]; + text = '' + # Default values + PSQL_VERSION="15" + PORTNO="${defaults.port}" + PSQL_USER="postgres" + + # Function to display help + print_help() { + echo "Usage: nix run .#start-client -- [options]" + echo + echo "Options:" + echo " -v, --version [15|17|orioledb-17] Specify the PostgreSQL version to use (default: 15)" + echo " -u, --user USER Specify the user/role to use (default: postgres)" + echo " -p, --port PORT Specify the port (default: ${defaults.port})" + echo " -h, --help Show this help message" + echo + echo "Description:" + echo " Starts an interactive 'psql' session connecting to a Postgres database started with the" + echo " 'nix run .#start-server' command." + echo + echo "Examples:" + echo " nix run .#start-client" + echo " nix run .#start-client -- --version 15" + echo " nix run .#start-client -- --version 17 --port 5433" + echo " nix run .#start-client -- --version 17 --user supabase_admin" + } + + # Parse arguments + while [[ "$#" -gt 0 ]]; do + case "$1" in + -v|--version) + if [[ -n "$2" && ! "$2" =~ ^- ]]; then + PSQL_VERSION="$2" + shift 2 + else + echo "Error: --version requires an argument (15, 17, or orioledb-17)" + exit 1 + fi + ;; + -u|--user) + if [[ -n "$2" && ! "$2" =~ ^- ]]; then + PSQL_USER="$2" + shift 2 + else + echo "Error: --user requires an argument" + exit 1 + fi + ;; + -p|--port) + if [[ -n "$2" && ! "$2" =~ ^- ]]; then + PORTNO="$2" + shift 2 + else + echo "Error: --port requires an argument" + exit 1 + fi + ;; + -h|--help) + print_help + exit 0 + ;; + *) + echo "Unknown option: $1" + print_help + exit 1 + ;; + esac + done + + # Determine PostgreSQL version + if [ "$PSQL_VERSION" == "15" ]; then + echo "Starting client for PSQL 15" + BINDIR="${psql_15}" + elif [ "$PSQL_VERSION" == "17" ]; then + echo "Starting client for PSQL 17" + BINDIR="${psql_17}" + elif [ "$PSQL_VERSION" == "orioledb-17" ]; then + echo "Starting client for PSQL ORIOLEDB 17" + BINDIR="${psql_orioledb-17}" + else + echo "Please provide a valid Postgres version (15, 17, or orioledb-17)" + exit 1 + fi + + # Set up environment for psql + export PATH="$BINDIR/bin:$PATH" + export POSTGRES_DB=postgres + export POSTGRES_HOST=localhost + + # Start interactive psql session + exec psql -U "$PSQL_USER" -p "$PORTNO" -h localhost postgres + ''; +} diff --git a/nix/packages/start-replica.nix b/nix/packages/start-replica.nix index 129ccd9cc..bd63764fa 100644 --- a/nix/packages/start-replica.nix +++ b/nix/packages/start-replica.nix @@ -1,12 +1,48 @@ { - runCommand, + writeShellApplication, pgsqlSuperuser, psql_15, }: -runCommand "start-postgres-replica" { } '' - mkdir -p $out/bin - substitute ${./start-replica.sh.in} $out/bin/start-postgres-replica \ - --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ - --subst-var-by 'PSQL15_BINDIR' '${psql_15}' - chmod +x $out/bin/start-postgres-replica -'' +writeShellApplication { + name = "start-postgres-replica"; + runtimeInputs = [ ]; + text = '' + # first argument should be '15' or '16' for the version + if [ "$1" == "15" ]; then + echo "Starting server for PSQL 15" + BINDIR="${psql_15}" + elif [ "$1" == "16" ]; then + echo "Starting server for PSQL 16" + echo "Error: PSQL 16 is not configured in this package" + exit 1 + elif [ "$1" == "orioledb-16" ]; then + echo "Starting server for PSQL ORIOLEDB 16" + echo "Error: PSQL ORIOLEDB 16 is not configured in this package" + exit 1 + else + echo "Please provide a valid Postgres version (15, 16 or orioledb-16)" + exit 1 + fi + + export PATH="$BINDIR/bin:$PATH" + + PGSQL_SUPERUSER="${pgsqlSuperuser}" + MASTER_PORTNO="$2" + REPLICA_PORTNO="$3" + REPLICA_SLOT="replica_$RANDOM" + DATDIR=$(mktemp -d) + mkdir -p "$DATDIR" + + echo "NOTE: runing pg_basebackup for server on port $MASTER_PORTNO" + echo "NOTE: using replica slot $REPLICA_SLOT" + + pg_basebackup -p "$MASTER_PORTNO" -h localhost -U "''${PGSQL_SUPERUSER}" -X stream -C -S "$REPLICA_SLOT" -v -R -D "$DATDIR" + + echo "NOTE: using port $REPLICA_PORTNO for replica" + echo "NOTE: using temporary directory $DATDIR for data, which will not be removed" + echo "NOTE: you are free to re-use this data directory at will" + echo + + exec postgres -p "$REPLICA_PORTNO" -D "$DATDIR" -k /tmp + ''; +} diff --git a/nix/packages/start-replica.sh.in b/nix/packages/start-replica.sh.in deleted file mode 100644 index e2096b17a..000000000 --- a/nix/packages/start-replica.sh.in +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# shellcheck shell=bash - -[ ! -z "$DEBUG" ] && set -x - -# first argument should be '15' or '16' for the version -if [ "$1" == "15" ]; then - echo "Starting server for PSQL 15" - PSQL15=@PSQL15_BINDIR@ - BINDIR="$PSQL15" -elif [ "$1" == "16" ]; then - echo "Starting server for PSQL 16" - PSQL16=@PSQL16_BINDIR@ - BINDIR="$PSQL16" -elif [ "$1" == "orioledb-16" ]; then - echo "Starting server for PSQL ORIOLEDB 16" - PSQLORIOLEDB16=@PSQLORIOLEDB16_BINDIR@ - BINDIR="$PSQLORIOLEDB16" -else - echo "Please provide a valid Postgres version (15, 16 or orioledb-16)" - exit 1 -fi - -export PATH=$BINDIR/bin:$PATH - -PGSQL_SUPERUSER=@PGSQL_SUPERUSER@ -MASTER_PORTNO="$2" -REPLICA_PORTNO="$3" -REPLICA_SLOT="replica_$RANDOM" -DATDIR=$(mktemp -d) -mkdir -p "$DATDIR" - -echo "NOTE: runing pg_basebackup for server on port $MASTER_PORTNO" -echo "NOTE: using replica slot $REPLICA_SLOT" - -pg_basebackup -p "$MASTER_PORTNO" -h localhost -U "${PGSQL_SUPERUSER}" -X stream -C -S "$REPLICA_SLOT" -v -R -D "$DATDIR" - -echo "NOTE: using port $REPLICA_PORTNO for replica" -echo "NOTE: using temporary directory $DATDIR for data, which will not be removed" -echo "NOTE: you are free to re-use this data directory at will" -echo - -exec postgres -p "$REPLICA_PORTNO" -D "$DATDIR" -k /tmp diff --git a/nix/packages/sync-exts-versions.nix b/nix/packages/sync-exts-versions.nix index cd043f840..c660a66c6 100644 --- a/nix/packages/sync-exts-versions.nix +++ b/nix/packages/sync-exts-versions.nix @@ -1,17 +1,17 @@ { - runCommand, + writeShellApplication, jq, yq, nix-editor, - nixVersions, + nix, }: -runCommand "sync-exts-versions" { } '' - mkdir -p $out/bin - substitute ${../tools/sync-exts-versions.sh.in} $out/bin/sync-exts-versions \ - --subst-var-by 'YQ' '${yq}/bin/yq' \ - --subst-var-by 'JQ' '${jq}/bin/jq' \ - --subst-var-by 'NIX_EDITOR' '${nix-editor.packages.nix-editor}/bin/nix-editor' \ - --subst-var-by 'NIXPREFETCHURL' '${nixVersions.nix_2_31}/bin/nix-prefetch-url' \ - --subst-var-by 'NIX' '${nixVersions.nix_2_31}/bin/nix' - chmod +x $out/bin/sync-exts-versions -'' +writeShellApplication { + name = "sync-exts-versions"; + runtimeInputs = [ + jq + yq + nix-editor.packages.nix-editor + nix + ]; + text = builtins.readFile ../tools/sync-exts-versions.sh.in; +} diff --git a/nix/tools/dbmate-tool.sh.in b/nix/tools/dbmate-tool.sh.in deleted file mode 100644 index ec38ecf1a..000000000 --- a/nix/tools/dbmate-tool.sh.in +++ /dev/null @@ -1,306 +0,0 @@ -#!/usr/bin/env bash -# shellcheck shell=bash - -[ ! -z "$DEBUG" ] && set -x - -# Default values -PSQL_VERSION="ALL" -PORTNO="@PGSQL_DEFAULT_PORT@" -PGSQL_SUPERUSER="@PGSQL_SUPERUSER@" -PGPASSWORD="${PGPASSWORD:-postgres}" -PGSQL_USER="postgres" -FLAKE_URL="github:supabase/postgres" -MIGRATIONS_DIR="@MIGRATIONS_DIR@" -CURRENT_SYSTEM="@CURRENT_SYSTEM@" -ANSIBLE_VARS="@ANSIBLE_VARS@" -PGBOUNCER_AUTH_SCHEMA_SQL=@PGBOUNCER_AUTH_SCHEMA_SQL@ -STAT_EXTENSION_SQL=@STAT_EXTENSION_SQL@ - -# Start PostgreSQL using nix -start_postgres() { - DATDIR=$(mktemp -d) - echo "Starting PostgreSQL in directory: $DATDIR" # Create the DATDIR if it doesn't exist - nix run "$FLAKE_URL#start-server" -- "$PSQL_VERSION" --skip-migrations --daemonize --datdir "$DATDIR" - echo "PostgreSQL started." -} - -# Cleanup function -cleanup() { - echo "Cleaning up..." - - # Check if PostgreSQL processes exist - if pgrep -f "postgres" >/dev/null; then - echo "Stopping PostgreSQL gracefully..." - - # Use pg_ctl to stop PostgreSQL - pg_ctl -D "$DATDIR" stop - - # Wait a bit for graceful shutdown - sleep 5 - - # Check if processes are still running - if pgrep -f "postgres" >/dev/null; then - echo "Warning: Some PostgreSQL processes could not be stopped gracefully." - fi - else - echo "PostgreSQL is not running, skipping stop." - fi - - # Always exit successfully, log any remaining processes - if pgrep -f "postgres" >/dev/null; then - echo "Warning: Some PostgreSQL processes could not be cleaned up:" - pgrep -f "postgres" - else - echo "Cleanup completed successfully" - fi -} - - -# Function to display help -print_help() { - echo "Usage: nix run .#dbmate-tool -- [options]" - echo - echo "Options:" - echo " -v, --version [15|17|orioledb-17|all] Specify the PostgreSQL version to use (required defaults to --version all)" - echo " -p, --port PORT Specify the port number to use (default: 5435)" - echo " -h, --help Show this help message" - echo " -f, --flake-url URL Specify the flake URL to use (default: github:supabase/postgres)" - echo "Description:" - echo " Runs 'dbmate up' against a locally running the version of database you specify. Or 'all' to run against all versions." - echo " NOTE: To create a migration, you must run 'nix develop' and then 'dbmate new ' to create a new migration file." - echo - echo "Examples:" - echo " nix run .#dbmate-tool" - echo " nix run .#dbmate-tool -- --version 15" - echo " nix run .#dbmate-tool -- --version 16 --port 5433" - echo " nix run .#dbmate-tool -- --version 16 --port 5433 --flake-url github:supabase/postgres/" -} - -# Parse arguments -while [[ "$#" -gt 0 ]]; do - case "$1" in - -v|--version) - if [[ -n "$2" && ! "$2" =~ ^- ]]; then - PSQL_VERSION="$2" - shift 2 - else - echo "Error: --version requires an argument (15, 16, or orioledb-17)" - exit 1 - fi - ;; - -u|--user) - if [[ -n "$2" && ! "$2" =~ ^- ]]; then - PGSQL_USER="$2" - shift 2 - else - echo "Error: --user requires an argument" - exit 1 - fi - ;; - -f|--flake-url) - if [[ -n "$2" && ! "$2" =~ ^- ]]; then - FLAKE_URL="$2" - shift 2 - else - echo "Error: --flake-url requires an argument" - exit 1 - fi - ;; - -p|--port) - if [[ -n "$2" && ! "$2" =~ ^- ]]; then - PORTNO="$2" - shift 2 - else - echo "Error: --port requires an argument" - exit 1 - fi - ;; - -h|--help) - print_help - exit 0 - ;; - *) - echo "Unknown option: $1" - print_help - exit 1 - ;; - esac -done - -# Function to wait for PostgreSQL to be ready -wait_for_postgres() { - local max_attempts=30 # Increased significantly - local attempt=1 - - # Give PostgreSQL a moment to actually start the process - sleep 2 - - while [ $attempt -le $max_attempts ]; do - "${PSQLBIN}/pg_isready" -h localhost -p "$PORTNO" -U "$PGSQL_SUPERUSER" -d postgres - local status=$? - - if [ $status -eq 0 ]; then - echo "PostgreSQL is ready!" - return 0 - fi - echo "Waiting for PostgreSQL to start (attempt $attempt/$max_attempts)..." - sleep 2 - attempt=$((attempt + 1)) - done - - echo "PostgreSQL failed to start after $max_attempts attempts" - return 1 -} - -check_orioledb_ready() { - local max_attempts=30 - local attempt=1 - - while [ $attempt -le $max_attempts ]; do - if "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 -U "$PGSQL_SUPERUSER" -p "$PORTNO" -h localhost -d postgres -c "SELECT * FROM pg_am WHERE amname = 'orioledb'" | grep -q orioledb; then - echo "Orioledb extension is ready!" - return 0 - fi - echo "Waiting for orioledb to be ready (attempt $attempt/$max_attempts)..." - sleep 2 - attempt=$((attempt + 1)) - done - - echo "Orioledb failed to initialize after $max_attempts attempts" - return 1 -} - -perform_dump() { - local max_attempts=3 - local attempt=1 - - while [ $attempt -le $max_attempts ]; do - echo "Attempting pg_dump (attempt $attempt/$max_attempts)" - - # Build the dump command - local dump_cmd="${PSQLBIN}/pg_dump -h localhost -p $PORTNO -U $PGSQL_SUPERUSER -d postgres --schema-only --no-owner --no-privileges" - - # Only use --restrict-key for standard PostgreSQL 15 and 17 versions - # OrioleDB doesn't support this flag yet - if [ "$PSQL_VERSION" = "15" ] || [ "$PSQL_VERSION" = "17" ] || [ "$PSQL_VERSION" = "orioledb-17" ]; then - # Use a fixed restrict key for reproducible test dumps - # This is safe in testing contexts but should not be used in production - dump_cmd="$dump_cmd --restrict-key=SupabaseTestDumpKey123" - echo "Using --restrict-key for reproducible dumps (PostgreSQL $PSQL_VERSION)" - else - echo "Skipping --restrict-key (version: $PSQL_VERSION)" - fi - - if $dump_cmd > "./db/schema.sql"; then - return 0 - fi - - echo "Dump attempt $attempt failed, waiting before retry..." - sleep 5 - attempt=$((attempt + 1)) - done - - echo "All dump attempts failed" - return 1 -} - -migrate_version() { - echo "PSQL_VERSION: $PSQL_VERSION" - #pkill -f "postgres" || true # Ensure PostgreSQL is stopped before starting - PSQLBIN=$(nix build --no-link "$FLAKE_URL#psql_$PSQL_VERSION/bin" --json | jq -r '.[].outputs.out + "/bin"') - echo "Using PostgreSQL version $PSQL_VERSION from $PSQLBIN" - - # Start PostgreSQL - start_postgres - echo "Waiting for PostgreSQL to be ready..." - - # Wait for PostgreSQL to be ready to accept connections - if ! wait_for_postgres; then - echo "Failed to connect to PostgreSQL server" - cleanup - exit 1 - fi - - if [ "$PSQL_VERSION" = "orioledb-17" ]; then - if ! check_orioledb_ready; then - echo "Failed to initialize orioledb extension" - exit 1 - fi - fi - - echo "PostgreSQL server is ready" - - # Configure PostgreSQL roles and permissions - if ! "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U "$PGSQL_SUPERUSER" -p "$PORTNO" -h localhost -d postgres <<-EOSQL -create role postgres superuser login password '$PGPASSWORD'; -alter database postgres owner to postgres; -EOSQL - then - echo "Failed to configure PostgreSQL roles and permissions" - exit 1 - fi - "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -d postgres -f "$PGBOUNCER_AUTH_SCHEMA_SQL" - "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -d postgres -f "$STAT_EXTENSION_SQL" - - # Set db url to run dbmate - export DATABASE_URL="postgres://$PGSQL_USER:$PGPASSWORD@localhost:$PORTNO/postgres?sslmode=disable" - # Export path so dbmate can find correct psql and pg_dump - export PATH="$PSQLBIN:$PATH" - # Run init scripts - if ! dbmate --migrations-dir "$MIGRATIONS_DIR/init-scripts" up; then - echo "Error: Initial migration failed" - exit 1 - fi - - # Password update command - if ! "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -c "ALTER USER supabase_admin WITH PASSWORD '$PGPASSWORD'"; then - echo "Error: Failed to update supabase_admin password" - exit 1 - fi - - # Set up database URL - export DATABASE_URL="postgres://$PGSQL_SUPERUSER:$PGPASSWORD@localhost:$PORTNO/postgres?sslmode=disable" - # Run migrations - if ! dbmate --migrations-dir "$MIGRATIONS_DIR/migrations" up; then - echo "Error: Final migration failed" - exit 1 - fi - - echo "Running dbmate dump with $PSQLBIN" - perform_dump - - echo "CURRENT_SYSTEM: $CURRENT_SYSTEM" - if [ -f "./db/schema.sql" ]; then - cp "./db/schema.sql" "./migrations/schema-$PSQL_VERSION.sql" - echo "Schema file moved to ./migrations/schema-$PSQL_VERSION.sql" - echo "PSQLBIN is $PSQLBIN" - else - echo "Warning: schema.sql file not found in ./db directory" - exit 1 - fi - - # If we get here, all commands succeeded - echo "PostgreSQL migration completed successfully" - echo "Check migrations are idempotent" - for sql in ./migrations/db/migrations/*.sql; do - echo "$0: running $sql" - "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U "$PGSQL_SUPERUSER" -p "$PORTNO" -h localhost -d postgres -f "$sql" || { - echo "Failed to execute $sql" - exit 1 - } - done -} - -if [ "$PSQL_VERSION" == "all" ]; then - VERSIONS=$(yq '.postgres_major[]' "$ANSIBLE_VARS" | tr -d '"') - echo "$VERSIONS" | while read -r version; do - PSQL_VERSION="$version" - echo "Migrating to PostgreSQL version $PSQL_VERSION" - migrate_version - cleanup - done -else - echo "Migrating to PostgreSQL version $PSQL_VERSION" - migrate_version - cleanup -fi diff --git a/nix/tools/migrate-tool.sh.in b/nix/tools/migrate-tool.sh.in deleted file mode 100644 index 10239ed80..000000000 --- a/nix/tools/migrate-tool.sh.in +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash - -[ ! -z "$DEBUG" ] && set -x - -# first argument is the old version; a path 15 or 16 -if [[ $1 == /nix/store* ]]; then - if [ ! -L "$1/receipt.json" ] || [ ! -e "$1/receipt.json" ]; then - echo "ERROR: $1 does not look like a valid Postgres install" - exit 1 - fi - OLDVER="$1" -elif [ "$1" == "15" ]; then - PSQL15=@PSQL15_BINDIR@ - OLDVER="$PSQL15" -elif [ "$1" == "16" ]; then - PSQL16=@PSQL16_BINDIR@ - OLDVER="$PSQL16" -else - echo "Please provide a valid Postgres version (15 or 16), or a /nix/store path" - exit 1 -fi - -# second argument is the new version; 15 or 16 -if [[ $2 == /nix/store* ]]; then - if [ ! -L "$2/receipt.json" ] || [ ! -e "$2/receipt.json" ]; then - echo "ERROR: $1 does not look like a valid Postgres install" - exit 1 - fi - NEWVER="$2" -elif [ "$2" == "15" ]; then - PSQL15=@PSQL15_BINDIR@ - NEWVER="$PSQL15" -elif [ "$2" == "16" ]; then - PSQL16=@PSQL16_BINDIR@ - NEWVER="$PSQL16" - echo "NEWVER IS $NEWVER" -else - echo "Please provide a valid Postgres version (15 or 16), or a /nix/store path" - exit 1 -fi - -# thid argument is the upgrade method: either pg_dumpall or pg_ugprade -if [ "$3" != "pg_dumpall" ] && [ "$3" != "pg_upgrade" ]; then - echo "Please provide a valid upgrade method (pg_dumpall or pg_upgrade)" - exit 1 -fi -UPGRADE_METHOD="$3" - -echo "Old server build: PSQL $1" -echo "New server build: PSQL $2" -echo "Upgrade method: $UPGRADE_METHOD" - -PORTNO="${2:-@PGSQL_DEFAULT_PORT@}" -DATDIR=$(mktemp -d) -NEWDAT=$(mktemp -d) -mkdir -p "$DATDIR" "$NEWDAT" - -echo "NOTE: using temporary directory $DATDIR for PSQL $1 data, which will not be removed" -echo "NOTE: you are free to re-use this data directory at will" -echo - -$OLDVER/bin/initdb -D "$DATDIR" --locale=C --username=supabase_admin -$NEWVER/bin/initdb -D "$NEWDAT" --locale=C --username=supabase_admin - -# NOTE (aseipp): we need to patch postgresql.conf to have the right pgsodium_getkey script -PSQL_CONF_FILE=@PSQL_CONF_FILE@ -PGSODIUM_GETKEY_SCRIPT=@PGSODIUM_GETKEY@ -echo "NOTE: patching postgresql.conf files" -for x in "$DATDIR" "$NEWDAT"; do - sed \ - "s#@PGSODIUM_GETKEY_SCRIPT@#$PGSODIUM_GETKEY_SCRIPT#g" \ - $PSQL_CONF_FILE > "$x/postgresql.conf" -done - -echo "NOTE: Starting first server (v${1}) to load data into the system" -$OLDVER/bin/pg_ctl start -D "$DATDIR" - -PRIMING_SCRIPT=@PRIMING_SCRIPT@ -MIGRATION_DATA=@MIGRATION_DATA@ - -$OLDVER/bin/psql -h localhost -d postgres -Xf "$PRIMING_SCRIPT" -$OLDVER/bin/psql -h localhost -d postgres -Xf "$MIGRATION_DATA" - -if [ "$UPGRADE_METHOD" == "pg_upgrade" ]; then - echo "NOTE: Stopping old server (v${1}) to prepare for migration" - $OLDVER/bin/pg_ctl stop -D "$DATDIR" - - echo "NOTE: Migrating old data $DATDIR to $NEWDAT using pg_upgrade" - - export PGDATAOLD="$DATDIR" - export PGDATANEW="$NEWDAT" - export PGBINOLD="$OLDVER/bin" - export PGBINNEW="$NEWVER/bin" - - if ! $NEWVER/bin/pg_upgrade --check; then - echo "ERROR: pg_upgrade check failed" - exit 1 - fi - - echo "NOTE: pg_upgrade check passed, proceeding with migration" - $NEWVER/bin/pg_upgrade - rm -f delete_old_cluster.sh # we don't need this - exit 0 -fi - -if [ "$UPGRADE_METHOD" == "pg_dumpall" ]; then - SQLDAT="$DATDIR/dump.sql" - echo "NOTE: Exporting data via pg_dumpall ($SQLDAT)" - $NEWVER/bin/pg_dumpall -h localhost > "$SQLDAT" - - echo "NOTE: Stopping old server (v${1}) to prepare for migration" - $OLDVER/bin/pg_ctl stop -D "$DATDIR" - - echo "NOTE: Starting second server (v${2}) to load data into the system" - $NEWVER/bin/pg_ctl start -D "$NEWDAT" - - echo "NOTE: Loading data into new server (v${2}) via 'cat | psql'" - cat "$SQLDAT" | $NEWVER/bin/psql -h localhost -d postgres - - printf "\n\n\n\n" - echo "NOTE: Done, check logs. Stopping the server; new database is located at $NEWDAT" - $NEWVER/bin/pg_ctl stop -D "$NEWDAT" -fi diff --git a/nix/tools/run-client.sh.in b/nix/tools/run-client.sh.in deleted file mode 100644 index 6acb4d6c0..000000000 --- a/nix/tools/run-client.sh.in +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env bash -# shellcheck shell=bash - -[ ! -z "$DEBUG" ] && set -x - -# Default values -PSQL_VERSION="15" -PORTNO="@PGSQL_DEFAULT_PORT@" -PSQL_USER="postgres" - -# Function to display help -print_help() { - echo "Usage: nix run .#start-client -- [options]" - echo - echo "Options:" - echo " -v, --version [15|16|orioledb-16] Specify the PostgreSQL version to use (required)" - echo " -u, --user USER Specify the user/role to use (default: postgres)" - echo " -h, --help Show this help message" - echo - echo "Description:" - echo " Starts an interactive 'psql' session connecting to a Postgres database started with the" - echo " 'nix run .#start-server' command. If a migration file is not provided, the client" - echo " initializes the database with the default migrations for a new Supabase project." - echo " If a migrations file is provided, default migrations are skipped" - echo " If no migration file is provided, it runs the default Supabase migrations." - echo - echo "Examples:" - echo " nix run .#start-client" - echo " nix run .#start-client -- --version 15" - echo " nix run .#start-client -- --version 16 --port 5433" - echo " nix run .#start-client -- --version 16 --user supabase_admin" -} - -# Parse arguments -while [[ "$#" -gt 0 ]]; do - case "$1" in - -v|--version) - if [[ -n "$2" && ! "$2" =~ ^- ]]; then - PSQL_VERSION="$2" - shift 2 - else - echo "Error: --version requires an argument (15, 16, or orioledb-16)" - exit 1 - fi - ;; - -u|--user) - if [[ -n "$2" && ! "$2" =~ ^- ]]; then - PSQL_USER="$2" - shift 2 - else - echo "Error: --user requires an argument" - exit 1 - fi - ;; - -p|--port) - if [[ -n "$2" && ! "$2" =~ ^- ]]; then - PORTNO="$2" - shift 2 - else - echo "Error: --port requires an argument" - exit 1 - fi - ;; - -h|--help) - print_help - exit 0 - ;; - *) - echo "Unknown option: $1" - print_help - exit 1 - ;; - esac -done - -# Check if version is provided -if [[ -z "$PSQL_VERSION" ]]; then - echo "Error: PostgreSQL version is required." - print_help - exit 1 -fi - -# Determine PostgreSQL version -if [ "$PSQL_VERSION" == "15" ]; then - echo "Starting client for PSQL 15" - PSQL15=@PSQL15_BINDIR@ - BINDIR="$PSQL15" -elif [ "$PSQL_VERSION" == "17" ]; then - echo "Starting client for PSQL 17" - PSQL17=@PSQL17_BINDIR@ - BINDIR="$PSQL17" -elif [ "$PSQL_VERSION" == "orioledb-17" ]; then - echo "Starting client for PSQL ORIOLEDB 17" - PSQLORIOLEDB17=@PSQLORIOLEDB17_BINDIR@ - BINDIR="$PSQLORIOLEDB17" -else - echo "Please provide a valid Postgres version (15, 17, or orioledb-17)" - exit 1 -fi - -#vars for migration.sh -export PATH=$BINDIR/bin:$PATH -export POSTGRES_DB=postgres -export POSTGRES_HOST=localhost - -# Start interactive psql session -exec psql -U "$PSQL_USER" -p "$PORTNO" -h localhost postgres diff --git a/nix/tools/sync-exts-versions.sh.in b/nix/tools/sync-exts-versions.sh.in index 1b120e988..8e588b002 100644 --- a/nix/tools/sync-exts-versions.sh.in +++ b/nix/tools/sync-exts-versions.sh.in @@ -1,15 +1,9 @@ #!/usr/bin/env bash # shellcheck shell=bash -[ ! -z "$DEBUG" ] && set -x +[ -n "$DEBUG" ] && set -x -#pass in env vars supplied by nix -yq=@YQ@ -jq=@JQ@ -editor=@NIX_EDITOR@ -ansible_vars=$($yq '.' $PWD/ansible/vars.yml) -prefetchurl=@NIXPREFETCHURL@ -_nix=@NIX@ +ansible_vars=$(yq '.' "$PWD/ansible/vars.yml") fetch_source_url() { local source_url=${1//\"/} # Remove double quotes source_url=${source_url//\'/} # Remove single quotes @@ -21,11 +15,13 @@ fetch_source_url() { fi echo "$source_url" - + # Run nix-prefetch-url command - local initial_hash=$($prefetchurl --type sha256 "$source_url" --unpack | cut -d ' ' -f 2) + local initial_hash + initial_hash=$(nix-prefetch-url --type sha256 "$source_url" --unpack | cut -d ' ' -f 2) #once we can bump up nix version, we can use nix hash convert --hash-algo sha256 - local final_hash=$($_nix hash to-sri --type sha256 $initial_hash) + local final_hash + final_hash=$(nix hash to-sri --type sha256 "$initial_hash") echo "$final_hash" } @@ -37,19 +33,19 @@ sync_version() { # Update the version and hash in the Nix expression - $editor $PWD/nix/ext/$package_name.nix version --inplace -v "$version" - $editor $PWD/nix/ext/$package_name.nix src.hash --inplace -v $hash + nix-editor "$PWD/nix/ext/$package_name.nix" version --inplace -v "$version" + nix-editor "$PWD/nix/ext/$package_name.nix" src.hash --inplace -v "$hash" } run_sync() { local varname=$1 local package_name=$2 - version=$(echo $ansible_vars | $jq -r '.'$varname'') + version=$(echo "$ansible_vars" | jq -r ".$varname") echo "$key: $version" - url=$($_nix eval .#psql_15/exts/$package_name.src.url) - hash=$(fetch_source_url $url | tail -n 1) - $(sync_version $package_name $version $hash) + url=$(nix eval ".#psql_15/exts/$package_name.src.url") + hash=$(fetch_source_url "$url" | tail -n 1) + sync_version "$package_name" "$version" "$hash" echo "synced $package_name to version $version with hash $hash" @@ -68,11 +64,13 @@ fetchurl_source_url() { fi echo "$source_url" - + # Run nix-prefetch-url command - local initial_hash=$($prefetchurl --type sha256 "$source_url" | cut -d ' ' -f 2) + local initial_hash + initial_hash=$(nix-prefetch-url --type sha256 "$source_url" | cut -d ' ' -f 2) #once we can bump up nix version, we can use nix hash convert --hash-algo sha256 - local final_hash=$($_nix hash to-sri --type sha256 $initial_hash) + local final_hash + final_hash=$(nix hash to-sri --type sha256 "$initial_hash") echo "$final_hash" } @@ -84,8 +82,8 @@ sync_version_fetchurl() { # Update the version and hash in the Nix expression - $editor $PWD/nix/ext/$package_name.nix version --inplace -v "$version" - $editor $PWD/nix/ext/$package_name.nix src.sha256 --inplace -v $hash + nix-editor "$PWD/nix/ext/$package_name.nix" version --inplace -v "$version" + nix-editor "$PWD/nix/ext/$package_name.nix" src.sha256 --inplace -v "$hash" } @@ -93,11 +91,11 @@ run_sync_fetchurl() { local varname=$1 local package_name=$2 - version=$(echo $ansible_vars | $jq -r '.'$varname'') + version=$(echo "$ansible_vars" | jq -r ".$varname") echo "$key: $version" - url=$($_nix eval .#psql_15/exts/$package_name.src.url) - hash=$(fetchurl_source_url $url | tail -n 1) - $(sync_version_fetchurl $package_name $version $hash) + url=$(nix eval ".#psql_15/exts/$package_name.src.url") + hash=$(fetchurl_source_url "$url" | tail -n 1) + sync_version_fetchurl "$package_name" "$version" "$hash" echo "synced $package_name to version $version with hash $hash" @@ -106,16 +104,13 @@ run_sync_fetchurl() { #for use on derivations that use cargoHash update_cargo_vendor_hash() { local package_name=$1 - $editor $PWD/nix/ext/$package_name.nix cargoHash --inplace -v "" - output=$($_nix build .#psql_15/exts/$package_name 2>&1) - - # Check if the command exited with an error - if [ $? -ne 0 ]; then + nix-editor "$PWD/nix/ext/$package_name.nix" cargoHash --inplace -v "" + if ! output=$(nix build .#psql_15/exts/"$package_name" 2>&1); then # Extract the hash value after "got: " hash_value_scraped=$(echo "$output" | grep "got:" | awk '{for (i=1; i<=NF; i++) if ($i ~ /^sha/) print $i}') hash_value="\"$hash_value_scraped\"" # Continue using the captured hash value - $editor $PWD/nix/ext/$package_name.nix cargoHash --inplace -v $hash_value + nix-editor "$PWD/nix/ext/$package_name.nix" cargoHash --inplace -v "$hash_value" echo "Updated cargoHash for $package_name to $hash_value" else echo "$package_name builds successfully, moving on..." @@ -124,146 +119,146 @@ update_cargo_vendor_hash() { #iterate values in ansible vars, case statement # to match ansible var to package name -keys=$(echo "$ansible_vars" | $jq -r 'keys[]') +keys=$(echo "$ansible_vars" | jq -r 'keys[]') for key in $keys; do case $key in "pg_hashids_release") varname="pg_hashids_release" package_name="pg_hashids" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "hypopg_release") varname="hypopg_release" package_name="hypopg" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "pg_graphql_release") varname="pg_graphql_release" package_name="pg_graphql" - run_sync $varname $package_name - update_cargo_vendor_hash $package_name + run_sync "$varname" "$package_name" + update_cargo_vendor_hash "$package_name" ;; "pg_cron_release") varname="pg_cron_release" package_name="pg_cron" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "pgsql_http_release") varname="pgsql_http_release" package_name="pgsql-http" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "pg_jsonschema_release") varname="pg_jsonschema_release" package_name="pg_jsonschema" - run_sync $varname $package_name - update_cargo_vendor_hash $package_name + run_sync "$varname" "$package_name" + update_cargo_vendor_hash "$package_name" ;; "pg_net_release") varname="pg_net_release" package_name="pg_net" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "pg_plan_filter_release") varname="pg_plan_filter_release" package_name="pg_plan_filter" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "pg_safeupdate_release") varname="pg_safeupdate_release" package_name="pg-safeupdate" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "pgsodium_release") varname="pgsodium_release" package_name="pgsodium" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "pg_repack_release") varname="pg_repack_release" package_name="pg_repack" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "pgrouting_release") varname="pgrouting_release" package_name="pgrouting" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "ptap_release") varname="pgtap_release" package_name="pgtap" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "pg_stat_monitor_release") varname="pg_stat_monitor_release" package_name="pg_stat_monitor" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "pg_tle_release") varname="pg_tle_release" package_name="pg_tle" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "pgaudit_release") varname="pgaudit_release" package_name="pgaudit" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "plpgsql_check_release") varname="plpgsql_check_release" package_name="plpgsql-check" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "pgvector_release") varname="pgvector_release" package_name="pgvector" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "pgjwt_release") varname="pgjwt_release" package_name="pgjwt" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "plv8_release") varname="plv8_release" package_name="plv8" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "postgis_release") varname="postgis_release" package_name="postgis" - run_sync_fetchurl $varname $package_name + run_sync_fetchurl "$varname" "$package_name" ;; "pgroonga_release") varname="pgroonga_release" package_name="pgroonga" - run_sync_fetchurl $varname $package_name + run_sync_fetchurl "$varname" "$package_name" ;; "rum_release") varname="rum_release" package_name="rum" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "timescaledb_release") varname="timescaledb_release" package_name="timescaledb" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "supautils_release") varname="supautils_release" package_name="supautils" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "vault_release") varname="vault_release" package_name="vault" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; "wal2json_release") varname="wal2json_release" package_name="wal2json" - run_sync $varname $package_name + run_sync "$varname" "$package_name" ;; *) ;; From efc2e26342e9bb59d9d2a6d1d14c0e47cd795428 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 11 Dec 2025 22:17:58 +0100 Subject: [PATCH 3/3] chore: use packer from nixpkgs We don't need to maintain our own packer package anymore since we are using a recently updated nixpkgs that includes packer. --- nix/checks.nix | 2 -- nix/packages/dbmate-tool.nix | 1 - nix/packages/default.nix | 3 +-- nix/packages/packer.nix | 43 ------------------------------------ 4 files changed, 1 insertion(+), 48 deletions(-) delete mode 100644 nix/packages/packer.nix diff --git a/nix/checks.nix b/nix/checks.nix index 31c7bfaba..48915e8a8 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -390,8 +390,6 @@ ); inherit (self'.packages) wal-g-2 - dbmate-tool - packer pg_regress goss supascan diff --git a/nix/packages/dbmate-tool.nix b/nix/packages/dbmate-tool.nix index 1d450f26c..a4c71f034 100644 --- a/nix/packages/dbmate-tool.nix +++ b/nix/packages/dbmate-tool.nix @@ -1,5 +1,4 @@ { - stdenv, writeShellApplication, overmind, dbmate, diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 18bb7c5d0..7d5020f1e 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -33,7 +33,7 @@ { packages = ( { - build-ami = pkgs.callPackage ./build-ami.nix { packer = self'.packages.packer; }; + build-ami = pkgs.callPackage ./build-ami.nix { }; build-test-ami = pkgs.callPackage ./build-test-ami.nix { }; cleanup-ami = pkgs.callPackage ./cleanup-ami.nix { }; dbmate-tool = pkgs.callPackage ./dbmate-tool.nix { inherit (self.supabase) defaults; }; @@ -47,7 +47,6 @@ mecab-naist-jdic = pkgs.callPackage ./mecab-naist-jdic.nix { }; migrate-tool = pkgs.callPackage ./migrate-tool.nix { psql_15 = self'.packages."psql_15/bin"; }; overlayfs-on-package = pkgs.callPackage ./overlayfs-on-package.nix { }; - packer = pkgs.callPackage ./packer.nix { inherit inputs; }; pg-backrest = inputs.nixpkgs.legacyPackages.${pkgs.stdenv.hostPlatform.system}.pgbackrest; pg-restore = pkgs.callPackage ./pg-restore.nix { psql_15 = self'.packages."psql_15/bin"; }; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; diff --git a/nix/packages/packer.nix b/nix/packages/packer.nix deleted file mode 100644 index aa0d797c9..000000000 --- a/nix/packages/packer.nix +++ /dev/null @@ -1,43 +0,0 @@ -{ - pkgs, - lib, - fetchFromGitHub, - installShellFiles, - ... -}: -let - buildGoModule = pkgs.buildGoModule; -in -buildGoModule rec { - pname = "packer"; - version = "1.14.1"; - - src = fetchFromGitHub { - owner = "hashicorp"; - repo = "packer"; - rev = "v${version}"; - hash = "sha256-3g9hsmrfLzGhjcGvUza/L9PMGUFw+KLbg2pIK0CxlQI="; - }; - - vendorHash = "sha256-F6hn+pXPyPe70UTK8EF24lk7ArYz7ygUyVVsatW6+hI="; - - subPackages = [ "." ]; - - ldflags = [ - "-s" - "-w" - ]; - - nativeBuildInputs = [ installShellFiles ]; - - postInstall = '' - installShellCompletion --zsh contrib/zsh-completion/_packer - ''; - - meta = { - description = "Tool for creating identical machine images for multiple platforms from a single source configuration"; - homepage = "https://www.packer.io"; - license = lib.licenses.bsl11; - changelog = "https://github.com/hashicorp/packer/blob/v${version}/CHANGELOG.md"; - }; -}