diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index afc432683a2..8c3fa6737cc 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -15,34 +15,45 @@ on: required: true type: choice options: - - burkina - - togo - - tanzania + - bfaso + - boad + - chad + - civ - drc + - ecowas + - egypt + - ethiopia + - gambia + - ggw + - haiti + - haitiiati + - haititraining + - honduras + - honduraslight + - honduraslightssc + - jordan + - kosovo + - kyrgyzstan + - liberia + - madagascar - malawi - - zambia - - uganda - - kenya - - ghana - - senegal - - mali + - moldova + - nepal - niger - - benin - - guinea - - sierraleone - - liberia - - gambia - - mauritania - - chad - - cameroon + - rdidemo - rwanda - - burundi - - ethiopia - - mozambique - - madagascar - - comoros - - seychelles - - mauritius + - rwandatest + - senegal + - senegalgiz + - tanzania + - timor + - togo + - uganda + - xchad + pr_number: + description: 'PR number (optional - if provided, will use pr-{number} format for tag and URL). See workflow logs for available PRs.' + required: false + type: string env: PG_VERSION: 14 @@ -53,10 +64,101 @@ jobs: build-and-deploy: runs-on: ubuntu-latest steps: - - name: Checkout code + - name: Setup SSH for submodules + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.DOCKER_BUILD_SSH_KEY || '' }} + + - name: Configure git to use SSH for submodules + run: | + # Configure git to rewrite HTTPS URLs to SSH for submodules + git config --global url."git@github.com:".insteadOf "https://github.com/" + # Add GitHub to known hosts + mkdir -p ~/.ssh + ssh-keyscan -H github.com >> ~/.ssh/known_hosts 2>/dev/null || true + + - name: Checkout code with submodules (default branch) + if: inputs.pr_number == '' + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: true + # Use SSH for private submodules + ssh-key: ${{ secrets.DOCKER_BUILD_SSH_KEY || '' }} + + - name: List available PRs and validate + if: inputs.pr_number != '' + id: pr_info + run: | + set -e + echo "📋 Fetching list of open pull requests..." + echo "" + + # Fetch open PRs using GitHub API + REPO="${{ github.repository }}" + PR_NUMBER="${{ inputs.pr_number }}" + + # Validate PR number is numeric + if ! [[ "$PR_NUMBER" =~ ^[0-9]+$ ]]; then + echo "❌ Error: PR number must be numeric, got: ${PR_NUMBER}" + exit 1 + fi + + # Get list of open PRs (limit to 50 most recent) + PRS_RESPONSE=$(curl -s -H "Authorization: token ${{ github.token }}" \ + "https://api.github.com/repos/${REPO}/pulls?state=open&per_page=50&sort=updated&direction=desc") + + if [ -z "$PRS_RESPONSE" ] || [ "$PRS_RESPONSE" == "[]" ]; then + echo "❌ Error: Could not fetch PR list or no open PRs found" + echo " Cannot validate PR #${PR_NUMBER}" + exit 1 + fi + + echo "Available open PRs:" + echo "$PRS_RESPONSE" | jq -r '.[] | " #\(.number) - \(.title) (branch: \(.head.ref))"' || echo "Could not parse PR list" + echo "" + + # Validate the provided PR number exists in open PRs + PR_EXISTS=$(echo "$PRS_RESPONSE" | jq -r --arg pr "$PR_NUMBER" '.[] | select(.number == ($pr | tonumber)) | .number' || echo "") + + if [ -z "$PR_EXISTS" ]; then + echo "❌ Error: PR #${PR_NUMBER} not found in open PRs list" + echo "" + echo "The PR might be:" + echo " - Closed or merged" + echo " - Not yet created" + echo " - Incorrect number" + echo "" + echo "Please verify the PR number and try again." + exit 1 + fi + + # Get PR details + PR_TITLE=$(echo "$PRS_RESPONSE" | jq -r --arg pr "$PR_NUMBER" '.[] | select(.number == ($pr | tonumber)) | .title' || echo "") + PR_BRANCH=$(echo "$PRS_RESPONSE" | jq -r --arg pr "$PR_NUMBER" '.[] | select(.number == ($pr | tonumber)) | .head.ref' || echo "") + PR_HEAD_REPO=$(echo "$PRS_RESPONSE" | jq -r --arg pr "$PR_NUMBER" '.[] | select(.number == ($pr | tonumber)) | .head.repo.full_name' || echo "") + PR_HEAD_SHA=$(echo "$PRS_RESPONSE" | jq -r --arg pr "$PR_NUMBER" '.[] | select(.number == ($pr | tonumber)) | .head.sha' || echo "") + + echo "✅ PR #${PR_NUMBER} found: ${PR_TITLE}" + echo " Branch: ${PR_BRANCH}" + echo " Repository: ${PR_HEAD_REPO}" + echo " SHA: ${PR_HEAD_SHA}" + + # Store PR branch info for checkout step + echo "PR_BRANCH=${PR_BRANCH}" >> $GITHUB_OUTPUT + echo "PR_HEAD_REPO=${PR_HEAD_REPO}" >> $GITHUB_OUTPUT + echo "PR_HEAD_SHA=${PR_HEAD_SHA}" >> $GITHUB_OUTPUT + + - name: Checkout PR branch + if: inputs.pr_number != '' uses: actions/checkout@v4 with: + ref: ${{ steps.pr_info.outputs.PR_BRANCH }} + repository: ${{ steps.pr_info.outputs.PR_HEAD_REPO }} fetch-depth: 0 + submodules: true + # Use SSH for private submodules + ssh-key: ${{ secrets.DOCKER_BUILD_SSH_KEY || '' }} - name: Set up JDK for Maven uses: actions/setup-java@v4 @@ -67,22 +169,51 @@ jobs: - name: Read AMP version from pom.xml id: amp_version run: | - VERSION=$(mvn -f amp/pom.xml -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec) + # Parse version directly from pom.xml (much faster than running Maven) + # Try to get project.version property first, then fallback to version tag + # This avoids Maven initialization which can take 30+ seconds + + # Method 1: Try to extract project.version property using sed + VERSION=$(sed -n 's/.*\([^<]*\)<\/project\.version>.*/\1/p' amp/pom.xml | head -1) + + # Method 2: If not found, extract from version tag and remove -SNAPSHOT + if [ -z "$VERSION" ]; then + VERSION=$(sed -n 's/.*\([^<]*\)<\/version>.*/\1/p' amp/pom.xml | head -1 | sed 's/-SNAPSHOT//') + fi + + # Clean up: trim whitespace + VERSION=$(echo "$VERSION" | xargs) + + # Fallback to default if still empty + if [ -z "$VERSION" ]; then + echo "âš ī¸ Could not parse version from amp/pom.xml, using default 4.0" + VERSION="4.0" + fi + echo "AMP_VERSION=$VERSION" >> $GITHUB_OUTPUT - echo "AMP Version: $VERSION" + echo "AMP Version: $VERSION (parsed from amp/pom.xml in <1s vs 30s+ for Maven)" - name: Generate deployment tag id: tag run: | - BRANCH_NAME="${GITHUB_REF#refs/heads/}" - if [[ "$BRANCH_NAME" =~ ^feature/AMP-[0-9]+.* ]]; then - JIRA_ID=$(echo "$BRANCH_NAME" | sed -n 's/^feature\/AMP-\([0-9]\+\).*/\1/p') - TAG="feature-${JIRA_ID}" + # Check if PR number is provided + if [ -n "${{ inputs.pr_number }}" ]; then + PR_NUMBER="${{ inputs.pr_number }}" + TAG="pr-${PR_NUMBER}" + echo "PR_NUMBER=${PR_NUMBER}" >> $GITHUB_OUTPUT + echo "Deployment tag: $TAG (PR #${PR_NUMBER})" else - TAG=$(echo "$BRANCH_NAME" | sed 's/[^a-zA-Z0-9_-]/-/g' | tr '[:upper:]' '[:lower:]') + # Handle branch-based deployments + BRANCH_NAME="${GITHUB_REF#refs/heads/}" + if [[ "$BRANCH_NAME" =~ ^feature/AMP-[0-9]+.* ]]; then + JIRA_ID=$(echo "$BRANCH_NAME" | sed -n 's/^feature\/AMP-\([0-9]\+\).*/\1/p') + TAG="feature-${JIRA_ID}" + else + TAG=$(echo "$BRANCH_NAME" | sed 's/[^a-zA-Z0-9_-]/-/g' | tr '[:upper:]' '[:lower:]') + fi + echo "Deployment tag: $TAG" fi echo "TAG=$TAG" >> $GITHUB_OUTPUT - echo "Deployment tag: $TAG" - name: Login to Container Registry (Push) uses: docker/login-action@v3 @@ -94,19 +225,42 @@ jobs: - name: Set deployment hostname and user id: deploy_config run: | - if [[ "${{ inputs.environment }}" == "de" ]]; then - echo "DEPLOY_HOST=${{ secrets.AMP_DE_HOSTNAME }}" >> $GITHUB_OUTPUT - echo "AMP_URL=http://amp-${{ inputs.country }}-${{ steps.tag.outputs.TAG }}.de.ampsite.net/" >> $GITHUB_OUTPUT + # Use inputs from workflow_dispatch + ENV="${{ inputs.environment }}" + COUNTRY="${{ inputs.country }}" + + # Store environment and country + echo "ENV=${ENV}" >> $GITHUB_OUTPUT + echo "COUNTRY=${COUNTRY}" >> $GITHUB_OUTPUT + + if [[ "$ENV" == "de" ]]; then + echo "DEPLOY_HOST=${{ vars.AMP_DE_HOSTNAME }}" >> $GITHUB_OUTPUT + # For PRs, use pr-{number} format in URL + if [ -n "${{ inputs.pr_number }}" ]; then + echo "AMP_URL=http://amp-${COUNTRY}-pr-${{ inputs.pr_number }}.de.ampsite.net/" >> $GITHUB_OUTPUT + else + echo "AMP_URL=http://amp-${COUNTRY}-${{ steps.tag.outputs.TAG }}.de.ampsite.net/" >> $GITHUB_OUTPUT + fi else - echo "DEPLOY_HOST=${{ secrets.AMP_STAGING_HOSTNAME }}" >> $GITHUB_OUTPUT - echo "AMP_URL=http://amp-${{ inputs.country }}-${{ steps.tag.outputs.TAG }}.stg.ampsite.net/" >> $GITHUB_OUTPUT + echo "DEPLOY_HOST=${{ vars.AMP_STAGING_HOSTNAME }}" >> $GITHUB_OUTPUT + # For PRs, use pr-{number} format in URL + if [ -n "${{ inputs.pr_number }}" ]; then + echo "AMP_URL=http://amp-${COUNTRY}-pr-${{ inputs.pr_number }}.stg.ampsite.net/" >> $GITHUB_OUTPUT + else + echo "AMP_URL=http://amp-${COUNTRY}-${{ steps.tag.outputs.TAG }}.stg.ampsite.net/" >> $GITHUB_OUTPUT + fi + fi + + # Store PR number if provided + if [ -n "${{ inputs.pr_number }}" ]; then + echo "PR_NUMBER=${{ inputs.pr_number }}" >> $GITHUB_OUTPUT fi - # Set deploy user from secret with fallback to 'jenkins' - if [ -n "${{ secrets.DEPLOY_USER }}" ]; then - echo "DEPLOY_USER=${{ secrets.DEPLOY_USER }}" >> $GITHUB_OUTPUT + # Set deploy user from vars with fallback to 'jenkins' + if [ -n "${{ vars.DEPLOY_USER }}" ]; then + echo "DEPLOY_USER=${{ vars.DEPLOY_USER }}" >> $GITHUB_OUTPUT else - echo "DEPLOY_USER=jenkins" >> $GITHUB_OUTPUT + echo "DEPLOY_USER=bmokandu" >> $GITHUB_OUTPUT fi - name: Setup SSH @@ -122,7 +276,7 @@ jobs: DEPLOY_HOST="${{ steps.deploy_config.outputs.DEPLOY_HOST }}" DEPLOY_USER="${{ steps.deploy_config.outputs.DEPLOY_USER }}" BASTION_HOST="${{ secrets.BASTION_HOST }}" - BASTION_USER="${{ secrets.BASTION_USER }}" + BASTION_USER="${{ vars.BASTION_USER }}" # Use default bastion user if not specified (matching local config: bmokandu) if [ -z "$BASTION_USER" ] || [ "$BASTION_USER" = "" ]; then @@ -135,15 +289,20 @@ jobs: # Check if deployment host matches *.aws pattern for ProxyCommand if echo "$DEPLOY_HOST" | grep -q "\.aws"; then # Use ProxyCommand pattern for *.aws hosts (matching local SSH config) - # Transform hostname: something.aws.devgateway.org -> something.devgateway.org - if echo "$DEPLOY_HOST" | grep -q "\.aws\.devgateway\.org"; then - TARGET_HOST=$(echo "$DEPLOY_HOST" | sed 's/\.aws\.devgateway\.org$/.devgateway.org/') + # Pattern: %h.devgateway.org means if host is "ampdev.aws", target is "ampdev.aws.devgateway.org" + if echo "$DEPLOY_HOST" | grep -q "\.aws\.devgateway\.org$"; then + # Already in full format: something.aws.devgateway.org + TARGET_HOST="$DEPLOY_HOST" elif echo "$DEPLOY_HOST" | grep -q "\.aws$"; then - TARGET_HOST=$(echo "$DEPLOY_HOST" | sed 's/\.aws$/.devgateway.org/') + # Transform: something.aws -> something.aws.devgateway.org (matching %h.devgateway.org pattern) + TARGET_HOST="${DEPLOY_HOST}.devgateway.org" else + # Fallback: use as-is TARGET_HOST="$DEPLOY_HOST" fi + echo "DEBUG: DEPLOY_HOST=$DEPLOY_HOST, TARGET_HOST=$TARGET_HOST" + cat >> ~/.ssh/config << EOF Host bastion HostName $BASTION_HOST @@ -234,26 +393,31 @@ jobs: - name: Validate selected country run: | + # Get country from deploy_config output + COUNTRY="${{ steps.deploy_config.outputs.COUNTRY }}" + # Convert countries list to newline-separated for validation COUNTRIES=$(echo "${{ steps.countries_list.outputs.COUNTRIES }}" | tr ' ' '\n') # Check if selected country is available - if ! echo "$COUNTRIES" | grep -q "^${{ inputs.country }}$"; then - echo "❌ Country '${{ inputs.country }}' not found in available countries" + if ! echo "$COUNTRIES" | grep -q "^${COUNTRY}$"; then + echo "❌ Country '${COUNTRY}' not found in available countries" echo "" echo "Available countries:" echo "$COUNTRIES" exit 1 fi - echo "✅ Country '${{ inputs.country }}' is available" + echo "✅ Country '${COUNTRY}' is available" - name: Get database version id: db_version run: | DEPLOY_HOST="${{ steps.deploy_config.outputs.DEPLOY_HOST }}" DEPLOY_USER="${{ steps.deploy_config.outputs.DEPLOY_USER }}" - DB_VERSION=$(ssh $DEPLOY_USER@$DEPLOY_HOST "cd /opt/amp_dbs && amp-db find ${{ steps.amp_version.outputs.AMP_VERSION }} ${{ inputs.country }}") + # Get country from deploy_config output + COUNTRY="${{ steps.deploy_config.outputs.COUNTRY }}" + DB_VERSION=$(ssh $DEPLOY_USER@$DEPLOY_HOST "cd /opt/amp_dbs && amp-db find ${{ steps.amp_version.outputs.AMP_VERSION }} ${COUNTRY}") echo "DB_VERSION=$DB_VERSION" >> $GITHUB_OUTPUT echo "Database version: $DB_VERSION" @@ -277,11 +441,42 @@ jobs: with: ssh-private-key: ${{ secrets.DOCKER_BUILD_SSH_KEY || '' }} + - name: Check if image with commit hash already exists + id: check_image + env: + DOCKER_BUILDKIT: 1 + run: | + COMMIT_HASH="${{ steps.commit_hash.outputs.COMMIT_HASH }}" + # Create a commit-hash-based image tag for content-based lookup + IMAGE_BY_HASH="${{ secrets.DOCKER_REGISTRY }}/amp/webapp:commit-${COMMIT_HASH:0:12}" + DEPLOY_TAG="${{ steps.tag.outputs.TAG }}" + IMAGE_BY_TAG="${{ secrets.DOCKER_REGISTRY }}/amp/webapp:${DEPLOY_TAG}" + + echo "Checking for existing image with commit hash ${COMMIT_HASH:0:12}..." + + # Try to pull image by commit hash + if docker pull "$IMAGE_BY_HASH" 2>/dev/null; then + echo "✅ Found existing image for commit ${COMMIT_HASH:0:12}" + echo "SKIP_BUILD=true" >> $GITHUB_OUTPUT + echo "EXISTING_IMAGE=$IMAGE_BY_HASH" >> $GITHUB_OUTPUT + # Tag it with the deployment tag for consistency + docker tag "$IMAGE_BY_HASH" "$IMAGE_BY_TAG" + echo "IMAGE=$IMAGE_BY_TAG" >> $GITHUB_ENV + else + echo "â„šī¸ No existing image found for commit ${COMMIT_HASH:0:12}, will build new image" + echo "SKIP_BUILD=false" >> $GITHUB_OUTPUT + echo "IMAGE=$IMAGE_BY_TAG" >> $GITHUB_ENV + fi + - name: Build Docker image + if: steps.check_image.outputs.SKIP_BUILD == 'false' env: DOCKER_BUILDKIT: 1 + BUILDKIT_PROGRESS: plain run: | - IMAGE="${{ secrets.DOCKER_REGISTRY }}/amp/webapp:${{ steps.tag.outputs.TAG }}" + COMMIT_HASH="${{ steps.commit_hash.outputs.COMMIT_HASH }}" + IMAGE_BY_HASH="${{ secrets.DOCKER_REGISTRY }}/amp/webapp:commit-${COMMIT_HASH:0:12}" + IMAGE_BY_TAG="${{ secrets.DOCKER_REGISTRY }}/amp/webapp:${{ steps.tag.outputs.TAG }}" # Build SSH args - only add if SSH_AUTH_SOCK is available (from ssh-agent) SSH_ARGS="" @@ -289,24 +484,76 @@ jobs: SSH_ARGS="--ssh default" fi + # Collect cache sources for better layer reuse + CACHE_FROM_ARGS=() + + # Try to use commit-hash-based image as cache (if it exists from a previous build) + if docker pull "$IMAGE_BY_HASH" 2>/dev/null; then + echo "✅ Using commit-hash-based image as cache" + CACHE_FROM_ARGS+=("--cache-from" "$IMAGE_BY_HASH") + fi + + # Try to use deployment tag image as cache + if docker pull "$IMAGE_BY_TAG" 2>/dev/null; then + echo "✅ Using deployment tag image as cache" + CACHE_FROM_ARGS+=("--cache-from" "$IMAGE_BY_TAG") + fi + + # Try to use a recent 'latest' or 'main' branch image as cache (if exists) + LATEST_IMAGE="${{ secrets.DOCKER_REGISTRY }}/amp/webapp:latest" + if docker pull "$LATEST_IMAGE" 2>/dev/null; then + echo "✅ Using latest image as additional cache source" + CACHE_FROM_ARGS+=("--cache-from" "$LATEST_IMAGE") + fi + + if [ ${#CACHE_FROM_ARGS[@]} -eq 0 ]; then + echo "â„šī¸ No existing image found, building from scratch" + fi + + # Build the image with both tags + # SKIP_TESTS=true to skip Maven and npm tests for faster deployment builds docker build \ --progress=plain \ $SSH_ARGS \ - -t "$IMAGE" \ + "${CACHE_FROM_ARGS[@]}" \ + -t "$IMAGE_BY_HASH" \ + -t "$IMAGE_BY_TAG" \ --build-arg BUILD_SOURCE="${{ steps.tag.outputs.TAG }}" \ --build-arg AMP_URL="${{ steps.deploy_config.outputs.AMP_URL }}" \ - --build-arg AMP_PULL_REQUEST="" \ - --build-arg AMP_BRANCH="${{ github.ref_name }}" \ + --build-arg AMP_PULL_REQUEST="${{ steps.deploy_config.outputs.PR_NUMBER || '' }}" \ + --build-arg AMP_BRANCH="${GITHUB_REF#refs/heads/}" \ --build-arg AMP_REGISTRY_PRIVATE_KEY="${{ secrets.AMP_REGISTRY_PRIVATE_KEY || '' }}" \ - --label git-hash="${{ steps.commit_hash.outputs.COMMIT_HASH }}" \ + --build-arg SKIP_TESTS=true \ + --label git-hash="$COMMIT_HASH" \ amp - echo "IMAGE=$IMAGE" >> $GITHUB_ENV + echo "✅ Image built successfully" - name: Push Docker image to registry run: | - docker push "${{ env.IMAGE }}" > /dev/null - echo "✅ Image pushed successfully" + COMMIT_HASH="${{ steps.commit_hash.outputs.COMMIT_HASH }}" + IMAGE_BY_HASH="${{ secrets.DOCKER_REGISTRY }}/amp/webapp:commit-${COMMIT_HASH:0:12}" + IMAGE_BY_TAG="${{ secrets.DOCKER_REGISTRY }}/amp/webapp:${{ steps.tag.outputs.TAG }}" + SKIP_BUILD="${{ steps.check_image.outputs.SKIP_BUILD }}" + + # Push commit-hash-based image (for future reuse) - only if we built it + if [ "$SKIP_BUILD" != "true" ]; then + echo "Pushing commit-hash-based image (for future reuse)..." + docker push "$IMAGE_BY_HASH" > /dev/null + fi + + # Push deployment tag image + echo "Pushing deployment tag image..." + docker push "$IMAGE_BY_TAG" > /dev/null + + if [ "$SKIP_BUILD" == "true" ]; then + echo "✅ Reused existing image (no build needed) - saved build time!" + else + echo "✅ Image built and pushed successfully" + fi + + # Set IMAGE for cleanup step + echo "IMAGE=$IMAGE_BY_TAG" >> $GITHUB_ENV - name: Logout from Container Registry if: always() @@ -334,8 +581,94 @@ jobs: DEPLOY_HOST="${{ steps.deploy_config.outputs.DEPLOY_HOST }}" DEPLOY_USER="${{ steps.deploy_config.outputs.DEPLOY_USER }}" - ssh $DEPLOY_USER@$DEPLOY_HOST \ - "amp-up2 ${{ steps.tag.outputs.TAG }} ${{ inputs.country }} ${{ steps.db_version.outputs.DB_VERSION }} ${{ env.PG_VERSION }}" + # Set variables locally for passing to remote server + TAG="${{ steps.tag.outputs.TAG }}" + COUNTRY="${{ steps.deploy_config.outputs.COUNTRY }}" + DB_VERSION="${{ steps.db_version.outputs.DB_VERSION }}" + PG_VERSION="${{ env.PG_VERSION }}" + REGISTRY="${{ secrets.DOCKER_REGISTRY }}" + REGISTRY_USERNAME="${{ secrets.REGISTRY_PULL_USERNAME || '' }}" + REGISTRY_PASSWORD="${{ secrets.REGISTRY_PULL_PASSWORD || '' }}" + PR_NUMBER="${{ inputs.pr_number || '' }}" + + # Debug: Print values before sending to remote server + echo "Debug - Values to be passed to amp-up2:" + echo " TAG: $TAG" + echo " COUNTRY: $COUNTRY" + echo " DB_VERSION: $DB_VERSION" + echo " PG_VERSION: $PG_VERSION" + + # Pass variables as environment variables through SSH + ssh $DEPLOY_USER@$DEPLOY_HOST bash -s << DEPLOY_SCRIPT + set -exo pipefail + TAG="$TAG" + COUNTRY="$COUNTRY" + DB_VERSION="$DB_VERSION" + PG_VERSION="$PG_VERSION" + REGISTRY="$REGISTRY" + REGISTRY_USERNAME="$REGISTRY_USERNAME" + REGISTRY_PASSWORD="$REGISTRY_PASSWORD" + PR_NUMBER="$PR_NUMBER" + + # Debug: Print received values + echo "==========================================" + echo "Values received on remote server:" + echo " TAG: '$TAG'" + echo " COUNTRY: '$COUNTRY'" + echo " DB_VERSION: '$DB_VERSION'" + echo " PG_VERSION: '$PG_VERSION'" + echo "==========================================" + + # Validate required variables + if [ -z "$TAG" ] || [ -z "$COUNTRY" ] || [ -z "$DB_VERSION" ] || [ -z "$PG_VERSION" ]; then + echo "❌ Error: One or more required variables are empty!" + echo " TAG: '$TAG'" + echo " COUNTRY: '$COUNTRY'" + echo " DB_VERSION: '$DB_VERSION'" + echo " PG_VERSION: '$PG_VERSION'" + exit 1 + fi + + # Set image name to match the registry used in the workflow + export AMP_WEBAPP_IMAGE_NAME="${REGISTRY}/amp/webapp" + + # For PRs, log the PR number + if [ -n "$PR_NUMBER" ]; then + echo "Deploying PR #$PR_NUMBER to ${COUNTRY}" + fi + + # Authenticate with main registry if credentials are provided + if [ -n "$REGISTRY_USERNAME" ] && [ -n "$REGISTRY_PASSWORD" ]; then + echo "Authenticating with main registry ($REGISTRY)..." + echo "$REGISTRY_PASSWORD" | docker login --username "$REGISTRY_USERNAME" --password-stdin "$REGISTRY" || true + fi + + # Check if user can access docker without sudo + if docker ps > /dev/null 2>&1; then + echo "==========================================" + echo "Running amp-up2 without sudo" + echo "Command: amp-up2 \"$TAG\" \"$COUNTRY\" \"$DB_VERSION\" \"$PG_VERSION\"" + echo "==========================================" + amp-up2 "$TAG" "$COUNTRY" "$DB_VERSION" "$PG_VERSION" + elif sudo docker ps > /dev/null 2>&1; then + echo "==========================================" + echo "Running amp-up2 with sudo" + echo "Command: sudo amp-up2 \"$TAG\" \"$COUNTRY\" \"$DB_VERSION\" \"$PG_VERSION\"" + echo "==========================================" + # If using sudo, authenticate with registry using sudo as well + if [ -n "$REGISTRY_USERNAME" ] && [ -n "$REGISTRY_PASSWORD" ]; then + echo "$REGISTRY_PASSWORD" | sudo docker login --username "$REGISTRY_USERNAME" --password-stdin "$REGISTRY" || true + fi + sudo amp-up2 "$TAG" "$COUNTRY" "$DB_VERSION" "$PG_VERSION" + else + echo "❌ Cannot access Docker daemon. User may need to be added to docker group." + echo "Run: sudo usermod -aG docker $USER" + exit 1 + fi + echo "==========================================" + echo "amp-up2 completed" + echo "==========================================" + DEPLOY_SCRIPT echo "✅ Deployment successful" diff --git a/amp/Dockerfile b/amp/Dockerfile index f69ec36b48c..fb062bcf777 100644 --- a/amp/Dockerfile +++ b/amp/Dockerfile @@ -2,9 +2,16 @@ FROM node:16.4.0 as node FROM node as compile-amp-state WORKDIR /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-state -COPY TEMPLATE/ampTemplate/node_modules/amp-state . +# Copy package files first for better caching +COPY TEMPLATE/ampTemplate/node_modules/amp-state/package*.json ./ RUN --mount=type=cache,target=/root/.npm \ - npm ci + npm config set fetch-retries 5 \ + && npm config set fetch-retry-mintimeout 20000 \ + && npm config set fetch-retry-maxtimeout 120000 \ + && npm config set fetch-timeout 300000 \ + && (npm ci || (sleep 10 && npm ci) || (sleep 20 && npm ci)) +# Copy source code after dependencies are installed +COPY TEMPLATE/ampTemplate/node_modules/amp-state . FROM node as compile-amp-translate @@ -16,7 +23,11 @@ COPY TEMPLATE/ampTemplate/node_modules/amp-translate/package*.json ./ # Install dependencies RUN --mount=type=cache,target=/root/.npm \ - npm ci + npm config set fetch-retries 5 \ + && npm config set fetch-retry-mintimeout 20000 \ + && npm config set fetch-retry-maxtimeout 120000 \ + && npm config set fetch-timeout 300000 \ + && (npm ci || (sleep 10 && npm ci) || (sleep 20 && npm ci)) # Copy the rest of the application code COPY TEMPLATE/ampTemplate/node_modules/amp-translate ./ @@ -27,102 +38,171 @@ COPY TEMPLATE/ampTemplate/node_modules/amp-translate ./ FROM node as compile-amp-boilerplate WORKDIR /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-boilerplate -COPY TEMPLATE/ampTemplate/node_modules/amp-boilerplate/.git . -COPY TEMPLATE/ampTemplate/node_modules/amp-boilerplate . +# Copy dependencies first COPY --from=compile-amp-translate /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-translate ../amp-translate +# Copy package files for dependency installation +COPY TEMPLATE/ampTemplate/node_modules/amp-boilerplate/package*.json ./ +COPY TEMPLATE/ampTemplate/node_modules/amp-boilerplate/.git ./ RUN --mount=type=cache,target=/root/.npm \ - npm ci \ - && npm run build + npm config set fetch-retries 5 \ + && npm config set fetch-retry-mintimeout 20000 \ + && npm config set fetch-retry-maxtimeout 120000 \ + && npm config set fetch-timeout 300000 \ + && (npm ci || (sleep 10 && npm ci) || (sleep 20 && npm ci)) +# Copy source code after dependencies are installed +COPY TEMPLATE/ampTemplate/node_modules/amp-boilerplate . +RUN npm run build FROM node as compile-amp-filter WORKDIR /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-filter -COPY TEMPLATE/ampTemplate/node_modules/amp-filter/.git . +# Copy dependencies first COPY --from=compile-amp-translate /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-translate ../amp-translate +# Copy package files for dependency installation +COPY TEMPLATE/ampTemplate/node_modules/amp-filter/package*.json ./ +COPY TEMPLATE/ampTemplate/node_modules/amp-filter/.git ./ +RUN --mount=type=cache,target=/root/.npm \ + npm config set fetch-retries 5 \ + && npm config set fetch-retry-mintimeout 20000 \ + && npm config set fetch-retry-maxtimeout 120000 \ + && npm config set fetch-timeout 300000 \ + && (npm ci || (sleep 10 && npm ci) || (sleep 20 && npm ci)) +# Copy source code after dependencies are installed COPY TEMPLATE/ampTemplate/node_modules/amp-filter . # HACK otherwise amp-filter won't compile! COPY TEMPLATE/reamp/tools/log /tmp/amp/TEMPLATE/reamp/tools/log -RUN --mount=type=cache,target=/root/.npm \ - npm ci \ - && npm run build +RUN npm run build FROM node as compile-amp-settings WORKDIR /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-settings -#COPY TEMPLATE/ampTemplate/node_modules/amp-settings/.git . -COPY TEMPLATE/ampTemplate/node_modules/amp-settings . +# Copy dependencies first COPY --from=compile-amp-translate /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-translate ../amp-translate +# Copy package files for dependency installation +COPY TEMPLATE/ampTemplate/node_modules/amp-settings/package*.json ./ +RUN --mount=type=cache,target=/root/.npm \ + npm config set fetch-retries 5 \ + && npm config set fetch-retry-mintimeout 20000 \ + && npm config set fetch-retry-maxtimeout 120000 \ + && npm config set fetch-timeout 300000 \ + && (npm ci || (sleep 10 && npm ci) || (sleep 20 && npm ci)) +# Copy source code after dependencies are installed +COPY TEMPLATE/ampTemplate/node_modules/amp-settings . # HACK otherwise amp-settings won't compile! COPY TEMPLATE/reamp/tools/log /tmp/amp/TEMPLATE/reamp/tools/log -RUN --mount=type=cache,target=/root/.npm \ - npm ci \ - && npm run build +RUN npm run build FROM node as compile-gis-layers-manager WORKDIR /tmp/amp/TEMPLATE/ampTemplate/node_modules/gis-layers-manager +# Copy dependencies first COPY --from=compile-amp-translate /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-translate ../amp-translate -COPY TEMPLATE/ampTemplate/node_modules/gis-layers-manager . +# Copy package files for dependency installation +COPY TEMPLATE/ampTemplate/node_modules/gis-layers-manager/package*.json ./ RUN --mount=type=cache,target=/root/.npm \ - npm ci \ - && npm run build + npm config set fetch-retries 5 \ + && npm config set fetch-retry-mintimeout 20000 \ + && npm config set fetch-retry-maxtimeout 120000 \ + && npm config set fetch-timeout 300000 \ + && (npm ci || (sleep 10 && npm ci) || (sleep 20 && npm ci)) +# Copy source code after dependencies are installed +COPY TEMPLATE/ampTemplate/node_modules/gis-layers-manager . +RUN npm run build FROM node as compile-gis-module WORKDIR /tmp/amp/TEMPLATE/ampTemplate/gisModule +# Copy dependencies first (these change less frequently) COPY --from=compile-amp-state /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-state ../node_modules/amp-state COPY --from=compile-amp-translate /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-translate ../node_modules/amp-translate COPY --from=compile-amp-boilerplate /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-boilerplate ../node_modules/amp-boilerplate COPY --from=compile-amp-filter /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-filter ../node_modules/amp-filter -# HACK otherwise amp-filter won't compile! -COPY TEMPLATE/reamp/tools/log /tmp/amp/TEMPLATE/reamp/tools/log COPY --from=compile-amp-settings /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-settings ../node_modules/amp-settings COPY --from=compile-gis-layers-manager /tmp/amp/TEMPLATE/ampTemplate/node_modules/gis-layers-manager ../node_modules/gis-layers-manager -COPY TEMPLATE/ampTemplate/gisModule . +# HACK otherwise amp-filter won't compile! +COPY TEMPLATE/reamp/tools/log /tmp/amp/TEMPLATE/reamp/tools/log +# Copy package files for dependency installation +COPY TEMPLATE/ampTemplate/gisModule/dev/package*.json ./dev/ RUN --mount=type=cache,target=/root/.npm \ cd dev \ - && npm ci \ - && npm run test \ + && npm config set fetch-retries 5 \ + && npm config set fetch-retry-mintimeout 20000 \ + && npm config set fetch-retry-maxtimeout 120000 \ + && npm config set fetch-timeout 300000 \ + && (npm ci || (sleep 10 && npm ci) || (sleep 20 && npm ci)) +# Copy source code after dependencies are installed +COPY TEMPLATE/ampTemplate/gisModule . +ARG SKIP_TESTS=false +RUN cd dev \ + && if [ "$SKIP_TESTS" != "true" ]; then npm run test; fi \ && npm run build \ && rm -rf node_modules FROM node as compile-dashboard WORKDIR /tmp/amp/TEMPLATE/ampTemplate/dashboard +# Copy dependencies first (these change less frequently) COPY --from=compile-amp-state /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-state ../node_modules/amp-state COPY --from=compile-amp-translate /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-translate ../node_modules/amp-translate COPY --from=compile-amp-boilerplate /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-boilerplate ../node_modules/amp-boilerplate COPY --from=compile-amp-filter /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-filter ../node_modules/amp-filter -# HACK otherwise amp-filter won't compile! -COPY TEMPLATE/reamp/tools/log /tmp/amp/TEMPLATE/reamp/tools/log COPY --from=compile-amp-settings /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-settings ../node_modules/amp-settings COPY TEMPLATE/ampTemplate/node_modules/amp-url ../node_modules/amp-url -COPY TEMPLATE/ampTemplate/dashboard . +# HACK otherwise amp-filter won't compile! +COPY TEMPLATE/reamp/tools/log /tmp/amp/TEMPLATE/reamp/tools/log +# Copy package files for dependency installation +COPY TEMPLATE/ampTemplate/dashboard/dev/package*.json ./dev/ RUN --mount=type=cache,target=/root/.npm \ cd dev \ - && npm ci \ + && npm config set fetch-retries 5 \ + && npm config set fetch-retry-mintimeout 20000 \ + && npm config set fetch-retry-maxtimeout 120000 \ + && npm config set fetch-timeout 300000 \ + && (npm ci || (sleep 10 && npm ci) || (sleep 20 && npm ci)) +# Copy source code after dependencies are installed +COPY TEMPLATE/ampTemplate/dashboard . +RUN cd dev \ && npm run build \ && rm -rf node_modules FROM node as compile-reamp WORKDIR /tmp/amp/TEMPLATE/reamp +# Copy dependencies first COPY --from=compile-amp-boilerplate /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-boilerplate ../ampTemplate/node_modules/amp-boilerplate -COPY TEMPLATE/reamp . +# Copy package files for dependency installation +COPY TEMPLATE/reamp/package*.json ./ RUN --mount=type=cache,target=/root/.npm \ --mount=type=ssh \ - npm ci \ - && npm run build \ + npm config set fetch-retries 5 \ + && npm config set fetch-retry-mintimeout 20000 \ + && npm config set fetch-retry-maxtimeout 120000 \ + && npm config set fetch-timeout 300000 \ + && (npm ci || (sleep 10 && npm ci) || (sleep 20 && npm ci)) +# Copy source code after dependencies are installed +COPY TEMPLATE/reamp . +RUN npm run build \ && rm -rf node_modules FROM node as compile-reampv2 WORKDIR /tmp/amp/TEMPLATE/reampv2 +# Copy dependencies first COPY --from=compile-amp-boilerplate /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-boilerplate ../ampTemplate/node_modules/amp-boilerplate COPY --from=compile-amp-filter /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-filter ../ampTemplate/node_modules/amp-filter COPY --from=compile-amp-settings /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-settings ../ampTemplate/node_modules/amp-settings -COPY TEMPLATE/reampv2 . +# Copy package files for dependency installation +COPY TEMPLATE/reampv2/package*.json ./ RUN --mount=type=cache,target=/root/.npm \ --mount=type=ssh \ - npm ci \ - && npm run build \ + npm config set fetch-retries 5 \ + && npm config set fetch-retry-mintimeout 20000 \ + && npm config set fetch-retry-maxtimeout 120000 \ + && npm config set fetch-timeout 300000 \ + && (npm ci || (sleep 10 && npm ci) || (sleep 20 && npm ci)) +# Copy source code after dependencies are installed +COPY TEMPLATE/reampv2 . +RUN npm run build \ && rm -rf node_modules FROM maven:3.8.4-jdk-8 as compile-mvn WORKDIR /tmp/amp -COPY . . +# Copy pom.xml first for better Maven caching +COPY pom.xml . +#COPY amp/pom.xml amp/ +# Copy only Maven dependencies (these change less frequently) COPY --from=compile-amp-boilerplate /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-boilerplate/dist TEMPLATE/ampTemplate/node_modules/amp-boilerplate/dist COPY --from=compile-amp-filter /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-filter/dist TEMPLATE/ampTemplate/node_modules/amp-filter/dist COPY --from=compile-amp-settings /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-settings/dist TEMPLATE/ampTemplate/node_modules/amp-settings/dist @@ -131,15 +211,31 @@ COPY --from=compile-gis-module /tmp/amp/TEMPLATE/ampTemplate/gisModule/dist TEMP COPY --from=compile-dashboard /tmp/amp/TEMPLATE/ampTemplate/dashboard/build TEMPLATE/ampTemplate/dashboard/build COPY --from=compile-reamp /tmp/amp/TEMPLATE/reamp TEMPLATE/reamp COPY --from=compile-reampv2 /tmp/amp/TEMPLATE/reampv2 TEMPLATE/reampv2 +# Download Maven dependencies (this layer will be cached if pom.xml doesn't change) ARG BUILD_SOURCE RUN --mount=type=cache,target=/root/.m2 \ - mvn -B test war:exploded \ - -DbuildSource=$BUILD_SOURCE \ - -Djdbc.user=amp -Djdbc.password=amp122006 -Djdbc.db=amp -Djdbc.host=db \ - -Djdbc.port=5432 -DdbName=postgresql -Djdbc.driverClassName=org.postgresql.Driver \ - -Dskip.npm -Dskip.installnodenpm \ - && mv target/amp exploded \ - && rm -rf target + mvn -B dependency:go-offline -f pom.xml || true +# Copy source code after dependencies are downloaded +COPY . . +ARG SKIP_TESTS=false +RUN --mount=type=cache,target=/root/.m2 \ + if [ "$SKIP_TESTS" = "true" ]; then \ + mvn -B clean compile war:exploded \ + -DbuildSource=$BUILD_SOURCE \ + -Djdbc.user=amp -Djdbc.password=amp122006 -Djdbc.db=amp -Djdbc.host=db \ + -Djdbc.port=5432 -DdbName=postgresql -Djdbc.driverClassName=org.postgresql.Driver \ + -Dskip.npm -Dskip.installnodenpm -DskipTests \ + && mv target/amp exploded \ + && rm -rf target; \ + else \ + mvn -B test war:exploded \ + -DbuildSource=$BUILD_SOURCE \ + -Djdbc.user=amp -Djdbc.password=amp122006 -Djdbc.db=amp -Djdbc.host=db \ + -Djdbc.port=5432 -DdbName=postgresql -Djdbc.driverClassName=org.postgresql.Driver \ + -Dskip.npm -Dskip.installnodenpm \ + && mv target/amp exploded \ + && rm -rf target; \ + fi FROM tomcat:8.5.79-jdk8