Enhances Playwright test workflow

Adds support for different application types (Vue and .NET) and improves server readiness checks.

This change introduces a new `applicationType` parameter to the Playwright test template, allowing it to handle both Vue and .NET applications. It improves the reliability of the test execution by implementing a readiness check mechanism that waits for the server to be fully up and running before starting the tests. This ensures that the tests are not run against an unavailable server, which can lead to false negatives.

The changes include:
- Parameterization of application type (vue or dotnet)
- Server readiness check implementation
- Docker build arguments can now be passed through

azure-pipelines-automated-testing.yml

@@ -33,6 +33,7 @@ stages:
     # TODO: Change to ADO Playwright template
     - template: temp/playwright-test.yml
       parameters:
+        applicationType: 'vue'
         totalShards: ${{ variables.totalShards }}
         targetUrl: $(BASE_URL)
         dockerFileName: 'Dockerfile.playwright'

azure-pipelines.yml

@@ -64,6 +64,7 @@ stages:
             jobs:
                 - template: temp/playwright-test.yml
                   parameters:
+                      applicationType: 'vue'
                       totalShards: 2
                       targetUrl: $(BASE_URL)
                       dockerFileName: 'Dockerfile.playwright'

temp/playwright-test.yml

@@ -32,11 +32,23 @@ parameters:
   - name: imageTag
     type: string
     default: '$(Build.BuildId)'
+  - name: dockerBuildArgs
+    type: string
+    default: ''
   - name: envFileName
     type: string
     default: '.env.ci'
+  - name: applicationType
+    type: string
+    default: 'vue'
+    values: ['vue', 'dotnet']
+  - name: readinessEndpoint
+    type: string
+    default: '/healthcheck'
+  - name: serverTimeoutSeconds
+    type: number
+    default: 60
 
-      
 jobs:
 - job: playwright_tests
   continueOnError: true
@@ -103,6 +115,14 @@ jobs:
       echo "Docker cleanup completed!"
     displayName: "Docker Cleanup"
 
+  - bash: |
+      printenv > "${{ parameters.envFileName }}"
+      if [ -z "$(GITHUB_TOKEN)" ]; then
+        echo "Printed env, but GITHUB_TOKEN was undefined"
+      fi
+    env: ${{ parameters.secrets }}
+    displayName: "Make Azure Pipeline Variables Available to Docker"
+
   - task: Docker@2
     displayName: 'Build Docker Image'
     inputs:
@@ -110,20 +130,23 @@ jobs:
       dockerfile: ${{ parameters.dockerFileName }}
       repository: ${{ parameters.dockerImageName }}
       tags: ${{ parameters.imageTag }}
-      arguments: '--no-cache --pull'
+      arguments: |
+        --no-cache --pull ${{ parameters.dockerBuildArgs }}
 
   - script: |
-      printenv > "${{ parameters.envFileName }}"
-
+      # Determine JIRA card number and filters
       JIRA_CARD_NUMBER=""
       IS_REGRESSION="${{ parameters.isRegression }}"
-      echo "isRegression: $IS_REGRESSION";
+      echo "isRegression: $IS_REGRESSION"
+      
       if [[ "${{ parameters.isRegression }}" == "False" ]]; then
-        branch_name=$(System.PullRequest.SourceBranch)
+        branch_name="$(System.PullRequest.SourceBranch)"
         echo "Retrieved branch name: '$branch_name'"
         JIRA_CARD_NUMBER="${branch_name##*/}"
         echo "Extracted JIRA Card number: '$JIRA_CARD_NUMBER'"
       fi
+      
+      # Build test filter
       FILTER=""
       if [[ -n "${{ parameters.filterTags }}" && -n "$JIRA_CARD_NUMBER" ]]; then
         echo "Filtering by filterTags and Jira Card Number..."
@@ -140,57 +163,213 @@ jobs:
       fi
 
       echo "Filter value: $FILTER"
-      TARGET_URL=${{ parameters.targetUrl }}
-      echo "BASE_URL value: $BASE_URL"
+      TARGET_URL="${{ parameters.targetUrl }}"
+      echo "Target URL: $TARGET_URL"
+      echo "Application Type: ${{ parameters.applicationType }}"
 
-      # Create container and run tests
+      # Set common variables
+      PLAYWRIGHT_PATH="${{ parameters.playwrightTestsPath }}"
+      SERVE_PATH="${{ parameters.npmServePath }}"
+      SHARD_NUMBER=$(shardNumber)
+      TOTAL_SHARDS=${{ parameters.totalShards }}
+      APPLICATION_TYPE="${{ parameters.applicationType }}"
+      ENV_FILE="${{ parameters.envFileName }}"
 
+      # Handle localhost URLs
       if [[ "$TARGET_URL" == *localhost* ]]; then
-        echo 'Npx version:'
-        npx --version
-        port="$TARGET_URL"
-        echo "URL: '$port'"
-        echo "URL '$port' contains 'localhost'. Extracting port... "
-        port=$(echo "$port" | sed -E 's/.*:([0-9]+).*/\1/')
-        echo "Port is '$port'"
-        TARGET_URL="$TARGET_URL"
-        JIRA_CARD_NUMBER="$JIRA_CARD_NUMBER"
-        FILTER="$FILTER"
-        PLAYWRIGHT_PATH="${{ parameters.playwrightTestsPath }}"
-        SERVE_PATH="${{ parameters.npmServePath }}"
-        SHARD_NUMBER=$(shardNumber)
-        TOTAL_SHARDS=${{ parameters.totalShards }}
+        echo "Localhost URL detected, extracting port..."
+        port=$(echo "$TARGET_URL" | sed -E 's/.*:([0-9]+).*/\1/')
+        echo "Port extracted: $port"
+        
+        # Create readiness check URL based on application type
+        if [[ "$APPLICATION_TYPE" == "dotnet" ]]; then
+          # For .NET APIs, append the readiness endpoint to the target URL
+          READINESS_URL="${TARGET_URL}${{ parameters.readinessEndpoint }}"
+          echo "Using .NET readiness check URL: $READINESS_URL"
+        else
+          # For Vue apps, use the target URL as-is
+          READINESS_URL="$TARGET_URL"
+          echo "Using Vue app URL for readiness check: $READINESS_URL"
+        fi
+        
+        echo "Starting tests for shard $SHARD_NUMBER of $TOTAL_SHARDS..."
+        
         container_id=$(docker create \
           --ipc=host \
           --env CI=true \
+          --env TARGET_URL="$TARGET_URL" \
+          --env READINESS_URL="$READINESS_URL" \
+          --env PLAYWRIGHT_PATH="$PLAYWRIGHT_PATH" \
+          --env SERVE_PATH="$SERVE_PATH" \
+          --env SHARD_NUMBER="$SHARD_NUMBER" \
+          --env TOTAL_SHARDS="$TOTAL_SHARDS" \
+          --env FILTER="$FILTER" \
+          --env APPLICATION_TYPE="$APPLICATION_TYPE" \
+          --env SERVER_TIMEOUT="${{ parameters.serverTimeoutSeconds }}" \
+          --env ASPNETCORE_URLS="https://localhost:$port" \
+          --env ENV_FILE="$ENV_FILE" \
           --env-file "${{ parameters.envFileName }}" \
           ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} \
-          npx concurrently -k -n 'server,playwright' \
-            "npm --prefix $SERVE_PATH run serve -- --port=$port" \
-            "npx --prefix $PLAYWRIGHT_PATH wait-on $TARGET_URL && PLAYWRIGHT_BLOB_OUTPUT_DIR='/app/blob-report' npx --prefix $PLAYWRIGHT_PATH playwright test $PLAYWRIGHT_PATH --config=$PLAYWRIGHT_PATH/playwright.config.ts --shard=$SHARD_NUMBER/$TOTAL_SHARDS --reporter=list,blob $FILTER" \
+          bash -c '
+            echo "=== Starting Application Server ==="
+            echo "Application Type: $APPLICATION_TYPE"
+            echo "Target URL: $TARGET_URL"
+            echo "Readiness Check URL: $READINESS_URL"
+            echo "Server Timeout: $SERVER_TIMEOUT seconds"
+            
+            # Function to wait for server readiness
+            wait_for_server() {
+              local url=$1
+              local max_attempts=$SERVER_TIMEOUT
+              local attempt=1
+              
+              echo "Waiting for server at: $url"
+              
+              while [ $attempt -le $max_attempts ]; do
+                echo "Attempt $attempt/$max_attempts: Checking server health..."
+                
+                if [[ "$APPLICATION_TYPE" == "vue" ]]; then
+                  # For Vue apps, use a simple HTTP check
+                  if curl -s -f "$url" --max-time 10 > /dev/null 2>&1; then
+                    echo "Vue server is ready!"
+                    return 0
+                  fi
+                else
+                  # For .NET APIs, use healthcheck endpoint
+                  if curl -k -s -f "$url" --max-time 10 > /dev/null 2>&1; then
+                    echo ".NET server is ready!"
+                    return 0
+                  fi
+                fi
+                
+                if [ $attempt -eq $max_attempts ]; then
+                  echo "Server failed to become ready after $max_attempts attempts"
+                  return 1
+                fi
+                
+                echo "Server not ready yet, waiting 2 seconds..."
+                sleep 2
+                attempt=$((attempt + 1))
+              done
+            }
+            
+            # Install curl if not available
+            if ! command -v curl &> /dev/null; then
+              echo "Installing curl..."
+              apk add --no-cache curl 2>/dev/null || apt-get update && apt-get install -y curl 2>/dev/null || true
+            fi
+            
+            # Start server and run tests based on application type
+            if [[ "$APPLICATION_TYPE" == "vue" ]]; then
+              echo "=== Using Vue Mode with wait-on and concurrently ==="
+              
+              # Extract port for Vue server
+              port=$(echo "$TARGET_URL" | sed -E "s/.*:([0-9]+).*/\1/")
+              echo "Starting Vue server on port: $port"
+              
+              # Create blob report directory
+              mkdir -p /app/blob-report
+              
+              # Use concurrently for Vue apps (original approach)
+              if [ -n "$FILTER" ]; then
+                echo "Running Vue tests with filter: $FILTER"
+                npx concurrently -k -n "server,playwright" \
+                  "npm --prefix $SERVE_PATH run serve -- --port=$port" \
+                  "npx --prefix $PLAYWRIGHT_PATH wait-on $TARGET_URL && cd $PLAYWRIGHT_PATH && PLAYWRIGHT_BLOB_OUTPUT_DIR=\"/app/blob-report\" npx dotenv-cli -e \"../$ENV_FILE\" -- playwright test --config=./playwright.config.ts --shard=$SHARD_NUMBER/$TOTAL_SHARDS --reporter=list,blob $FILTER"
+              else
+                echo "Running all Vue tests"
+                npx concurrently -k -n "server,playwright" \
+                  "npm --prefix $SERVE_PATH run serve -- --port=$port" \
+                  "npx --prefix $PLAYWRIGHT_PATH wait-on $TARGET_URL && cd $PLAYWRIGHT_PATH && PLAYWRIGHT_BLOB_OUTPUT_DIR=\"/app/blob-report\" npx dotenv-cli -e \"../$ENV_FILE\" -- playwright test --config=./playwright.config.ts --shard=$SHARD_NUMBER/$TOTAL_SHARDS --reporter=list,blob"
+              fi
+              
+            else
+              echo "=== Using .NET Mode with healthcheck ==="
+              echo "Starting .NET server..."
+              echo "ASPNETCORE_URLS: $ASPNETCORE_URLS"
+              
+              npm --prefix "$SERVE_PATH" run serve &
+              SERVER_PID=$!
+              echo ".NET server started with PID: $SERVER_PID"
+              
+              # Wait for .NET server readiness check
+              if ! wait_for_server "$READINESS_URL"; then
+                kill $SERVER_PID 2>/dev/null || true
+                exit 1
+              fi
+              
+              echo "=== Starting Playwright Tests ==="
+              
+              # Create blob report directory
+              mkdir -p /app/blob-report
+              cd "$PLAYWRIGHT_PATH"
+
+              # Run Playwright tests
+              if [ -n "$FILTER" ]; then
+                echo "Running .NET tests with filter: $FILTER"
+                PLAYWRIGHT_BLOB_OUTPUT_DIR="/app/blob-report" npx dotenv-cli -e "../$ENV_FILE" -- playwright test \
+                  --config="./playwright.config.ts" \
+                  --shard="$SHARD_NUMBER/$TOTAL_SHARDS" \
+                  --reporter=list,blob \
+                  $FILTER || true
+              else
+                echo "Running all .NET tests"
+                PLAYWRIGHT_BLOB_OUTPUT_DIR="/app/blob-report" npx dotenv-cli -e "../$ENV_FILE" -- playwright test \
+                  --config="./playwright.config.ts" \
+                  --shard="$SHARD_NUMBER/$TOTAL_SHARDS" \
+                  --reporter=list,blob || true
+              fi
+              
+              TEST_EXIT_CODE=$?
+              echo "Tests completed with exit code: $TEST_EXIT_CODE"
+              
+              # Clean up server
+              echo "Stopping server..."
+              kill $SERVER_PID 2>/dev/null || true
+              
+              exit $TEST_EXIT_CODE
+            fi
+          '
         )
       else
-        export TARGET_URL="$TARGET_URL"
-        export JIRA_CARD_NUMBER="$JIRA_CARD_NUMBER"
-        export FILTER="$FILTER"
-        export PLAYWRIGHT_PATH="${{ parameters.playwrightTestsPath }}"
-        export SERVE_PATH="${{ parameters.npmServePath }}"
-        export SHARD_NUMBER=$(shardNumber)
-        export TOTAL_SHARDS=${{ parameters.totalShards }}
-        export ENV_FILE="${{ parameters.envFileName }}"
+        # Remote URL testing
+        echo "Remote URL testing mode"
         container_id=$(docker create \
           --ipc=host \
           --env CI=true \
+          --env TARGET_URL="$TARGET_URL" \
+          --env PLAYWRIGHT_PATH="$PLAYWRIGHT_PATH" \
+          --env SHARD_NUMBER="$SHARD_NUMBER" \
+          --env TOTAL_SHARDS="$TOTAL_SHARDS" \
+          --env FILTER="$FILTER" \
+          --env ENV_FILE="$ENV_FILE" \
           --env-file "${{ parameters.envFileName }}" \
           ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} \
           bash -c "
-            set -a
-            source $ENV_FILE
-            set +a
-            PLAYWRIGHT_BLOB_OUTPUT_DIR='/app/blob-report' npx --prefix $PLAYWRIGHT_PATH playwright test $PLAYWRIGHT_PATH --config=$PLAYWRIGHT_PATH/playwright.config.ts --shard=$SHARD_NUMBER/$TOTAL_SHARDS --reporter=list,blob $FILTER
-          " \
+            echo '=== Running Tests Against Remote URL ==='
+            echo 'Target URL: $TARGET_URL'
+            
+            mkdir -p /app/blob-report
+            cd \$PLAYWRIGHT_PATH
+            
+            if [ -n \"\$FILTER\" ]; then
+              echo 'Running tests with filter: \$FILTER'
+              PLAYWRIGHT_BLOB_OUTPUT_DIR='/app/blob-report' npx dotenv-cli -e \"../\$ENV_FILE\" -- playwright test \
+                --config=./playwright.config.ts \
+                --shard=\$SHARD_NUMBER/\$TOTAL_SHARDS \
+                --reporter=list,blob \
+                \$FILTER || true
+            else
+              echo 'Running all tests'
+              PLAYWRIGHT_BLOB_OUTPUT_DIR='/app/blob-report' npx dotenv-cli -e \"../\$ENV_FILE\" -- playwright test \
+                --config=./playwright.config.ts \
+                --shard=\$SHARD_NUMBER/\$TOTAL_SHARDS \
+                --reporter=list,blob || true
+            fi
+          "
         )
       fi
+      
       # Start container and stream logs
       echo "Starting tests for shard $(shardNumber)..."
       docker start -a $container_id
@@ -206,12 +385,6 @@ jobs:
       # Remove container
       echo "Cleaning up container..."
       docker rm $container_id
-
-      # Check if tests failed
-      if [ $? -ne 0 ]; then
-        echo "Tests failed in shard $(shardNumber) or tests don't exist for this shard number!"
-        exit 0  # Suppress error. It will be visible in report.
-      fi
     displayName: 'Run Playwright Tests - Shard $(shardNumber)'
     env: ${{ parameters.secrets }}
 
@@ -227,6 +400,7 @@ jobs:
       docker rmi ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} -f
     displayName: 'Cleanup Docker Image'
     condition: always()
+
 - job: download_and_merge_reports
   dependsOn: playwright_tests
   timeoutInMinutes: 8
@@ -238,6 +412,10 @@ jobs:
   - task: DownloadPipelineArtifact@2
     inputs:
       targetPath: '$(System.DefaultWorkingDirectory)/playwright-reports'
+  - bash: |
+      printenv > "${{ parameters.envFileName }}"
+    env: ${{ parameters.secrets }}
+    displayName: "Make Azure Pipeline Variables Available to Docker"
   - task: Docker@2
     displayName: 'Build Docker Image'
     inputs:
@@ -245,9 +423,9 @@ jobs:
       dockerfile: ${{ parameters.dockerFileName }}
       repository: ${{ parameters.dockerImageName }}
       tags: ${{ parameters.imageTag }}
-      arguments: '--no-cache --pull'
+      arguments: |
+        --no-cache --pull ${{ parameters.dockerBuildArgs }}
   - bash: |
-      printenv > "${{ parameters.envFileName }}"
       branch_name=$(System.PullRequest.SourceBranch)
       echo "Retrieved branch name: '$branch_name'"
       JIRA_CARD_NUMBER="${branch_name##*/}"
@@ -263,14 +441,12 @@ jobs:
               --env-file "${{ parameters.envFileName }}" \
               ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} \
               bash -c "
-                set -a
-                source $ENV_FILE
-                set +a
                 echo 'Moving Playwright reports out of subfolders...'
                 find ./playwright-reports/ -mindepth 2 -type f -exec mv {} ./playwright-reports/ \;
                 echo 'Value of JIRA_CARD_NUMBER in bash -c command: $JIRA_CARD_NUMBER'
                 echo 'Merging reports...'
-                JIRA_CARD_NUMBER='$JIRA_CARD_NUMBER' PLAYWRIGHT_JUNIT_OUTPUT_DIR='/app/test-results' PLAYWRIGHT_JUNIT_OUTPUT_NAME='junit_results.xml' npx --prefix $PLAYWRIGHT_PATH playwright merge-reports --config=$PLAYWRIGHT_PATH/playwright.config.ts ./playwright-reports
+                cd $PLAYWRIGHT_PATH
+                JIRA_CARD_NUMBER='$JIRA_CARD_NUMBER' PLAYWRIGHT_JUNIT_OUTPUT_DIR='/app/test-results' PLAYWRIGHT_JUNIT_OUTPUT_NAME='junit_results.xml' npx  dotenv-cli -e ../$ENV_FILE -- playwright merge-reports --config=./playwright.config.ts ../playwright-reports
               "
       )