From 9c643ba48cda75bffca5278f29bccc1323717469 Mon Sep 17 00:00:00 2001 From: maguire-arman Date: Tue, 29 Jul 2025 12:45:00 -0400 Subject: [PATCH] Enhances Playwright test workflow Adds support for different application types (Vue and .NET) and improves server readiness checks. This change introduces a new `applicationType` parameter to the Playwright test template, allowing it to handle both Vue and .NET applications. It improves the reliability of the test execution by implementing a readiness check mechanism that waits for the server to be fully up and running before starting the tests. This ensures that the tests are not run against an unavailable server, which can lead to false negatives. The changes include: - Parameterization of application type (vue or dotnet) - Server readiness check implementation - Docker build arguments can now be passed through --- azure-pipelines-automated-testing.yml | 1 + azure-pipelines.yml | 1 + temp/playwright-test.yml | 278 +++++++++++++++++++++----- 3 files changed, 229 insertions(+), 51 deletions(-) diff --git a/azure-pipelines-automated-testing.yml b/azure-pipelines-automated-testing.yml index 34cda2917..132da13ce 100644 --- a/azure-pipelines-automated-testing.yml +++ b/azure-pipelines-automated-testing.yml @@ -33,6 +33,7 @@ stages: # TODO: Change to ADO Playwright template - template: temp/playwright-test.yml parameters: + applicationType: 'vue' totalShards: ${{ variables.totalShards }} targetUrl: $(BASE_URL) dockerFileName: 'Dockerfile.playwright' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9774bd48a..39519804c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -64,6 +64,7 @@ stages: jobs: - template: temp/playwright-test.yml parameters: + applicationType: 'vue' totalShards: 2 targetUrl: $(BASE_URL) dockerFileName: 'Dockerfile.playwright' diff --git a/temp/playwright-test.yml b/temp/playwright-test.yml index 5c6692586..474a0bd75 100644 --- a/temp/playwright-test.yml +++ b/temp/playwright-test.yml @@ -32,11 +32,23 @@ parameters: - name: imageTag type: string default: '$(Build.BuildId)' + - name: dockerBuildArgs + type: string + default: '' - name: envFileName type: string default: '.env.ci' + - name: applicationType + type: string + default: 'vue' + values: ['vue', 'dotnet'] + - name: readinessEndpoint + type: string + default: '/healthcheck' + - name: serverTimeoutSeconds + type: number + default: 60 - jobs: - job: playwright_tests continueOnError: true @@ -103,6 +115,14 @@ jobs: echo "Docker cleanup completed!" displayName: "Docker Cleanup" + - bash: | + printenv > "${{ parameters.envFileName }}" + if [ -z "$(GITHUB_TOKEN)" ]; then + echo "Printed env, but GITHUB_TOKEN was undefined" + fi + env: ${{ parameters.secrets }} + displayName: "Make Azure Pipeline Variables Available to Docker" + - task: Docker@2 displayName: 'Build Docker Image' inputs: @@ -110,20 +130,23 @@ jobs: dockerfile: ${{ parameters.dockerFileName }} repository: ${{ parameters.dockerImageName }} tags: ${{ parameters.imageTag }} - arguments: '--no-cache --pull' + arguments: | + --no-cache --pull ${{ parameters.dockerBuildArgs }} - script: | - printenv > "${{ parameters.envFileName }}" - + # Determine JIRA card number and filters JIRA_CARD_NUMBER="" IS_REGRESSION="${{ parameters.isRegression }}" - echo "isRegression: $IS_REGRESSION"; + echo "isRegression: $IS_REGRESSION" + if [[ "${{ parameters.isRegression }}" == "False" ]]; then - branch_name=$(System.PullRequest.SourceBranch) + branch_name="$(System.PullRequest.SourceBranch)" echo "Retrieved branch name: '$branch_name'" JIRA_CARD_NUMBER="${branch_name##*/}" echo "Extracted JIRA Card number: '$JIRA_CARD_NUMBER'" fi + + # Build test filter FILTER="" if [[ -n "${{ parameters.filterTags }}" && -n "$JIRA_CARD_NUMBER" ]]; then echo "Filtering by filterTags and Jira Card Number..." @@ -140,57 +163,213 @@ jobs: fi echo "Filter value: $FILTER" - TARGET_URL=${{ parameters.targetUrl }} - echo "BASE_URL value: $BASE_URL" + TARGET_URL="${{ parameters.targetUrl }}" + echo "Target URL: $TARGET_URL" + echo "Application Type: ${{ parameters.applicationType }}" - # Create container and run tests + # Set common variables + PLAYWRIGHT_PATH="${{ parameters.playwrightTestsPath }}" + SERVE_PATH="${{ parameters.npmServePath }}" + SHARD_NUMBER=$(shardNumber) + TOTAL_SHARDS=${{ parameters.totalShards }} + APPLICATION_TYPE="${{ parameters.applicationType }}" + ENV_FILE="${{ parameters.envFileName }}" + # Handle localhost URLs if [[ "$TARGET_URL" == *localhost* ]]; then - echo 'Npx version:' - npx --version - port="$TARGET_URL" - echo "URL: '$port'" - echo "URL '$port' contains 'localhost'. Extracting port... " - port=$(echo "$port" | sed -E 's/.*:([0-9]+).*/\1/') - echo "Port is '$port'" - TARGET_URL="$TARGET_URL" - JIRA_CARD_NUMBER="$JIRA_CARD_NUMBER" - FILTER="$FILTER" - PLAYWRIGHT_PATH="${{ parameters.playwrightTestsPath }}" - SERVE_PATH="${{ parameters.npmServePath }}" - SHARD_NUMBER=$(shardNumber) - TOTAL_SHARDS=${{ parameters.totalShards }} + echo "Localhost URL detected, extracting port..." + port=$(echo "$TARGET_URL" | sed -E 's/.*:([0-9]+).*/\1/') + echo "Port extracted: $port" + + # Create readiness check URL based on application type + if [[ "$APPLICATION_TYPE" == "dotnet" ]]; then + # For .NET APIs, append the readiness endpoint to the target URL + READINESS_URL="${TARGET_URL}${{ parameters.readinessEndpoint }}" + echo "Using .NET readiness check URL: $READINESS_URL" + else + # For Vue apps, use the target URL as-is + READINESS_URL="$TARGET_URL" + echo "Using Vue app URL for readiness check: $READINESS_URL" + fi + + echo "Starting tests for shard $SHARD_NUMBER of $TOTAL_SHARDS..." + container_id=$(docker create \ --ipc=host \ --env CI=true \ + --env TARGET_URL="$TARGET_URL" \ + --env READINESS_URL="$READINESS_URL" \ + --env PLAYWRIGHT_PATH="$PLAYWRIGHT_PATH" \ + --env SERVE_PATH="$SERVE_PATH" \ + --env SHARD_NUMBER="$SHARD_NUMBER" \ + --env TOTAL_SHARDS="$TOTAL_SHARDS" \ + --env FILTER="$FILTER" \ + --env APPLICATION_TYPE="$APPLICATION_TYPE" \ + --env SERVER_TIMEOUT="${{ parameters.serverTimeoutSeconds }}" \ + --env ASPNETCORE_URLS="https://localhost:$port" \ + --env ENV_FILE="$ENV_FILE" \ --env-file "${{ parameters.envFileName }}" \ ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} \ - npx concurrently -k -n 'server,playwright' \ - "npm --prefix $SERVE_PATH run serve -- --port=$port" \ - "npx --prefix $PLAYWRIGHT_PATH wait-on $TARGET_URL && PLAYWRIGHT_BLOB_OUTPUT_DIR='/app/blob-report' npx --prefix $PLAYWRIGHT_PATH playwright test $PLAYWRIGHT_PATH --config=$PLAYWRIGHT_PATH/playwright.config.ts --shard=$SHARD_NUMBER/$TOTAL_SHARDS --reporter=list,blob $FILTER" \ + bash -c ' + echo "=== Starting Application Server ===" + echo "Application Type: $APPLICATION_TYPE" + echo "Target URL: $TARGET_URL" + echo "Readiness Check URL: $READINESS_URL" + echo "Server Timeout: $SERVER_TIMEOUT seconds" + + # Function to wait for server readiness + wait_for_server() { + local url=$1 + local max_attempts=$SERVER_TIMEOUT + local attempt=1 + + echo "Waiting for server at: $url" + + while [ $attempt -le $max_attempts ]; do + echo "Attempt $attempt/$max_attempts: Checking server health..." + + if [[ "$APPLICATION_TYPE" == "vue" ]]; then + # For Vue apps, use a simple HTTP check + if curl -s -f "$url" --max-time 10 > /dev/null 2>&1; then + echo "Vue server is ready!" + return 0 + fi + else + # For .NET APIs, use healthcheck endpoint + if curl -k -s -f "$url" --max-time 10 > /dev/null 2>&1; then + echo ".NET server is ready!" + return 0 + fi + fi + + if [ $attempt -eq $max_attempts ]; then + echo "Server failed to become ready after $max_attempts attempts" + return 1 + fi + + echo "Server not ready yet, waiting 2 seconds..." + sleep 2 + attempt=$((attempt + 1)) + done + } + + # Install curl if not available + if ! command -v curl &> /dev/null; then + echo "Installing curl..." + apk add --no-cache curl 2>/dev/null || apt-get update && apt-get install -y curl 2>/dev/null || true + fi + + # Start server and run tests based on application type + if [[ "$APPLICATION_TYPE" == "vue" ]]; then + echo "=== Using Vue Mode with wait-on and concurrently ===" + + # Extract port for Vue server + port=$(echo "$TARGET_URL" | sed -E "s/.*:([0-9]+).*/\1/") + echo "Starting Vue server on port: $port" + + # Create blob report directory + mkdir -p /app/blob-report + + # Use concurrently for Vue apps (original approach) + if [ -n "$FILTER" ]; then + echo "Running Vue tests with filter: $FILTER" + npx concurrently -k -n "server,playwright" \ + "npm --prefix $SERVE_PATH run serve -- --port=$port" \ + "npx --prefix $PLAYWRIGHT_PATH wait-on $TARGET_URL && cd $PLAYWRIGHT_PATH && PLAYWRIGHT_BLOB_OUTPUT_DIR=\"/app/blob-report\" npx dotenv-cli -e \"../$ENV_FILE\" -- playwright test --config=./playwright.config.ts --shard=$SHARD_NUMBER/$TOTAL_SHARDS --reporter=list,blob $FILTER" + else + echo "Running all Vue tests" + npx concurrently -k -n "server,playwright" \ + "npm --prefix $SERVE_PATH run serve -- --port=$port" \ + "npx --prefix $PLAYWRIGHT_PATH wait-on $TARGET_URL && cd $PLAYWRIGHT_PATH && PLAYWRIGHT_BLOB_OUTPUT_DIR=\"/app/blob-report\" npx dotenv-cli -e \"../$ENV_FILE\" -- playwright test --config=./playwright.config.ts --shard=$SHARD_NUMBER/$TOTAL_SHARDS --reporter=list,blob" + fi + + else + echo "=== Using .NET Mode with healthcheck ===" + echo "Starting .NET server..." + echo "ASPNETCORE_URLS: $ASPNETCORE_URLS" + + npm --prefix "$SERVE_PATH" run serve & + SERVER_PID=$! + echo ".NET server started with PID: $SERVER_PID" + + # Wait for .NET server readiness check + if ! wait_for_server "$READINESS_URL"; then + kill $SERVER_PID 2>/dev/null || true + exit 1 + fi + + echo "=== Starting Playwright Tests ===" + + # Create blob report directory + mkdir -p /app/blob-report + cd "$PLAYWRIGHT_PATH" + + # Run Playwright tests + if [ -n "$FILTER" ]; then + echo "Running .NET tests with filter: $FILTER" + PLAYWRIGHT_BLOB_OUTPUT_DIR="/app/blob-report" npx dotenv-cli -e "../$ENV_FILE" -- playwright test \ + --config="./playwright.config.ts" \ + --shard="$SHARD_NUMBER/$TOTAL_SHARDS" \ + --reporter=list,blob \ + $FILTER || true + else + echo "Running all .NET tests" + PLAYWRIGHT_BLOB_OUTPUT_DIR="/app/blob-report" npx dotenv-cli -e "../$ENV_FILE" -- playwright test \ + --config="./playwright.config.ts" \ + --shard="$SHARD_NUMBER/$TOTAL_SHARDS" \ + --reporter=list,blob || true + fi + + TEST_EXIT_CODE=$? + echo "Tests completed with exit code: $TEST_EXIT_CODE" + + # Clean up server + echo "Stopping server..." + kill $SERVER_PID 2>/dev/null || true + + exit $TEST_EXIT_CODE + fi + ' ) else - export TARGET_URL="$TARGET_URL" - export JIRA_CARD_NUMBER="$JIRA_CARD_NUMBER" - export FILTER="$FILTER" - export PLAYWRIGHT_PATH="${{ parameters.playwrightTestsPath }}" - export SERVE_PATH="${{ parameters.npmServePath }}" - export SHARD_NUMBER=$(shardNumber) - export TOTAL_SHARDS=${{ parameters.totalShards }} - export ENV_FILE="${{ parameters.envFileName }}" + # Remote URL testing + echo "Remote URL testing mode" container_id=$(docker create \ --ipc=host \ --env CI=true \ + --env TARGET_URL="$TARGET_URL" \ + --env PLAYWRIGHT_PATH="$PLAYWRIGHT_PATH" \ + --env SHARD_NUMBER="$SHARD_NUMBER" \ + --env TOTAL_SHARDS="$TOTAL_SHARDS" \ + --env FILTER="$FILTER" \ + --env ENV_FILE="$ENV_FILE" \ --env-file "${{ parameters.envFileName }}" \ ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} \ bash -c " - set -a - source $ENV_FILE - set +a - PLAYWRIGHT_BLOB_OUTPUT_DIR='/app/blob-report' npx --prefix $PLAYWRIGHT_PATH playwright test $PLAYWRIGHT_PATH --config=$PLAYWRIGHT_PATH/playwright.config.ts --shard=$SHARD_NUMBER/$TOTAL_SHARDS --reporter=list,blob $FILTER - " \ + echo '=== Running Tests Against Remote URL ===' + echo 'Target URL: $TARGET_URL' + + mkdir -p /app/blob-report + cd \$PLAYWRIGHT_PATH + + if [ -n \"\$FILTER\" ]; then + echo 'Running tests with filter: \$FILTER' + PLAYWRIGHT_BLOB_OUTPUT_DIR='/app/blob-report' npx dotenv-cli -e \"../\$ENV_FILE\" -- playwright test \ + --config=./playwright.config.ts \ + --shard=\$SHARD_NUMBER/\$TOTAL_SHARDS \ + --reporter=list,blob \ + \$FILTER || true + else + echo 'Running all tests' + PLAYWRIGHT_BLOB_OUTPUT_DIR='/app/blob-report' npx dotenv-cli -e \"../\$ENV_FILE\" -- playwright test \ + --config=./playwright.config.ts \ + --shard=\$SHARD_NUMBER/\$TOTAL_SHARDS \ + --reporter=list,blob || true + fi + " ) fi + # Start container and stream logs echo "Starting tests for shard $(shardNumber)..." docker start -a $container_id @@ -206,12 +385,6 @@ jobs: # Remove container echo "Cleaning up container..." docker rm $container_id - - # Check if tests failed - if [ $? -ne 0 ]; then - echo "Tests failed in shard $(shardNumber) or tests don't exist for this shard number!" - exit 0 # Suppress error. It will be visible in report. - fi displayName: 'Run Playwright Tests - Shard $(shardNumber)' env: ${{ parameters.secrets }} @@ -227,6 +400,7 @@ jobs: docker rmi ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} -f displayName: 'Cleanup Docker Image' condition: always() + - job: download_and_merge_reports dependsOn: playwright_tests timeoutInMinutes: 8 @@ -238,6 +412,10 @@ jobs: - task: DownloadPipelineArtifact@2 inputs: targetPath: '$(System.DefaultWorkingDirectory)/playwright-reports' + - bash: | + printenv > "${{ parameters.envFileName }}" + env: ${{ parameters.secrets }} + displayName: "Make Azure Pipeline Variables Available to Docker" - task: Docker@2 displayName: 'Build Docker Image' inputs: @@ -245,9 +423,9 @@ jobs: dockerfile: ${{ parameters.dockerFileName }} repository: ${{ parameters.dockerImageName }} tags: ${{ parameters.imageTag }} - arguments: '--no-cache --pull' + arguments: | + --no-cache --pull ${{ parameters.dockerBuildArgs }} - bash: | - printenv > "${{ parameters.envFileName }}" branch_name=$(System.PullRequest.SourceBranch) echo "Retrieved branch name: '$branch_name'" JIRA_CARD_NUMBER="${branch_name##*/}" @@ -263,14 +441,12 @@ jobs: --env-file "${{ parameters.envFileName }}" \ ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} \ bash -c " - set -a - source $ENV_FILE - set +a echo 'Moving Playwright reports out of subfolders...' find ./playwright-reports/ -mindepth 2 -type f -exec mv {} ./playwright-reports/ \; echo 'Value of JIRA_CARD_NUMBER in bash -c command: $JIRA_CARD_NUMBER' echo 'Merging reports...' - JIRA_CARD_NUMBER='$JIRA_CARD_NUMBER' PLAYWRIGHT_JUNIT_OUTPUT_DIR='/app/test-results' PLAYWRIGHT_JUNIT_OUTPUT_NAME='junit_results.xml' npx --prefix $PLAYWRIGHT_PATH playwright merge-reports --config=$PLAYWRIGHT_PATH/playwright.config.ts ./playwright-reports + cd $PLAYWRIGHT_PATH + JIRA_CARD_NUMBER='$JIRA_CARD_NUMBER' PLAYWRIGHT_JUNIT_OUTPUT_DIR='/app/test-results' PLAYWRIGHT_JUNIT_OUTPUT_NAME='junit_results.xml' npx dotenv-cli -e ../$ENV_FILE -- playwright merge-reports --config=./playwright.config.ts ../playwright-reports " )