diff --git a/azure-pipelines-automated-testing.yml b/azure-pipelines-automated-testing.yml index 34cda2917..132da13ce 100644 --- a/azure-pipelines-automated-testing.yml +++ b/azure-pipelines-automated-testing.yml @@ -33,6 +33,7 @@ stages: # TODO: Change to ADO Playwright template - template: temp/playwright-test.yml parameters: + applicationType: 'vue' totalShards: ${{ variables.totalShards }} targetUrl: $(BASE_URL) dockerFileName: 'Dockerfile.playwright' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9774bd48a..39519804c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -64,6 +64,7 @@ stages: jobs: - template: temp/playwright-test.yml parameters: + applicationType: 'vue' totalShards: 2 targetUrl: $(BASE_URL) dockerFileName: 'Dockerfile.playwright' diff --git a/temp/playwright-test.yml b/temp/playwright-test.yml index 5c6692586..474a0bd75 100644 --- a/temp/playwright-test.yml +++ b/temp/playwright-test.yml @@ -32,11 +32,23 @@ parameters: - name: imageTag type: string default: '$(Build.BuildId)' + - name: dockerBuildArgs + type: string + default: '' - name: envFileName type: string default: '.env.ci' + - name: applicationType + type: string + default: 'vue' + values: ['vue', 'dotnet'] + - name: readinessEndpoint + type: string + default: '/healthcheck' + - name: serverTimeoutSeconds + type: number + default: 60 - jobs: - job: playwright_tests continueOnError: true @@ -103,6 +115,14 @@ jobs: echo "Docker cleanup completed!" displayName: "Docker Cleanup" + - bash: | + printenv > "${{ parameters.envFileName }}" + if [ -z "$(GITHUB_TOKEN)" ]; then + echo "Printed env, but GITHUB_TOKEN was undefined" + fi + env: ${{ parameters.secrets }} + displayName: "Make Azure Pipeline Variables Available to Docker" + - task: Docker@2 displayName: 'Build Docker Image' inputs: @@ -110,20 +130,23 @@ jobs: dockerfile: ${{ parameters.dockerFileName }} repository: ${{ parameters.dockerImageName }} tags: ${{ parameters.imageTag }} - arguments: '--no-cache --pull' + arguments: | + --no-cache --pull ${{ parameters.dockerBuildArgs }} - script: | - printenv > "${{ parameters.envFileName }}" - + # Determine JIRA card number and filters JIRA_CARD_NUMBER="" IS_REGRESSION="${{ parameters.isRegression }}" - echo "isRegression: $IS_REGRESSION"; + echo "isRegression: $IS_REGRESSION" + if [[ "${{ parameters.isRegression }}" == "False" ]]; then - branch_name=$(System.PullRequest.SourceBranch) + branch_name="$(System.PullRequest.SourceBranch)" echo "Retrieved branch name: '$branch_name'" JIRA_CARD_NUMBER="${branch_name##*/}" echo "Extracted JIRA Card number: '$JIRA_CARD_NUMBER'" fi + + # Build test filter FILTER="" if [[ -n "${{ parameters.filterTags }}" && -n "$JIRA_CARD_NUMBER" ]]; then echo "Filtering by filterTags and Jira Card Number..." @@ -140,57 +163,213 @@ jobs: fi echo "Filter value: $FILTER" - TARGET_URL=${{ parameters.targetUrl }} - echo "BASE_URL value: $BASE_URL" + TARGET_URL="${{ parameters.targetUrl }}" + echo "Target URL: $TARGET_URL" + echo "Application Type: ${{ parameters.applicationType }}" - # Create container and run tests + # Set common variables + PLAYWRIGHT_PATH="${{ parameters.playwrightTestsPath }}" + SERVE_PATH="${{ parameters.npmServePath }}" + SHARD_NUMBER=$(shardNumber) + TOTAL_SHARDS=${{ parameters.totalShards }} + APPLICATION_TYPE="${{ parameters.applicationType }}" + ENV_FILE="${{ parameters.envFileName }}" + # Handle localhost URLs if [[ "$TARGET_URL" == *localhost* ]]; then - echo 'Npx version:' - npx --version - port="$TARGET_URL" - echo "URL: '$port'" - echo "URL '$port' contains 'localhost'. Extracting port... " - port=$(echo "$port" | sed -E 's/.*:([0-9]+).*/\1/') - echo "Port is '$port'" - TARGET_URL="$TARGET_URL" - JIRA_CARD_NUMBER="$JIRA_CARD_NUMBER" - FILTER="$FILTER" - PLAYWRIGHT_PATH="${{ parameters.playwrightTestsPath }}" - SERVE_PATH="${{ parameters.npmServePath }}" - SHARD_NUMBER=$(shardNumber) - TOTAL_SHARDS=${{ parameters.totalShards }} + echo "Localhost URL detected, extracting port..." + port=$(echo "$TARGET_URL" | sed -E 's/.*:([0-9]+).*/\1/') + echo "Port extracted: $port" + + # Create readiness check URL based on application type + if [[ "$APPLICATION_TYPE" == "dotnet" ]]; then + # For .NET APIs, append the readiness endpoint to the target URL + READINESS_URL="${TARGET_URL}${{ parameters.readinessEndpoint }}" + echo "Using .NET readiness check URL: $READINESS_URL" + else + # For Vue apps, use the target URL as-is + READINESS_URL="$TARGET_URL" + echo "Using Vue app URL for readiness check: $READINESS_URL" + fi + + echo "Starting tests for shard $SHARD_NUMBER of $TOTAL_SHARDS..." + container_id=$(docker create \ --ipc=host \ --env CI=true \ + --env TARGET_URL="$TARGET_URL" \ + --env READINESS_URL="$READINESS_URL" \ + --env PLAYWRIGHT_PATH="$PLAYWRIGHT_PATH" \ + --env SERVE_PATH="$SERVE_PATH" \ + --env SHARD_NUMBER="$SHARD_NUMBER" \ + --env TOTAL_SHARDS="$TOTAL_SHARDS" \ + --env FILTER="$FILTER" \ + --env APPLICATION_TYPE="$APPLICATION_TYPE" \ + --env SERVER_TIMEOUT="${{ parameters.serverTimeoutSeconds }}" \ + --env ASPNETCORE_URLS="https://localhost:$port" \ + --env ENV_FILE="$ENV_FILE" \ --env-file "${{ parameters.envFileName }}" \ ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} \ - npx concurrently -k -n 'server,playwright' \ - "npm --prefix $SERVE_PATH run serve -- --port=$port" \ - "npx --prefix $PLAYWRIGHT_PATH wait-on $TARGET_URL && PLAYWRIGHT_BLOB_OUTPUT_DIR='/app/blob-report' npx --prefix $PLAYWRIGHT_PATH playwright test $PLAYWRIGHT_PATH --config=$PLAYWRIGHT_PATH/playwright.config.ts --shard=$SHARD_NUMBER/$TOTAL_SHARDS --reporter=list,blob $FILTER" \ + bash -c ' + echo "=== Starting Application Server ===" + echo "Application Type: $APPLICATION_TYPE" + echo "Target URL: $TARGET_URL" + echo "Readiness Check URL: $READINESS_URL" + echo "Server Timeout: $SERVER_TIMEOUT seconds" + + # Function to wait for server readiness + wait_for_server() { + local url=$1 + local max_attempts=$SERVER_TIMEOUT + local attempt=1 + + echo "Waiting for server at: $url" + + while [ $attempt -le $max_attempts ]; do + echo "Attempt $attempt/$max_attempts: Checking server health..." + + if [[ "$APPLICATION_TYPE" == "vue" ]]; then + # For Vue apps, use a simple HTTP check + if curl -s -f "$url" --max-time 10 > /dev/null 2>&1; then + echo "Vue server is ready!" + return 0 + fi + else + # For .NET APIs, use healthcheck endpoint + if curl -k -s -f "$url" --max-time 10 > /dev/null 2>&1; then + echo ".NET server is ready!" + return 0 + fi + fi + + if [ $attempt -eq $max_attempts ]; then + echo "Server failed to become ready after $max_attempts attempts" + return 1 + fi + + echo "Server not ready yet, waiting 2 seconds..." + sleep 2 + attempt=$((attempt + 1)) + done + } + + # Install curl if not available + if ! command -v curl &> /dev/null; then + echo "Installing curl..." + apk add --no-cache curl 2>/dev/null || apt-get update && apt-get install -y curl 2>/dev/null || true + fi + + # Start server and run tests based on application type + if [[ "$APPLICATION_TYPE" == "vue" ]]; then + echo "=== Using Vue Mode with wait-on and concurrently ===" + + # Extract port for Vue server + port=$(echo "$TARGET_URL" | sed -E "s/.*:([0-9]+).*/\1/") + echo "Starting Vue server on port: $port" + + # Create blob report directory + mkdir -p /app/blob-report + + # Use concurrently for Vue apps (original approach) + if [ -n "$FILTER" ]; then + echo "Running Vue tests with filter: $FILTER" + npx concurrently -k -n "server,playwright" \ + "npm --prefix $SERVE_PATH run serve -- --port=$port" \ + "npx --prefix $PLAYWRIGHT_PATH wait-on $TARGET_URL && cd $PLAYWRIGHT_PATH && PLAYWRIGHT_BLOB_OUTPUT_DIR=\"/app/blob-report\" npx dotenv-cli -e \"../$ENV_FILE\" -- playwright test --config=./playwright.config.ts --shard=$SHARD_NUMBER/$TOTAL_SHARDS --reporter=list,blob $FILTER" + else + echo "Running all Vue tests" + npx concurrently -k -n "server,playwright" \ + "npm --prefix $SERVE_PATH run serve -- --port=$port" \ + "npx --prefix $PLAYWRIGHT_PATH wait-on $TARGET_URL && cd $PLAYWRIGHT_PATH && PLAYWRIGHT_BLOB_OUTPUT_DIR=\"/app/blob-report\" npx dotenv-cli -e \"../$ENV_FILE\" -- playwright test --config=./playwright.config.ts --shard=$SHARD_NUMBER/$TOTAL_SHARDS --reporter=list,blob" + fi + + else + echo "=== Using .NET Mode with healthcheck ===" + echo "Starting .NET server..." + echo "ASPNETCORE_URLS: $ASPNETCORE_URLS" + + npm --prefix "$SERVE_PATH" run serve & + SERVER_PID=$! + echo ".NET server started with PID: $SERVER_PID" + + # Wait for .NET server readiness check + if ! wait_for_server "$READINESS_URL"; then + kill $SERVER_PID 2>/dev/null || true + exit 1 + fi + + echo "=== Starting Playwright Tests ===" + + # Create blob report directory + mkdir -p /app/blob-report + cd "$PLAYWRIGHT_PATH" + + # Run Playwright tests + if [ -n "$FILTER" ]; then + echo "Running .NET tests with filter: $FILTER" + PLAYWRIGHT_BLOB_OUTPUT_DIR="/app/blob-report" npx dotenv-cli -e "../$ENV_FILE" -- playwright test \ + --config="./playwright.config.ts" \ + --shard="$SHARD_NUMBER/$TOTAL_SHARDS" \ + --reporter=list,blob \ + $FILTER || true + else + echo "Running all .NET tests" + PLAYWRIGHT_BLOB_OUTPUT_DIR="/app/blob-report" npx dotenv-cli -e "../$ENV_FILE" -- playwright test \ + --config="./playwright.config.ts" \ + --shard="$SHARD_NUMBER/$TOTAL_SHARDS" \ + --reporter=list,blob || true + fi + + TEST_EXIT_CODE=$? + echo "Tests completed with exit code: $TEST_EXIT_CODE" + + # Clean up server + echo "Stopping server..." + kill $SERVER_PID 2>/dev/null || true + + exit $TEST_EXIT_CODE + fi + ' ) else - export TARGET_URL="$TARGET_URL" - export JIRA_CARD_NUMBER="$JIRA_CARD_NUMBER" - export FILTER="$FILTER" - export PLAYWRIGHT_PATH="${{ parameters.playwrightTestsPath }}" - export SERVE_PATH="${{ parameters.npmServePath }}" - export SHARD_NUMBER=$(shardNumber) - export TOTAL_SHARDS=${{ parameters.totalShards }} - export ENV_FILE="${{ parameters.envFileName }}" + # Remote URL testing + echo "Remote URL testing mode" container_id=$(docker create \ --ipc=host \ --env CI=true \ + --env TARGET_URL="$TARGET_URL" \ + --env PLAYWRIGHT_PATH="$PLAYWRIGHT_PATH" \ + --env SHARD_NUMBER="$SHARD_NUMBER" \ + --env TOTAL_SHARDS="$TOTAL_SHARDS" \ + --env FILTER="$FILTER" \ + --env ENV_FILE="$ENV_FILE" \ --env-file "${{ parameters.envFileName }}" \ ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} \ bash -c " - set -a - source $ENV_FILE - set +a - PLAYWRIGHT_BLOB_OUTPUT_DIR='/app/blob-report' npx --prefix $PLAYWRIGHT_PATH playwright test $PLAYWRIGHT_PATH --config=$PLAYWRIGHT_PATH/playwright.config.ts --shard=$SHARD_NUMBER/$TOTAL_SHARDS --reporter=list,blob $FILTER - " \ + echo '=== Running Tests Against Remote URL ===' + echo 'Target URL: $TARGET_URL' + + mkdir -p /app/blob-report + cd \$PLAYWRIGHT_PATH + + if [ -n \"\$FILTER\" ]; then + echo 'Running tests with filter: \$FILTER' + PLAYWRIGHT_BLOB_OUTPUT_DIR='/app/blob-report' npx dotenv-cli -e \"../\$ENV_FILE\" -- playwright test \ + --config=./playwright.config.ts \ + --shard=\$SHARD_NUMBER/\$TOTAL_SHARDS \ + --reporter=list,blob \ + \$FILTER || true + else + echo 'Running all tests' + PLAYWRIGHT_BLOB_OUTPUT_DIR='/app/blob-report' npx dotenv-cli -e \"../\$ENV_FILE\" -- playwright test \ + --config=./playwright.config.ts \ + --shard=\$SHARD_NUMBER/\$TOTAL_SHARDS \ + --reporter=list,blob || true + fi + " ) fi + # Start container and stream logs echo "Starting tests for shard $(shardNumber)..." docker start -a $container_id @@ -206,12 +385,6 @@ jobs: # Remove container echo "Cleaning up container..." docker rm $container_id - - # Check if tests failed - if [ $? -ne 0 ]; then - echo "Tests failed in shard $(shardNumber) or tests don't exist for this shard number!" - exit 0 # Suppress error. It will be visible in report. - fi displayName: 'Run Playwright Tests - Shard $(shardNumber)' env: ${{ parameters.secrets }} @@ -227,6 +400,7 @@ jobs: docker rmi ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} -f displayName: 'Cleanup Docker Image' condition: always() + - job: download_and_merge_reports dependsOn: playwright_tests timeoutInMinutes: 8 @@ -238,6 +412,10 @@ jobs: - task: DownloadPipelineArtifact@2 inputs: targetPath: '$(System.DefaultWorkingDirectory)/playwright-reports' + - bash: | + printenv > "${{ parameters.envFileName }}" + env: ${{ parameters.secrets }} + displayName: "Make Azure Pipeline Variables Available to Docker" - task: Docker@2 displayName: 'Build Docker Image' inputs: @@ -245,9 +423,9 @@ jobs: dockerfile: ${{ parameters.dockerFileName }} repository: ${{ parameters.dockerImageName }} tags: ${{ parameters.imageTag }} - arguments: '--no-cache --pull' + arguments: | + --no-cache --pull ${{ parameters.dockerBuildArgs }} - bash: | - printenv > "${{ parameters.envFileName }}" branch_name=$(System.PullRequest.SourceBranch) echo "Retrieved branch name: '$branch_name'" JIRA_CARD_NUMBER="${branch_name##*/}" @@ -263,14 +441,12 @@ jobs: --env-file "${{ parameters.envFileName }}" \ ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} \ bash -c " - set -a - source $ENV_FILE - set +a echo 'Moving Playwright reports out of subfolders...' find ./playwright-reports/ -mindepth 2 -type f -exec mv {} ./playwright-reports/ \; echo 'Value of JIRA_CARD_NUMBER in bash -c command: $JIRA_CARD_NUMBER' echo 'Merging reports...' - JIRA_CARD_NUMBER='$JIRA_CARD_NUMBER' PLAYWRIGHT_JUNIT_OUTPUT_DIR='/app/test-results' PLAYWRIGHT_JUNIT_OUTPUT_NAME='junit_results.xml' npx --prefix $PLAYWRIGHT_PATH playwright merge-reports --config=$PLAYWRIGHT_PATH/playwright.config.ts ./playwright-reports + cd $PLAYWRIGHT_PATH + JIRA_CARD_NUMBER='$JIRA_CARD_NUMBER' PLAYWRIGHT_JUNIT_OUTPUT_DIR='/app/test-results' PLAYWRIGHT_JUNIT_OUTPUT_NAME='junit_results.xml' npx dotenv-cli -e ../$ENV_FILE -- playwright merge-reports --config=./playwright.config.ts ../playwright-reports " )