parameters: - name: totalShards type: number default: 2 values: [1, 2, 4, 8] - name: targetUrl type: string - name: dockerFileName type: string default: 'Dockerfile.playwright' - name: isRegression type: boolean default: false - name: filterTags type: string default: '@smoke' - name: playwrightTestsPath type: string default: 'playwright-tests' - name: npmServePath type: string default: '.' - name: npmrcPath type: string default: 'playwright-tests/.npmrc' - name: secrets type: object default: {} - name: dockerImageName type: string default: 'playwright-tests' - name: imageTag type: string default: '$(Build.BuildId)' - name: dockerBuildArgs type: string default: '' - name: envFileName type: string default: '.env.ci' - name: applicationType type: string default: 'vue' values: ['vue', 'dotnet'] - name: readinessEndpoint type: string default: '/healthcheck' - name: serverTimeoutSeconds type: number default: 60 jobs: - job: playwright_tests continueOnError: true strategy: matrix: ${{ if eq(parameters.totalShards, 1) }}: shard1: shardNumber: 1 ${{ if eq(parameters.totalShards, 2) }}: shard1: shardNumber: 1 shard2: shardNumber: 2 ${{ if eq(parameters.totalShards, 4) }}: shard1: shardNumber: 1 shard2: shardNumber: 2 shard3: shardNumber: 3 shard4: shardNumber: 4 ${{ if eq(parameters.totalShards, 8) }}: shard1: shardNumber: 1 shard2: shardNumber: 2 shard3: shardNumber: 3 shard4: shardNumber: 4 shard5: shardNumber: 5 shard6: shardNumber: 6 shard7: shardNumber: 7 shard8: shardNumber: 8 steps: - task: npmAuthenticate@0 inputs: workingFile: ${{ parameters.npmrcPath }} - script: | echo "Starting Docker cleanup to free space..." # Remove all stopped containers echo "Removing stopped containers..." docker container prune -f # Remove dangling images (untagged) echo "Removing dangling images..." docker image prune -f # Remove unused volumes echo "Removing unused volumes..." docker volume prune -f # Show disk usage after cleanup echo "Docker system usage after cleanup:" docker system df echo "Docker cleanup completed!" displayName: "Docker Cleanup" - bash: | printenv > "${{ parameters.envFileName }}" if [ -z "$(GITHUB_TOKEN)" ]; then echo "Printed env, but GITHUB_TOKEN was undefined" fi env: ${{ parameters.secrets }} displayName: "Make Azure Pipeline Variables Available to Docker" - task: Docker@2 displayName: 'Build Docker Image' inputs: command: build dockerfile: ${{ parameters.dockerFileName }} repository: ${{ parameters.dockerImageName }} tags: ${{ parameters.imageTag }} arguments: | --no-cache --pull ${{ parameters.dockerBuildArgs }} - script: | # Determine JIRA card number and filters JIRA_CARD_NUMBER="" IS_REGRESSION="${{ parameters.isRegression }}" echo "isRegression: $IS_REGRESSION" if [[ "${{ parameters.isRegression }}" == "False" ]]; then branch_name="$(System.PullRequest.SourceBranch)" echo "Retrieved branch name: '$branch_name'" JIRA_CARD_NUMBER="${branch_name##*/}" echo "Extracted JIRA Card number: '$JIRA_CARD_NUMBER'" fi # Build test filter FILTER="" if [[ -n "${{ parameters.filterTags }}" && -n "$JIRA_CARD_NUMBER" ]]; then echo "Filtering by filterTags and Jira Card Number..." FILTER="--grep '@$JIRA_CARD_NUMBER|${{ parameters.filterTags }}'" elif [[ -n "${{ parameters.filterTags }}" ]]; then echo "Filtering by filterTags only..." FILTER="--grep '${{ parameters.filterTags }}'" elif [[ -n "$JIRA_CARD_NUMBER" ]]; then echo "Filtering by Jira card number only..." FILTER="--grep '@$JIRA_CARD_NUMBER'" else echo "No filter. Running all test cases..." FILTER="" fi echo "Filter value: $FILTER" TARGET_URL="${{ parameters.targetUrl }}" echo "Target URL: $TARGET_URL" echo "Application Type: ${{ parameters.applicationType }}" # Set common variables PLAYWRIGHT_PATH="${{ parameters.playwrightTestsPath }}" SERVE_PATH="${{ parameters.npmServePath }}" SHARD_NUMBER=$(shardNumber) TOTAL_SHARDS=${{ parameters.totalShards }} APPLICATION_TYPE="${{ parameters.applicationType }}" ENV_FILE="${{ parameters.envFileName }}" # Handle localhost URLs if [[ "$TARGET_URL" == *localhost* ]]; then echo "Localhost URL detected, extracting port..." port=$(echo "$TARGET_URL" | sed -E 's/.*:([0-9]+).*/\1/') echo "Port extracted: $port" # Create readiness check URL based on application type if [[ "$APPLICATION_TYPE" == "dotnet" ]]; then # For .NET APIs, append the readiness endpoint to the target URL READINESS_URL="${TARGET_URL}${{ parameters.readinessEndpoint }}" echo "Using .NET readiness check URL: $READINESS_URL" else # For Vue apps, use the target URL as-is READINESS_URL="$TARGET_URL" echo "Using Vue app URL for readiness check: $READINESS_URL" fi echo "Starting tests for shard $SHARD_NUMBER of $TOTAL_SHARDS..." container_id=$(docker create \ --ipc=host \ --env CI=true \ --env TARGET_URL="$TARGET_URL" \ --env READINESS_URL="$READINESS_URL" \ --env PLAYWRIGHT_PATH="$PLAYWRIGHT_PATH" \ --env SERVE_PATH="$SERVE_PATH" \ --env SHARD_NUMBER="$SHARD_NUMBER" \ --env TOTAL_SHARDS="$TOTAL_SHARDS" \ --env FILTER="$FILTER" \ --env APPLICATION_TYPE="$APPLICATION_TYPE" \ --env SERVER_TIMEOUT="${{ parameters.serverTimeoutSeconds }}" \ --env ASPNETCORE_URLS="https://localhost:$port" \ --env ENV_FILE="$ENV_FILE" \ --env-file "${{ parameters.envFileName }}" \ ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} \ bash -c ' echo "=== Starting Application Server ===" echo "Application Type: $APPLICATION_TYPE" echo "Target URL: $TARGET_URL" echo "Readiness Check URL: $READINESS_URL" echo "Server Timeout: $SERVER_TIMEOUT seconds" # Function to wait for server readiness wait_for_server() { local url=$1 local max_attempts=$SERVER_TIMEOUT local attempt=1 echo "Waiting for server at: $url" while [ $attempt -le $max_attempts ]; do echo "Attempt $attempt/$max_attempts: Checking server health..." if [[ "$APPLICATION_TYPE" == "vue" ]]; then # For Vue apps, use a simple HTTP check if curl -s -f "$url" --max-time 10 > /dev/null 2>&1; then echo "Vue server is ready!" return 0 fi else # For .NET APIs, use healthcheck endpoint if curl -k -s -f "$url" --max-time 10 > /dev/null 2>&1; then echo ".NET server is ready!" return 0 fi fi if [ $attempt -eq $max_attempts ]; then echo "Server failed to become ready after $max_attempts attempts" return 1 fi echo "Server not ready yet, waiting 2 seconds..." sleep 2 attempt=$((attempt + 1)) done } # Install curl if not available if ! command -v curl &> /dev/null; then echo "Installing curl..." apk add --no-cache curl 2>/dev/null || apt-get update && apt-get install -y curl 2>/dev/null || true fi # Start server and run tests based on application type if [[ "$APPLICATION_TYPE" == "vue" ]]; then echo "=== Using Vue Mode with wait-on and concurrently ===" # Extract port for Vue server port=$(echo "$TARGET_URL" | sed -E "s/.*:([0-9]+).*/\1/") echo "Starting Vue server on port: $port" # Create blob report directory mkdir -p /app/blob-report # Use concurrently for Vue apps (original approach) if [ -n "$FILTER" ]; then echo "Running Vue tests with filter: $FILTER" npx concurrently -k -n "server,playwright" \ "npm --prefix $SERVE_PATH run serve -- --port=$port" \ "npx --prefix $PLAYWRIGHT_PATH wait-on $TARGET_URL && cd $PLAYWRIGHT_PATH && PLAYWRIGHT_BLOB_OUTPUT_DIR=\"/app/blob-report\" npx dotenv-cli -e \"../$ENV_FILE\" -- playwright test --config=./playwright.config.ts --shard=$SHARD_NUMBER/$TOTAL_SHARDS --reporter=list,blob $FILTER" else echo "Running all Vue tests" npx concurrently -k -n "server,playwright" \ "npm --prefix $SERVE_PATH run serve -- --port=$port" \ "npx --prefix $PLAYWRIGHT_PATH wait-on $TARGET_URL && cd $PLAYWRIGHT_PATH && PLAYWRIGHT_BLOB_OUTPUT_DIR=\"/app/blob-report\" npx dotenv-cli -e \"../$ENV_FILE\" -- playwright test --config=./playwright.config.ts --shard=$SHARD_NUMBER/$TOTAL_SHARDS --reporter=list,blob" fi else echo "=== Using .NET Mode with healthcheck ===" echo "Starting .NET server..." echo "ASPNETCORE_URLS: $ASPNETCORE_URLS" npm --prefix "$SERVE_PATH" run serve & SERVER_PID=$! echo ".NET server started with PID: $SERVER_PID" # Wait for .NET server readiness check if ! wait_for_server "$READINESS_URL"; then kill $SERVER_PID 2>/dev/null || true exit 1 fi echo "=== Starting Playwright Tests ===" # Create blob report directory mkdir -p /app/blob-report cd "$PLAYWRIGHT_PATH" # Run Playwright tests if [ -n "$FILTER" ]; then echo "Running .NET tests with filter: $FILTER" PLAYWRIGHT_BLOB_OUTPUT_DIR="/app/blob-report" npx dotenv-cli -e "../$ENV_FILE" -- playwright test \ --config="./playwright.config.ts" \ --shard="$SHARD_NUMBER/$TOTAL_SHARDS" \ --reporter=list,blob \ $FILTER || true else echo "Running all .NET tests" PLAYWRIGHT_BLOB_OUTPUT_DIR="/app/blob-report" npx dotenv-cli -e "../$ENV_FILE" -- playwright test \ --config="./playwright.config.ts" \ --shard="$SHARD_NUMBER/$TOTAL_SHARDS" \ --reporter=list,blob || true fi TEST_EXIT_CODE=$? echo "Tests completed with exit code: $TEST_EXIT_CODE" # Clean up server echo "Stopping server..." kill $SERVER_PID 2>/dev/null || true exit $TEST_EXIT_CODE fi ' ) else # Remote URL testing echo "Remote URL testing mode" container_id=$(docker create \ --ipc=host \ --env CI=true \ --env TARGET_URL="$TARGET_URL" \ --env PLAYWRIGHT_PATH="$PLAYWRIGHT_PATH" \ --env SHARD_NUMBER="$SHARD_NUMBER" \ --env TOTAL_SHARDS="$TOTAL_SHARDS" \ --env FILTER="$FILTER" \ --env ENV_FILE="$ENV_FILE" \ --env-file "${{ parameters.envFileName }}" \ ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} \ bash -c " echo '=== Running Tests Against Remote URL ===' echo 'Target URL: $TARGET_URL' mkdir -p /app/blob-report cd \$PLAYWRIGHT_PATH if [ -n \"\$FILTER\" ]; then echo 'Running tests with filter: \$FILTER' PLAYWRIGHT_BLOB_OUTPUT_DIR='/app/blob-report' npx dotenv-cli -e \"../\$ENV_FILE\" -- playwright test \ --config=./playwright.config.ts \ --shard=\$SHARD_NUMBER/\$TOTAL_SHARDS \ --reporter=list,blob \ \$FILTER || true else echo 'Running all tests' PLAYWRIGHT_BLOB_OUTPUT_DIR='/app/blob-report' npx dotenv-cli -e \"../\$ENV_FILE\" -- playwright test \ --config=./playwright.config.ts \ --shard=\$SHARD_NUMBER/\$TOTAL_SHARDS \ --reporter=list,blob || true fi " ) fi # Start container and stream logs echo "Starting tests for shard $(shardNumber)..." docker start -a $container_id # Create directory for test results echo "Creating test results directory..." mkdir -p $(System.DefaultWorkingDirectory)/blob-reports/shard-$(shardNumber) # Copy test results from container echo "Copying test results..." docker cp $container_id:/app/blob-report/. $(System.DefaultWorkingDirectory)/blob-reports/shard-$(shardNumber)/ # Remove container echo "Cleaning up container..." docker rm $container_id displayName: 'Run Playwright Tests - Shard $(shardNumber)' env: ${{ parameters.secrets }} - task: PublishPipelineArtifact@1 displayName: 'Publish Test Reports - Shard $(shardNumber)' condition: always() inputs: targetPath: '$(System.DefaultWorkingDirectory)/blob-reports/shard-$(shardNumber)' artifact: 'playwright-report-shard-$(shardNumber)' publishLocation: 'pipeline' - script: | docker rmi ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} -f displayName: 'Cleanup Docker Image' condition: always() - job: download_and_merge_reports dependsOn: playwright_tests timeoutInMinutes: 8 cancelTimeoutInMinutes: 10 steps: - task: npmAuthenticate@0 inputs: workingFile: ${{ parameters.npmrcPath }} - task: DownloadPipelineArtifact@2 inputs: targetPath: '$(System.DefaultWorkingDirectory)/playwright-reports' - bash: | printenv > "${{ parameters.envFileName }}" env: ${{ parameters.secrets }} displayName: "Make Azure Pipeline Variables Available to Docker" - task: Docker@2 displayName: 'Build Docker Image' inputs: command: build dockerfile: ${{ parameters.dockerFileName }} repository: ${{ parameters.dockerImageName }} tags: ${{ parameters.imageTag }} arguments: | --no-cache --pull ${{ parameters.dockerBuildArgs }} - bash: | branch_name=$(System.PullRequest.SourceBranch) echo "Retrieved branch name: '$branch_name'" JIRA_CARD_NUMBER="${branch_name##*/}" echo "Extracted JIRA Card number: '$JIRA_CARD_NUMBER'" export PLAYWRIGHT_PATH="${{ parameters.playwrightTestsPath }}" export JIRA_CARD_NUMBER="$JIRA_CARD_NUMBER" export ENV_FILE="${{ parameters.envFileName }}" container_id=$(docker create \ --ipc=host \ --env CI=true \ --env-file "${{ parameters.envFileName }}" \ ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} \ bash -c " echo 'Moving Playwright reports out of subfolders...' find ./playwright-reports/ -mindepth 2 -type f -exec mv {} ./playwright-reports/ \; echo 'Value of JIRA_CARD_NUMBER in bash -c command: $JIRA_CARD_NUMBER' echo 'Merging reports...' cd $PLAYWRIGHT_PATH JIRA_CARD_NUMBER='$JIRA_CARD_NUMBER' PLAYWRIGHT_JUNIT_OUTPUT_DIR='/app/test-results' PLAYWRIGHT_JUNIT_OUTPUT_NAME='junit_results.xml' npx dotenv-cli -e ../$ENV_FILE -- playwright merge-reports --config=./playwright.config.ts ../playwright-reports " ) # Start container and stream logs echo "Starting merge" docker start -a $container_id # Create directory for test results echo "Creating test results directory..." mkdir -p $(System.DefaultWorkingDirectory)/ortoni-report mkdir -p $(System.DefaultWorkingDirectory)/test-results # Copy test results from container echo "Copying test results..." docker cp $container_id:/app/ortoni-report/. $(System.DefaultWorkingDirectory)/ortoni-report docker cp $container_id:/app/test-results/junit_results.xml $(System.DefaultWorkingDirectory)/test-results # Remove container echo "Cleaning up container..." docker rm $container_id displayName: merge_and_publish_results_to_jira env: ${{ parameters.secrets }} - task: PublishTestResults@2 displayName: 'Publish test results' inputs: searchFolder: 'test-results' testResultsFormat: 'JUnit' testResultsFiles: 'junit_results.xml' mergeTestResults: true failTaskOnFailedTests: false testRunTitle: 'Playwright Tests' condition: succeededOrFailed() - task: PublishPipelineArtifact@1 displayName: 'Publish Merged Report' condition: always() inputs: targetPath: '$(System.DefaultWorkingDirectory)/ortoni-report' artifact: 'playwright-merged-report' publishLocation: 'pipeline' - script: | docker rmi ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} -f displayName: 'Cleanup Docker Image' condition: always()