parameters: - name: totalShards type: number default: 2 values: [1, 2, 4, 8] - name: targetUrl type: string - name: dockerFileName type: string default: 'Dockerfile.playwright' - name: isRegression type: boolean default: false - name: filterTags type: string default: '@smoke' - name: playwrightTestsPath type: string default: 'playwright-tests' - name: npmServePath type: string default: '.' - name: npmrcPath type: string default: 'playwright-tests/.npmrc' - name: secrets type: object default: {} - name: dockerImageName type: string default: 'playwright-tests' - name: imageTag type: string default: '$(Build.BuildId)' - name: envFileName type: string default: '.env.ci' jobs: - job: playwright_tests continueOnError: true strategy: matrix: ${{ if eq(parameters.totalShards, 1) }}: shard1: shardNumber: 1 ${{ if eq(parameters.totalShards, 2) }}: shard1: shardNumber: 1 shard2: shardNumber: 2 ${{ if eq(parameters.totalShards, 4) }}: shard1: shardNumber: 1 shard2: shardNumber: 2 shard3: shardNumber: 3 shard4: shardNumber: 4 ${{ if eq(parameters.totalShards, 8) }}: shard1: shardNumber: 1 shard2: shardNumber: 2 shard3: shardNumber: 3 shard4: shardNumber: 4 shard5: shardNumber: 5 shard6: shardNumber: 6 shard7: shardNumber: 7 shard8: shardNumber: 8 steps: - task: npmAuthenticate@0 inputs: workingFile: ${{ parameters.npmrcPath }} - script: | echo "Starting Docker cleanup to free space..." # Remove all stopped containers echo "Removing stopped containers..." docker container prune -f # Remove dangling images (untagged) echo "Removing dangling images..." docker image prune -f # Remove unused volumes echo "Removing unused volumes..." docker volume prune -f # Show disk usage after cleanup echo "Docker system usage after cleanup:" docker system df echo "Docker cleanup completed!" displayName: "Docker Cleanup" - task: Docker@2 displayName: 'Build Docker Image' inputs: command: build dockerfile: ${{ parameters.dockerFileName }} repository: ${{ parameters.dockerImageName }} tags: ${{ parameters.imageTag }} arguments: '--no-cache --pull' - script: | printenv > "${{ parameters.envFileName }}" JIRA_CARD_NUMBER="" IS_REGRESSION="${{ parameters.isRegression }}" echo "isRegression: $IS_REGRESSION"; if [[ "${{ parameters.isRegression }}" == "False" ]]; then branch_name=$(System.PullRequest.SourceBranch) echo "Retrieved branch name: '$branch_name'" JIRA_CARD_NUMBER="${branch_name##*/}" echo "Extracted JIRA Card number: '$JIRA_CARD_NUMBER'" fi FILTER="" if [[ -n "${{ parameters.filterTags }}" && -n "$JIRA_CARD_NUMBER" ]]; then echo "Filtering by filterTags and Jira Card Number..." FILTER="--grep '@$JIRA_CARD_NUMBER|${{ parameters.filterTags }}'" elif [[ -n "${{ parameters.filterTags }}" ]]; then echo "Filtering by filterTags only..." FILTER="--grep '${{ parameters.filterTags }}'" elif [[ -n "$JIRA_CARD_NUMBER" ]]; then echo "Filtering by Jira card number only..." FILTER="--grep '@$JIRA_CARD_NUMBER'" else echo "No filter. Running all test cases..." FILTER="" fi echo "Filter value: $FILTER" TARGET_URL=${{ parameters.targetUrl }} echo "BASE_URL value: $BASE_URL" # Create container and run tests if [[ "$TARGET_URL" == *localhost* ]]; then echo 'Npx version:' npx --version port="$TARGET_URL" echo "URL: '$port'" echo "URL '$port' contains 'localhost'. Extracting port... " port=$(echo "$port" | sed -E 's/.*:([0-9]+).*/\1/') echo "Port is '$port'" TARGET_URL="$TARGET_URL" JIRA_CARD_NUMBER="$JIRA_CARD_NUMBER" FILTER="$FILTER" PLAYWRIGHT_PATH="${{ parameters.playwrightTestsPath }}" SERVE_PATH="${{ parameters.npmServePath }}" SHARD_NUMBER=$(shardNumber) TOTAL_SHARDS=${{ parameters.totalShards }} container_id=$(docker create \ --ipc=host \ --env-file "${{ parameters.envFileName }}" \ ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} \ npx concurrently -k -n 'server,playwright' \ "npm --prefix $SERVE_PATH run serve -- --port=$port" \ "npx --prefix $PLAYWRIGHT_PATH wait-on $TARGET_URL && PLAYWRIGHT_BLOB_OUTPUT_DIR='/app/blob-report' npx --prefix $PLAYWRIGHT_PATH playwright test $PLAYWRIGHT_PATH --config=$PLAYWRIGHT_PATH/playwright.config.ts --shard=$SHARD_NUMBER/$TOTAL_SHARDS --reporter=list,blob $FILTER" \ ) else export TARGET_URL="$TARGET_URL" export JIRA_CARD_NUMBER="$JIRA_CARD_NUMBER" export FILTER="$FILTER" export PLAYWRIGHT_PATH="${{ parameters.playwrightTestsPath }}" export SERVE_PATH="${{ parameters.npmServePath }}" export SHARD_NUMBER=$(shardNumber) export TOTAL_SHARDS=${{ parameters.totalShards }} export ENV_FILE="${{ parameters.envFileName }}" container_id=$(docker create \ --ipc=host \ --env-file "${{ parameters.envFileName }}" \ ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} \ bash -c " set -a source $ENV_FILE set +a PLAYWRIGHT_BLOB_OUTPUT_DIR='/app/blob-report' npx --prefix $PLAYWRIGHT_PATH playwright test $PLAYWRIGHT_PATH --config=$PLAYWRIGHT_PATH/playwright.config.ts --shard=$SHARD_NUMBER/$TOTAL_SHARDS --reporter=list,blob $FILTER " \ ) fi # Start container and stream logs echo "Starting tests for shard $(shardNumber)..." docker start -a $container_id # Create directory for test results echo "Creating test results directory..." mkdir -p $(System.DefaultWorkingDirectory)/blob-reports/shard-$(shardNumber) # Copy test results from container echo "Copying test results..." docker cp $container_id:/app/blob-report/. $(System.DefaultWorkingDirectory)/blob-reports/shard-$(shardNumber)/ # Remove container echo "Cleaning up container..." docker rm $container_id # Check if tests failed if [ $? -ne 0 ]; then echo "Tests failed in shard $(shardNumber) or tests don't exist for this shard number!" exit 0 # Suppress error. It will be visible in report. fi displayName: 'Run Playwright Tests - Shard $(shardNumber)' env: ${{ parameters.secrets }} - task: PublishPipelineArtifact@1 displayName: 'Publish Test Reports - Shard $(shardNumber)' condition: always() inputs: targetPath: '$(System.DefaultWorkingDirectory)/blob-reports/shard-$(shardNumber)' artifact: 'playwright-report-shard-$(shardNumber)' publishLocation: 'pipeline' - script: | docker rmi ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} -f displayName: 'Cleanup Docker Image' condition: always() - job: download_and_merge_reports dependsOn: playwright_tests timeoutInMinutes: 8 cancelTimeoutInMinutes: 10 steps: - task: npmAuthenticate@0 inputs: workingFile: ${{ parameters.npmrcPath }} - task: DownloadPipelineArtifact@2 inputs: targetPath: '$(System.DefaultWorkingDirectory)/playwright-reports' - task: Docker@2 displayName: 'Build Docker Image' inputs: command: build dockerfile: ${{ parameters.dockerFileName }} repository: ${{ parameters.dockerImageName }} tags: ${{ parameters.imageTag }} arguments: '--no-cache --pull' - bash: | printenv > "${{ parameters.envFileName }}" branch_name=$(System.PullRequest.SourceBranch) echo "Retrieved branch name: '$branch_name'" JIRA_CARD_NUMBER="${branch_name##*/}" echo "Extracted JIRA Card number: '$JIRA_CARD_NUMBER'" export PLAYWRIGHT_PATH="${{ parameters.playwrightTestsPath }}" export JIRA_CARD_NUMBER="$JIRA_CARD_NUMBER" export ENV_FILE="${{ parameters.envFileName }}" container_id=$(docker create \ --ipc=host \ --env-file "${{ parameters.envFileName }}" \ ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} \ bash -c " set -a source $ENV_FILE set +a echo 'Moving Playwright reports out of subfolders...' find ./playwright-reports/ -mindepth 2 -type f -exec mv {} ./playwright-reports/ \; echo 'Value of JIRA_CARD_NUMBER in bash -c command: $JIRA_CARD_NUMBER' echo 'Merging reports...' JIRA_CARD_NUMBER='$JIRA_CARD_NUMBER' PLAYWRIGHT_JUNIT_OUTPUT_DIR='/app/test-results' PLAYWRIGHT_JUNIT_OUTPUT_NAME='junit_results.xml' npx --prefix $PLAYWRIGHT_PATH playwright merge-reports --config=$PLAYWRIGHT_PATH/playwright.config.ts ./playwright-reports " ) # Start container and stream logs echo "Starting merge" docker start -a $container_id # Create directory for test results echo "Creating test results directory..." mkdir -p $(System.DefaultWorkingDirectory)/ortoni-report mkdir -p $(System.DefaultWorkingDirectory)/test-results # Copy test results from container echo "Copying test results..." docker cp $container_id:/app/ortoni-report/. $(System.DefaultWorkingDirectory)/ortoni-report docker cp $container_id:/app/test-results/junit_results.xml $(System.DefaultWorkingDirectory)/test-results # Remove container echo "Cleaning up container..." docker rm $container_id displayName: merge_and_publish_results_to_jira env: ${{ parameters.secrets }} - task: PublishTestResults@2 displayName: 'Publish test results' inputs: searchFolder: 'test-results' testResultsFormat: 'JUnit' testResultsFiles: 'junit_results.xml' mergeTestResults: true failTaskOnFailedTests: false testRunTitle: 'Playwright Tests' condition: succeededOrFailed() - task: PublishPipelineArtifact@1 displayName: 'Publish Merged Report' condition: always() inputs: targetPath: '$(System.DefaultWorkingDirectory)/ortoni-report' artifact: 'playwright-merged-report' publishLocation: 'pipeline' - script: | docker rmi ${{ parameters.dockerImageName }}:${{ parameters.imageTag }} -f displayName: 'Cleanup Docker Image' condition: always()