diff --git a/azure-pipelines-automated-testing.yml b/azure-pipelines-automated-testing.yml
index af19f67b..a1503a3c 100644
--- a/azure-pipelines-automated-testing.yml
+++ b/azure-pipelines-automated-testing.yml
@@ -158,8 +158,17 @@ stages:
               -e JIRA_API_KEY=$(JIRA_API_KEY) \
               -e PLAYWRIGHT_JUNIT_OUTPUT_FILE=$(PLAYWRIGHT_JUNIT_OUTPUT_FILE) \
               $(dockerImageName):$(imageTag) \
-              bash -c "echo \"Contents of /app/playwright-reports/:\" && ls /app/playwright-reports/ && echo \"Moving subfolders of playwright-reports...\" &&
-              find ./playwright-reports/ -mindepth 2 -type f -exec mv {} ./playwright-reports/ \;; echo \"Current directory:\" && pwd && echo \"Contents of current dir: \" && ls && echo \"Contents of playwright-reports:\" && ls ./playwright-reports/; echo \"Merging reports...\" && npx playwright merge-reports --reporter=ortoni-report,junit ./playwright-reports; echo \"Contents of current dir: \" && ls && echo \"Contents of ortoni-report:\" && ls ./ortoni-report && ./devops/scripts/jira_writeback.sh add_comment \"$(Build.SourceBranchName)\" /app/ortoni-report/ortoni-report.html \"AUTOMATED TEST RUN: $(date)\"")
+              bash -c "echo \"Moving Playwright reports out of subfolders...\" &&
+                find ./playwright-reports/ -mindepth 2 -type f -exec mv {} ./playwright-reports/ \; &&
+                echo \"Merging reports...\" && 
+                PLAYWRIGHT_JUNIT_OUTPUT_FILE=$PLAYWRIGHT_JUNIT_OUTPUT_FILE npx playwright merge-reports --reporter=ortoni-report,junit ./playwright-reports
+                num_failures=$(sed -n 's/.*failures="\([^"]*\)".*/\1/p' $PLAYWRIGHT_JUNIT_OUTPUT_FILE | head -n 1)
+                echo \"Number of test failures: '$num_failures'\"
+                (($num_failures == 0)) && jira_message='Playwright tests passed! ✅' || jira_message=\"$num_failures Playwright tests failed. ❌\"
+                echo \"Contents of ortoni-report:\" && ls ./ortoni-report && 
+                echo \"Writing report to Jira card '$(Build.SourceBranchName)'...\" && 
+                ./devops/scripts/jira_writeback.sh add_comment \"$(Build.SourceBranchName)\" /app/ortoni-report/ortoni-report.html \"AUTOMATED TEST RUN: $(date)\" \"$jira_message\" ")
+
               # Start container and stream logs
               echo "Starting merge"
               docker start -a $container_id
@@ -171,6 +180,7 @@ stages:
               # Copy test results from container
               echo "Copying test results..."
               docker cp $container_id:/app/ortoni-report/. $(System.DefaultWorkingDirectory)/ortoni-report
+              docker cp $container_id:/app/junit-results.xml $(System.DefaultWorkingDirectory)
               # TODO: Update to include junit results if necessary
 
               # Remove container
@@ -178,7 +188,7 @@ stages:
               docker rm $container_id
           env:
             JIRA_API_KEY: $(JIRA_API_KEY)
-            PLAYWRIGHT_JUNIT_OUTPUT_FILE: "test-results/results.xml"
+            PLAYWRIGHT_JUNIT_OUTPUT_FILE: "test-results/junit_results.xml"
           displayName: merge_and_publish_results_to_jira
 
           # TODO: Enable later when working on PR gate
@@ -187,7 +197,7 @@ stages:
           #   inputs:
           #     searchFolder: 'test-results'
           #     testResultsFormat: 'JUnit'
-          #     testResultsFiles: 'results.xml'
+          #     testResultsFiles: 'junit_results.xml'
           #     mergeTestResults: true
           #     failTaskOnFailedTests: false
           #     testRunTitle: 'Playwright Tests'