Merge pull request #1953 from DataDog/ncreated/RUM-4079/migrate-sr-sn…

…apshot-tests

RUM-4079 chore: Migrate SR Snapshot Tests automation to GitLab

.github/PULL_REQUEST_TEMPLATE.md

@@ -10,6 +10,3 @@ A brief description of implementation details of this PR.
 - [ ] Feature or bugfix MUST have appropriate tests (unit, integration)
 - [ ] Make sure each commit and the PR mention the Issue number or JIRA reference
 - [ ] Add CHANGELOG entry for user facing changes
-
-### Custom CI job configuration (optional)
-- [ ] Run unit tests for Session Replay

.gitlab-ci.yml

@@ -128,6 +128,27 @@ UI Tests:
     - make clean repo-setup ENV=ci
     - make ui-test TEST_PLAN="$TEST_PLAN" OS="$OS" PLATFORM="$PLATFORM" DEVICE="$DEVICE"
 
+SR Snapshot Tests:
+  stage: ui-test
+  rules: 
+    - !reference [.test-pipeline-job, rules]
+    - !reference [.release-pipeline-job, rules]
+  variables:
+    XCODE: "15.4.0"
+    OS: "17.5"
+    PLATFORM: "iOS Simulator"
+    DEVICE: "iPhone 15"
+    ARTIFACTS_PATH: "artifacts"
+  artifacts:
+    paths:
+      - artifacts
+    expire_in: 1 week
+    when: on_failure
+  script:
+    - ./tools/runner-setup.sh --xcode "$XCODE" --iOS --os "$OS" --ssh # temporary, waiting for AMI
+    - make clean repo-setup ENV=ci
+    - make sr-snapshots-pull sr-snapshot-test OS="$OS" PLATFORM="$PLATFORM" DEVICE="$DEVICE" ARTIFACTS_PATH="$ARTIFACTS_PATH"
+
 Tools Tests:
   stage: test
   rules:

Makefile

@@ -3,6 +3,7 @@ all: env-check repo-setup templates
 		lint license-check \
 		test test-ios test-ios-all test-tvos test-tvos-all \
 		ui-test ui-test-all ui-test-podinstall \
+		sr-snapshot-test sr-snapshots-pull sr-snapshots-push sr-snapshot-tests-open \
 		tools-test \
 		smoke-test smoke-test-ios smoke-test-ios-all smoke-test-tvos smoke-test-tvos-all \
 		spm-build spm-build-ios spm-build-tvos spm-build-visionos spm-build-macos spm-build-watchos \
@@ -104,6 +105,14 @@ DEFAULT_TVOS_OS := latest
 DEFAULT_TVOS_PLATFORM := tvOS Simulator
 DEFAULT_TVOS_DEVICE := Apple TV
 
+# Test env for running SR snapshot tests in local:
+DEFAULT_SR_SNAPSHOT_TESTS_OS := 17.5
+DEFAULT_SR_SNAPSHOT_TESTS_PLATFORM := iOS Simulator
+DEFAULT_SR_SNAPSHOT_TESTS_DEVICE := iPhone 15
+
+# Default location for deploying artifacts
+DEFAULT_ARTIFACTS_PATH := artifacts
+
 # Run unit tests for specified SCHEME
 test:
 	@$(call require_param,SCHEME)
@@ -255,12 +264,6 @@ xcodeproj-session-replay:
 		@cd DatadogSessionReplay/ && swift package generate-xcodeproj
 		@echo "OK 👌"
 
-open-sr-snapshot-tests:
-		@echo "⚙️  Opening SRSnapshotTests with DD_TEST_UTILITIES_ENABLED ..."
-		@pgrep -q Xcode && killall Xcode && echo "- Xcode killed" || echo "- Xcode not running"
-		@sleep 0.5 && echo "- launching" # Sleep, otherwise, if Xcode was running it often fails with "procNotFound: no eligible process with specified descriptor"
-		@open --env DD_TEST_UTILITIES_ENABLED ./DatadogSessionReplay/SRSnapshotTests/SRSnapshotTests.xcworkspace
-
 templates:
 	@$(ECHO_TITLE) "make templates"
 	./tools/xcode-templates/install-xcode-templates.sh
@@ -296,19 +299,30 @@ sr-models-generate:
 sr-models-verify:
 	@$(MAKE) models-verify PRODUCT="sr"
 
-sr-push-snapshots:
-		@echo "🎬 ↗️  Pushing SR snapshots to remote repo..."
-		@cd tools/sr-snapshots && swift run sr-snapshots push \
-			--local-folder ../../DatadogSessionReplay/SRSnapshotTests/SRSnapshotTests/_snapshots_ \
-			--remote-folder ../../../dd-mobile-session-replay-snapshots \
-			--remote-branch "main"
-
-sr-pull-snapshots:
-		@echo "🎬 ↙️  Pulling SR snapshots from remote repo..."
-		@cd tools/sr-snapshots && swift run sr-snapshots pull \
-			--local-folder ../../DatadogSessionReplay/SRSnapshotTests/SRSnapshotTests/_snapshots_ \
-			--remote-folder ../../../dd-mobile-session-replay-snapshots \
-			--remote-branch "main"
+# Pushes current SR snapshots to snapshots repo
+sr-snapshots-push:
+	@$(ECHO_TITLE) "make sr-snapshots-push"
+	./tools/sr-snapshot-test.sh --push
+
+# Pulls SR snapshots from snapshots repo
+sr-snapshots-pull:
+	@$(ECHO_TITLE) "make sr-snapshots-pull"
+	./tools/sr-snapshot-test.sh --pull
+
+# Run Session Replay snapshot tests
+sr-snapshot-test:
+	@:$(eval OS ?= $(DEFAULT_SR_SNAPSHOT_TESTS_OS))
+	@:$(eval PLATFORM ?= $(DEFAULT_SR_SNAPSHOT_TESTS_PLATFORM))
+	@:$(eval DEVICE ?= $(DEFAULT_SR_SNAPSHOT_TESTS_DEVICE))
+	@:$(eval ARTIFACTS_PATH ?= $(DEFAULT_ARTIFACTS_PATH))
+	@$(ECHO_TITLE) "make sr-snapshot-test OS='$(OS)' PLATFORM='$(PLATFORM)' DEVICE='$(DEVICE)' ARTIFACTS_PATH='$(ARTIFACTS_PATH)'"
+	./tools/sr-snapshot-test.sh \
+		--test --os "$(OS)" --device "$(DEVICE)" --platform "$(PLATFORM)" --artifacts-path "$(ARTIFACTS_PATH)"
+
+# Opens `SRSnapshotTests` project with passing required ENV variables
+sr-snapshot-tests-open:
+	@$(ECHO_TITLE) "make sr-snapshot-tests-open"
+	./tools/sr-snapshot-test.sh --open-project
 
 # Generate api-surface files for Datadog and DatadogObjc.
 api-surface:

bitrise.yml

@@ -7,28 +7,16 @@ project_type: other
 
 workflows:
   push_to_pull_request:
-    description: |-
-        This workflow is triggered on starting new PR or pushing new changes to existing PRs.
-        By default, it doesn't run any test phases, but this behaviour is overwritten in `choose_workflows.py` when:
-        - one or more `DD_OVERWRITE_RUN_(phase)_TESTS` ENVs are passed to the current CI job:
-            - DD_OVERRIDE_RUN_SR_UNIT_TESTS='1' to run unit tests phase for Session Replay product
-        - a phase is selected on the checklist in the PR description,
-        - the PR changes a file which matches phase filter (e.g. changing a file in `Sources/*` will trigger unit tests phase)
-    envs:
-      - DD_RUN_SR_UNIT_TESTS: '0'
-    after_run:
-    - _make_dependencies
-    - run_conditioned_workflows
-    - _deploy_artifacts
+    steps:
+    - script:
+        inputs:
+        - content: echo "NOP"
 
   push_to_develop_or_master:
     description: |-
         This workflow is triggered for each new commit pushed to `develop` or `master` branch.
-    envs:
-      - DD_RUN_SR_UNIT_TESTS: '1'
     after_run:
     - _make_dependencies
-    - run_conditioned_workflows
     - run_e2e_s8s_upload
     - _deploy_artifacts
     - _notify_failure_on_slack
@@ -45,13 +33,10 @@ workflows:
     - _notify_failure_on_slack
 
   tagged_commit:
-    description: |-
-        This workflow is triggered on pushing a new release tag.
-    envs:
-      - DD_RUN_SR_UNIT_TESTS: '1'
-    after_run:
-    - _make_dependencies
-    - run_conditioned_workflows
+    steps:
+    - script:
+        inputs:
+        - content: echo "NOP"
 
   _make_dependencies:
     description: |-
@@ -92,59 +77,6 @@ workflows:
         - icon_url: 'https://avatars.githubusercontent.com/t/3555052?s=128&v=4'
         - webhook_url: '${SLACK_INCOMING_WEBHOOK_MOBILE_CI}'
 
-  run_conditioned_workflows:
-    steps:
-    - script:
-        title: Choose which workflows to run
-        inputs:
-        - content: |-
-            #!/usr/bin/env bash
-
-            cd tools/ci && make
-
-            # The `choose_workflows.py` inspects current ENV variables and Pull Request metadata (if running in PR)
-            # and decides on which from the workflows in `after_run` should be ran. Workflows are opted-in
-            # by modifying `DD_RUN_*` ENV variables with `envman` (ref.: https://github.com/bitrise-io/envman).
-            venv/bin/python3 choose_workflows.py
-    after_run:
-      - run_unit_tests
-
-  run_unit_tests:
-    description: |-
-        Selectively runs:
-        - or Session Replay tests when when 'DD_RUN_SR_UNIT_TESTS' is '1'
-    steps:
-    - script:
-        title: Pull Session Replay snapshots
-        run_if: '{{enveq "DD_RUN_SR_UNIT_TESTS" "1"}}'
-        inputs:
-        - content: |-
-            #!/usr/bin/env zsh
-            set -e
-            make sr-pull-snapshots
-    - script:
-        title: Configure SRSnapshotTests project launch
-        run_if: '{{enveq "DD_RUN_SR_UNIT_TESTS" "1"}}'
-        inputs:
-        - content: |
-            #!/usr/bin/env zsh
-
-            # The `SRSnapshotTests.xcworkspace` depends on `dd-sdk-ios/Package.swift` but requires the `dd-sdk-ios/TestUtilities` library, 
-            # which is not defined statically in the root package. To add it dynamically, we leverage the `DD_TEST_UTILITIES_ENABLED` ENV
-            # variable respected by the main package. Here we export it so it is available in next CI steps:
-            envman add --key DD_TEST_UTILITIES_ENABLED --value '1'
-    - xcode-test:
-        title: Run snapshot tests for Session Replay - iOS Simulator
-        run_if: '{{enveq "DD_RUN_SR_UNIT_TESTS" "1"}}'
-        inputs:
-        - scheme: SRSnapshotTests
-        - destination: platform=iOS Simulator,name=iPhone 15,OS=17.5
-        - should_build_before_test: 'no'
-        - is_clean_build: 'no'
-        - generate_code_coverage_files: 'yes'
-        - project_path: DatadogSessionReplay/SRSnapshotTests/SRSnapshotTests.xcworkspace
-        - xcpretty_test_options: --color --report html --output "${BITRISE_DEPLOY_DIR}/DatadogSessionReplay-snapshot-tests.html"
-
   create_dogfooding_pr:
     description: |-
         Creates PRs to repositories using `dd-sdk-ios`.