WIP - NOT READY [E2E] Prepare native preload ready-flow validation

e2e/README.md

@@ -52,7 +52,7 @@ enforces it.
 
 | Group | Tags | Rule |
 |---|---|---|
-| Journey | `launch`, `cart`, `checkout`, `account` | Exactly one per test |
+| Journey | `launch`, `cart`, `checkout`, `account`, `preload` | Exactly one per test |
 | Cost tier | `smoke`, `full` | Exactly one per test |
 | Quarantine | `flaky`, `wip` | Excluded by default, in `config.yaml` |
 | Platform capability | `ios-only`, `android-only` | Needs a `# Platform capability:` comment |

e2e/config/matrix.yml

@@ -45,6 +45,14 @@ applications:
     app_id: com.shopify.checkoutkit.androiddemo
     artifact_env: E2E_KOTLIN_ANDROID_APP_PATH
     ready_marker: checkout-kit-sample-ready
+    # The preload journey reads the native samples' PreloadState callbacks, which only the
+    # Swift and Kotlin samples surface, so the two native rows adopt it ahead of the others.
+    # An override replaces the default list rather than extending it, so it restates the
+    # defaults; dropping `checkout` here would silently retire that coverage on this row.
+    include_tags:
+      - launch
+      - checkout
+      - preload
     changed_files_filters:
       - android
       - protocolKotlin
@@ -57,6 +65,11 @@ applications:
     app_id: com.shopify.checkoutkit.swiftdemo
     artifact_env: E2E_SWIFT_IOS_APP_PATH
     ready_marker: checkout-kit-sample-ready
+    # See kotlin-android: the override restates the defaults and adds the preload journey.
+    include_tags:
+      - launch
+      - checkout
+      - preload
     changed_files_filters:
       - swift
       - protocolSwift

e2e/flows/checkout/assert-native-preload-ready.yaml

@@ -0,0 +1,35 @@
+appId: ${E2E_APP_ID}
+---
+- extendedWaitUntil:
+    visible:
+      id: preload-state-ready
+    timeout: 60000
+
+# Pins the starting value, so a marker stuck on "observed" cannot pass the assertion below.
+- runFlow:
+    when:
+      platform: iOS
+    commands:
+      - assertVisible:
+          id: preload-cache-hit-none
+
+- tapOn:
+    id: checkout-button
+
+- extendedWaitUntil:
+    visible: "^(Email( or mobile phone number)?|Delivery|Card number)$"
+    timeout: 60000
+
+# Android reads its cache-hit line from the BrowserStack device log after the run, because its
+# SDK log sink is internal and the sample never sees it. iOS installs its own logger, so the
+# sample watches for the diagnostic and republishes it as an identifier here.
+- runFlow:
+    when:
+      platform: iOS
+    commands:
+      - tapOn:
+          id: shopify_checkout_kit_close_button
+      - extendedWaitUntil:
+          visible:
+            id: preload-cache-hit-observed
+          timeout: 30000

e2e/lib/browserstack_client.rb

@@ -2,19 +2,24 @@
 
 require "net/http"
 require "securerandom"
+require "uri"
 require_relative "../../scripts/lib/json_http_client"
 
 # BrowserStack App Automate API client. Owns endpoint paths, HTTP, and response
 # parsing; callers assemble request payloads and orchestrate build lifecycles.
 class BrowserStackClient
   API_HOST = "api-cloud.browserstack.com"
+  ARTIFACT_HOSTS = [API_HOST, "api.browserstack.com"].freeze
   DASHBOARD_BASE = "https://app-automate.browserstack.com/dashboard/v2/builds"
 
   def self.build_url(build_id)
     build_id.to_s.empty? ? DASHBOARD_BASE : "#{DASHBOARD_BASE}/#{build_id}"
   end
 
   def initialize(username:, access_key:, retries: 0)
+    @username = username
+    @access_key = access_key
+    @retries = retries
     @client = JsonHttpClient.new(
       host: API_HOST,
       error_label: "BrowserStack",
@@ -65,4 +70,49 @@ def stop_build(build_id)
   def get_session(build_id, session_id)
     @client.get("/app-automate/maestro/v2/builds/#{build_id}/sessions/#{session_id}")
   end
+
+  def get_artifact_text(url, redirects_remaining: 3, retries_remaining: nil)
+    retries_remaining = @retries if retries_remaining.nil?
+    uri = URI.parse(url)
+    unless uri.scheme == "https" && ARTIFACT_HOSTS.include?(uri.host)
+      raise "BrowserStack artifact URL has an unexpected origin"
+    end
+
+    request = Net::HTTP::Get.new(uri)
+    request.basic_auth(@username, @access_key)
+    begin
+      response = Net::HTTP.start(
+        uri.host,
+        uri.port,
+        use_ssl: true,
+        open_timeout: 10,
+        read_timeout: 120
+      ) { |http| http.request(request) }
+    rescue *JsonHttpClient::RETRYABLE_EXCEPTIONS
+      raise unless retries_remaining.positive?
+
+      sleep([@retries - retries_remaining + 1, JsonHttpClient::MAX_BACKOFF_SECONDS].min)
+      return get_artifact_text(url, redirects_remaining: redirects_remaining, retries_remaining: retries_remaining - 1)
+    end
+
+    return response.body.to_s if response.is_a?(Net::HTTPSuccess)
+
+    if response.is_a?(Net::HTTPRedirection) && redirects_remaining.positive?
+      location = response["location"]
+      raise "BrowserStack artifact redirect omitted Location" if location.to_s.empty?
+
+      return get_artifact_text(
+        URI.join(uri, location).to_s,
+        redirects_remaining: redirects_remaining - 1,
+        retries_remaining: retries_remaining
+      )
+    end
+
+    if retries_remaining.positive? && (response.code.to_i == 429 || response.code.to_i >= 500)
+      sleep([@retries - retries_remaining + 1, JsonHttpClient::MAX_BACKOFF_SECONDS].min)
+      return get_artifact_text(url, redirects_remaining: redirects_remaining, retries_remaining: retries_remaining - 1)
+    end
+
+    raise "BrowserStack artifact request failed #{response.code}"
+  end
 end

e2e/lib/e2e_github_reporter.rb

@@ -268,6 +268,11 @@ def failure_details(result)
     tests = result.fetch("failed_tests", [])
     if tests.empty?
       lines << "| — | #{status_icon(result)} | #{artifact_links(nil, result)} |"
+      unless blank?(result["error"])
+        error = result["error"].to_s.gsub(/\s+/, " ").gsub("`", "'")
+        lines << ""
+        lines << "> Diagnostic: `#{error}`"
+      end
     else
       tests.each do |testcase|
         lines << "| `#{testcase.fetch("name", "unknown")}` | ❌ | #{artifact_links(testcase, result)} |"

e2e/scripts/execute_browserstack_run

@@ -11,6 +11,12 @@ require_relative "../lib/browserstack_device_resolver"
 # build polling and normalized result persistence.
 class BrowserStackRunExecutor
   TERMINAL_STATUSES = %w[passed failed error timedout stopped done].freeze
+  PRELOAD_TAG = "preload"
+  PRELOAD_CACHE_HIT_LOGS = {
+    "kotlin" => "Returning cached preloaded WebView."
+  }.freeze
+  ANDROID_CONCURRENT_PRESENTATION_LOG = "Preloaded WebView is already presented; creating a new WebView."
+  BROWSERSTACK_LOG_ARTIFACT_KEYS = %w[device_log instrumentation_log maestro_log].freeze
 
   # BrowserStack builds the suite with its own Maestro. Without this parameter it picks a
   # default that is years old, and every iOS test that opens the control link fails on a
@@ -82,6 +88,77 @@ class BrowserStackRunExecutor
     value
   end
 
+  def self.build_request_body(run:, app_url:, test_suite_url:, device:, env: ENV)
+    {
+      app: app_url,
+      testSuite: test_suite_url,
+      project: env.fetch("E2E_BROWSERSTACK_PROJECT", "checkout-kit-e2e"),
+      maestroVersion: resolve_maestro_version(env),
+      buildTag: env.fetch("BITRISE_GIT_COMMIT", "local"),
+      customBuildName: run.fetch("id"),
+      devices: [device],
+      execute: [run.fetch("execute")],
+      deviceLogs: true,
+      tags: {
+        includeTags: run.fetch("include_tags"),
+        excludeTags: run.fetch("exclude_tags")
+      },
+      setEnvVariables: {
+        E2E_APP_ID: run.fetch("app_id"),
+        E2E_READY_MARKER: run.fetch("ready_marker"),
+        E2E_CONTROL_LINK: run.fetch("control_link")
+      }
+    }
+  end
+
+  def self.testcases_for(sessions)
+    sessions.flat_map do |session|
+      session.dig("testcases", "data").to_a.flat_map do |group|
+        group.fetch("testcases", [])
+      end
+    end
+  end
+
+  def self.preload_log_verification_run?(run)
+    PRELOAD_CACHE_HIT_LOGS.key?(run.fetch("target")) &&
+      Array(run.fetch("include_tags")).include?(PRELOAD_TAG)
+  end
+
+  def self.passing_log_urls(sessions)
+    testcases_for(sessions)
+      .select { |testcase| testcase.fetch("status", "") == "passed" }
+      .flat_map { |testcase| BROWSERSTACK_LOG_ARTIFACT_KEYS.filter_map { |key| testcase[key] } }
+      .reject { |url| url.to_s.empty? }
+  end
+
+  def self.sessions_have_preload_log_artifacts?(sessions)
+    passing_log_urls(sessions).any?
+  end
+
+  def self.verify_preload_cache_hit_log!(run:, sessions:, fetch_log:)
+    return unless preload_log_verification_run?(run)
+
+    expected = PRELOAD_CACHE_HIT_LOGS.fetch(run.fetch("target"))
+    passing_testcases = testcases_for(sessions).select do |testcase|
+      testcase.fetch("status", "") == "passed"
+    end
+    log_urls = passing_log_urls(sessions)
+    if log_urls.empty?
+      available_keys = passing_testcases.flat_map(&:keys).uniq.sort.join(", ")
+      raise "Passing BrowserStack testcase did not expose a supported log artifact; available keys: #{available_keys}"
+    end
+
+    device_output = log_urls.map { |url| fetch_log.call(url) }.join("\n")
+    if device_output.include?(ANDROID_CONCURRENT_PRESENTATION_LOG)
+      raise "Android BrowserStack logs recorded a concurrent fresh presentation"
+    end
+    unless device_output.include?(expected)
+      raise "BrowserStack log artifacts did not contain the #{run.fetch("target")} preload cache-hit signal"
+    end
+
+    puts "Verified #{run.fetch("target")} preload cache-hit signal in BrowserStack logs."
+  end
+
   def initialize(options)
     @options = options
     @client = BrowserStackClient.new(
@@ -102,6 +179,13 @@ class BrowserStackRunExecutor
     puts "BrowserStack build: #{BrowserStackClient.build_url(@build.fetch("build_id"))}"
     build_status = poll_build(@build.fetch("build_id"))
     sessions = fetch_sessions(build_status)
+    if build_status.fetch("status").to_s.downcase == "passed"
+      self.class.verify_preload_cache_hit_log!(
+        run: @run,
+        sessions: sessions,
+        fetch_log: ->(url) { @client.get_artifact_text(url) }
+      )
+    end
     result = normalize_result(@run, @device, @app, @suite, @build, build_status, sessions)
     write_json("result.json", result)
   rescue StandardError => error
@@ -138,25 +222,12 @@ class BrowserStackRunExecutor
   end
 
   def start_build(run, app_url, test_suite_url, device)
-    body = {
-      app: app_url,
-      testSuite: test_suite_url,
-      project: ENV.fetch("E2E_BROWSERSTACK_PROJECT", "checkout-kit-e2e"),
-      maestroVersion: self.class.resolve_maestro_version(ENV),
-      buildTag: ENV.fetch("BITRISE_GIT_COMMIT", "local"),
-      customBuildName: run.fetch("id"),
-      devices: [device],
-      execute: [run.fetch("execute")],
-      tags: {
-        includeTags: run.fetch("include_tags"),
-        excludeTags: run.fetch("exclude_tags")
-      },
-      setEnvVariables: {
-        E2E_APP_ID: run.fetch("app_id"),
-        E2E_READY_MARKER: run.fetch("ready_marker"),
-        E2E_CONTROL_LINK: run.fetch("control_link")
-      }
-    }
+    body = self.class.build_request_body(
+      run: run,
+      app_url: app_url,
+      test_suite_url: test_suite_url,
+      device: device
+    )
     response = @client.start_build(run.fetch("platform"), body)
     write_json("build-start.json", response)
     response
@@ -188,19 +259,31 @@ class BrowserStackRunExecutor
 
   def fetch_sessions(build_status)
     build_id = build_status.fetch("id")
-    build_status.fetch("devices", []).flat_map do |device|
-      device.fetch("sessions", []).map do |session|
-        @client.get_session(build_id, session.fetch("id"))
+    session_ids = build_status.fetch("devices", []).flat_map do |device|
+      device.fetch("sessions", []).map { |session| session.fetch("id") }
+    end
+    needs_testcase_artifacts =
+      build_status.fetch("status").to_s.downcase == "passed" &&
+      self.class.preload_log_verification_run?(@run)
+    deadline = Time.now + ENV.fetch("E2E_BROWSERSTACK_SESSION_DETAILS_TIMEOUT_SECONDS", "120").to_i
+
+    loop do
+      sessions = session_ids.map { |session_id| @client.get_session(build_id, session_id) }
+      return sessions unless needs_testcase_artifacts
+      return sessions if self.class.sessions_have_preload_log_artifacts?(sessions)
+
+      if Time.now >= deadline
+        raise "BrowserStack session details did not publish testcase artifacts before timeout"
       end
+
+      sleep ENV.fetch("E2E_BROWSERSTACK_SESSION_DETAILS_POLL_SECONDS", "5").to_i
     end
   end
 
   def normalize_result(run, device, app, suite, build, build_status, sessions)
     status = build_status.fetch("status").to_s.downcase
-    failed_tests = sessions.flat_map do |session|
-      session.dig("testcases", "data").to_a.flat_map do |group|
-        group.fetch("testcases", []).select { |testcase| testcase.fetch("status", "") != "passed" }
-      end
+    failed_tests = self.class.testcases_for(sessions).reject do |testcase|
+      testcase.fetch("status", "") == "passed"
     end
 
     {