diff --git a/Tests/XCTestHTMLReportTests/BaselineCaptureTests.swift b/Tests/XCTestHTMLReportTests/BaselineCaptureTests.swift new file mode 100644 index 00000000..aefd6916 --- /dev/null +++ b/Tests/XCTestHTMLReportTests/BaselineCaptureTests.swift @@ -0,0 +1,59 @@ +// +// BaselineCaptureTests.swift +// +// Writes renders of every fixture to $XCHR_BASELINE_DIR. +// Fixtures are regenerated on each `prepareTestResults.sh` run, so a golden +// file cannot be checked in; capture before a refactor and again after, +// against one fixture generation, then diff the two directories. +// +// Renders are captured verbatim. Identifiers are deterministic per structural +// path (#430), so an identifier that moved is a real finding about the +// refactor, not noise to normalize away. +// + +import XCTest +@testable import XCTestHTMLReportCore + +final class BaselineCaptureTests: XCTestCase { + func testCaptureRawRenders() throws { + guard let dir = ProcessInfo.processInfo.environment["XCHR_BASELINE_DIR"] else { + throw XCTSkip("Set XCHR_BASELINE_DIR to capture baseline renders") + } + try FileManager.default.createDirectory( + atPath: dir, withIntermediateDirectories: true + ) + + let resources = ["TestResults", "SanityResults", "RetryResults"] + for resource in resources { + // Deliberately not `continue`: a skipped fixture would produce a + // partial baseline, and Task 5a's `diff -r` reports two partial + // directories as identical. Fail here instead. + let url = try XCTUnwrap( + Bundle.testBundle.url(forResource: resource, withExtension: "xcresult"), + "Fixture \(resource).xcresult is missing — run ./prepareTestResults.sh" + ) + let html = Summary( + resultPaths: [url.path], + renderingMode: .linking, + downsizeImagesEnabled: false, + downsizeScaleFactor: 0.5 + ).generatedHtmlReport() + let path = "\(dir)/\(resource).html" + try html.write( + toFile: path, atomically: true, encoding: .utf8 + ) + let written = try XCTUnwrap( + FileManager.default.contents(atPath: path) + ) + XCTAssertFalse(written.isEmpty, "\(resource) captured an empty baseline") + } + + let captured = try FileManager.default + .contentsOfDirectory(atPath: dir) + .filter { $0.hasSuffix(".html") } + XCTAssertEqual( + Set(captured), Set(resources.map { "\($0).html" }), + "Baseline must contain exactly one file per fixture" + ) + } +} diff --git a/Tests/XCTestHTMLReportTests/ReportNormalizer.swift b/Tests/XCTestHTMLReportTests/ReportNormalizer.swift new file mode 100644 index 00000000..5e47cc28 --- /dev/null +++ b/Tests/XCTestHTMLReportTests/ReportNormalizer.swift @@ -0,0 +1,36 @@ +// +// ReportNormalizer.swift +// +// Report element identifiers are a digest of each element's structural path +// through the report (see `IdentifierPath`, #411/#430). Two renders on one +// backend therefore agree byte-for-byte and need no normalization. +// +// Two *backends* do not: the modern reader's tree omits the "All tests" and +// ".xctest" wrapper levels, so the same test case sits at a different +// path under each backend and digests differently. The cross-backend +// differential normalizes those digests away; nothing else does. +// + +import Foundation + +/// `IdentifierPath.identifier` is the first 128 bits of a SHA-256, lowercase hex. +/// Anchored with word boundaries so it cannot bite into a longer hex run. +private let identifierPattern: NSRegularExpression = { + guard let pattern = try? NSRegularExpression(pattern: "\\b[0-9a-f]{32}\\b") else { + preconditionFailure("The identifier pattern is a constant and must compile") + } + return pattern +}() + +/// Replaces every `IdentifierPath` digest with the literal `ID`. +/// +/// Only safe on reports rendered in `.linking` mode. Inline rendering embeds +/// base64 payloads, where a 32-character run drawn from `[0-9a-f]` is possible; +/// the differential renders `.linking` for this reason. +func normalizeIdentifiers(_ html: String) -> String { + identifierPattern.stringByReplacingMatches( + in: html, + range: NSRange(html.startIndex..., in: html), + withTemplate: "ID" + ) +} diff --git a/Tests/XCTestHTMLReportTests/ReproducibilityTests.swift b/Tests/XCTestHTMLReportTests/ReproducibilityTests.swift index 0a17a0ac..eae9e263 100644 --- a/Tests/XCTestHTMLReportTests/ReproducibilityTests.swift +++ b/Tests/XCTestHTMLReportTests/ReproducibilityTests.swift @@ -19,6 +19,10 @@ final class ReproducibilityTests: XCTestCase { Bundle.testBundle.url(forResource: "RetryResults", withExtension: "xcresult") } + private var sanityResultsUrl: URL? { + Bundle.testBundle.url(forResource: "SanityResults", withExtension: "xcresult") + } + /// How many times each bundle is rendered. Identifier drift shows up on /// every run, but ordering drift is sampled — a rendering that is stable /// four runs out of five needs more than one comparison to catch. @@ -154,6 +158,29 @@ final class ReproducibilityTests: XCTestCase { ) } } + + // MARK: - Cross-backend normalizer + + func testNormalizerReplacesIdentifiersAndNothingElse() { + let input = "id=3f9a1c07b25e48d1a6c3079e5b4d2f88 name=FirstSuite/testTwo()" + XCTAssertEqual(normalizeIdentifiers(input), "id=ID name=FirstSuite/testTwo()") + } + + /// The digests are the only thing separating two backends' markup, so a + /// normalizer that matched nothing would make the Task 12 differential + /// compare raw identifiers and fail on every run. + func testNormalizerActuallyMatchesARenderedIdentifier() throws { + // `renderReport(arguments:)` is #430's helper: it runs the CLI out of + // process and returns `Data`. There is no in-process `render(_:)`. + let html = try XCTUnwrap(String( + bytes: renderReport(arguments: [XCTUnwrap(sanityResultsUrl).path]), + encoding: .utf8 + )) + XCTAssertNotEqual( + normalizeIdentifiers(html), html, + "Expected at least one IdentifierPath digest in the rendered report" + ) + } } // MARK: - Helpers diff --git a/docs/superpowers/plans/2026-08-10-xcresulttool-legacy-migration.md b/docs/superpowers/plans/2026-08-10-xcresulttool-legacy-migration.md index 7ac2bf20..c68c5783 100644 --- a/docs/superpowers/plans/2026-08-10-xcresulttool-legacy-migration.md +++ b/docs/superpowers/plans/2026-08-10-xcresulttool-legacy-migration.md @@ -116,7 +116,7 @@ would hide exactly the identifier regressions #430 exists to catch. - Produces: `func normalizeIdentifiers(_ html: String) -> String` — replaces every `IdentifierPath` digest with the literal `ID`. Used by Task 12 only. -- [ ] **Step 0: Confirm the prerequisite** +- [x] **Step 0: Confirm the prerequisite** ```bash git log --oneline -1 -- Sources/XCTestHTMLReportCore/Classes/Helpers/IdentifierPath.swift @@ -127,9 +127,20 @@ Expected: the file exists, and the suite passes. If `IdentifierPath.swift` is absent you are on a tree without #430 — stop and rebase. Nothing downstream in this plan is trustworthy without it. -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** + +Append to the existing `ReproducibilityTests.swift` rather than creating it. +The second test renders `SanityResults`, and the shipped file only has URL +helpers for `TestResults` and `RetryResults` — add the missing one beside them +first: + +```swift + private var sanityResultsUrl: URL? { + Bundle.testBundle.url(forResource: "SanityResults", withExtension: "xcresult") + } +``` -Append to the existing `ReproducibilityTests.swift` rather than creating it: +Then the tests themselves: ```swift func testNormalizerReplacesIdentifiersAndNothingElse() { @@ -143,10 +154,12 @@ Append to the existing `ReproducibilityTests.swift` rather than creating it: func testNormalizerActuallyMatchesARenderedIdentifier() throws { // `renderReport(arguments:)` is #430's helper: it runs the CLI out of // process and returns `Data`. There is no in-process `render(_:)`. - let html = String( - decoding: try renderReport(arguments: [try XCTUnwrap(sanityResultsUrl).path]), - as: UTF8.self - ) + // `String(bytes:encoding:)` rather than `String(decoding:as:)` — the + // repo's SwiftLint config flags the non-failable conversion. + let html = try XCTUnwrap(String( + bytes: renderReport(arguments: [XCTUnwrap(sanityResultsUrl).path]), + encoding: .utf8 + )) XCTAssertNotEqual( normalizeIdentifiers(html), html, "Expected at least one IdentifierPath digest in the rendered report" @@ -154,7 +167,7 @@ Append to the existing `ReproducibilityTests.swift` rather than creating it: } ``` -- [ ] **Step 2: Run it to verify it fails** +- [x] **Step 2: Run it to verify it fails** ```bash swift test --filter ReproducibilityTests @@ -162,7 +175,7 @@ swift test --filter ReproducibilityTests Expected: FAIL — `cannot find 'normalizeIdentifiers' in scope`. -- [ ] **Step 3: Implement the normalizer** +- [x] **Step 3: Implement the normalizer** `Tests/XCTestHTMLReportTests/ReportNormalizer.swift`: @@ -182,11 +195,17 @@ Expected: FAIL — `cannot find 'normalizeIdentifiers' in scope`. import Foundation -// `IdentifierPath.identifier` is the first 128 bits of a SHA-256, lowercase hex. -// Anchored with word boundaries so it cannot bite into a longer hex run. -private let identifierPattern = try! NSRegularExpression( - pattern: "\\b[0-9a-f]{32}\\b" -) +/// `IdentifierPath.identifier` is the first 128 bits of a SHA-256, lowercase hex. +/// Anchored with word boundaries so it cannot bite into a longer hex run. +/// +/// Not `try!` — the pre-commit SwiftLint gate errors on `force_try`, so the +/// constant pattern unwraps through a precondition instead. +private let identifierPattern: NSRegularExpression = { + guard let pattern = try? NSRegularExpression(pattern: "\\b[0-9a-f]{32}\\b") else { + preconditionFailure("The identifier pattern is a constant and must compile") + } + return pattern +}() /// Replaces every `IdentifierPath` digest with the literal `ID`. /// @@ -202,7 +221,7 @@ func normalizeIdentifiers(_ html: String) -> String { } ``` -- [ ] **Step 4: Run tests to verify they pass** +- [x] **Step 4: Run tests to verify they pass** ```bash swift test --filter ReproducibilityTests @@ -212,7 +231,7 @@ Expected: PASS. The pre-existing #430 tests must stay green — if `testRenderingTheSameBundleTwiceProducesIdenticalBytes` starts failing, the normalizer has been wired into the same-backend path by mistake. -- [ ] **Step 5: Commit** +- [x] **Step 5: Commit** ```bash swiftformat . && git add Tests/XCTestHTMLReportTests/ReportNormalizer.swift \ @@ -244,7 +263,11 @@ report success anyway. The identifiers are signal here, not noise. skips otherwise. Used manually in Tasks 5a and 5b, then kept for any future rendering change. -- [ ] **Step 1: Write the capture test** +- [x] **Step 1: Write the capture test** + +(The test was originally named `testCaptureNormalizedRenders` — a leftover from +the pre-#430 revision that normalized. It captures raw bytes, and is named +`testCaptureRawRenders` accordingly.) ```swift // @@ -264,7 +287,7 @@ import XCTest @testable import XCTestHTMLReportCore final class BaselineCaptureTests: XCTestCase { - func testCaptureNormalizedRenders() throws { + func testCaptureRawRenders() throws { guard let dir = ProcessInfo.processInfo.environment["XCHR_BASELINE_DIR"] else { throw XCTSkip("Set XCHR_BASELINE_DIR to capture baseline renders") } @@ -308,7 +331,7 @@ final class BaselineCaptureTests: XCTestCase { } ``` -- [ ] **Step 2: Run it and confirm it skips by default** +- [x] **Step 2: Run it and confirm it skips by default** ```bash swift test --filter BaselineCaptureTests @@ -316,7 +339,7 @@ swift test --filter BaselineCaptureTests Expected: PASS with one skipped test. -- [ ] **Step 3: Capture the actual baseline** +- [x] **Step 3: Capture the actual baseline** ```bash XCHR_BASELINE_DIR=/tmp/xchr-baseline swift test --filter BaselineCaptureTests @@ -326,7 +349,7 @@ ls -la /tmp/xchr-baseline Expected: three non-empty `.html` files. **Do not regenerate fixtures again until Task 5a is verified** — a new fixture generation invalidates this baseline. -- [ ] **Step 4: Commit** +- [x] **Step 4: Commit** ```bash swiftformat . && git add Tests/XCTestHTMLReportTests/BaselineCaptureTests.swift