Jferg/hackweek replay accessibility issues

static/app/utils/replays/hooks/useCrumbHandlers.tsx

@@ -24,6 +24,29 @@ function useCrumbHandlers() {
 
  const onMouseEnter = useCallback(
  (frame: ReplayFrame) => {
+ if (frame.element) {
+ try {
+ const div = document.createElement('div');
+ div.innerHTML = frame.element!;
+ const element = div.firstChild;
+
+ const selector: string[] = [];
+ for (const attr of element!.attributes) {
+ selector.push(`[${attr.name}="${attr.value}"]`);
+ }
+
+ clearAllHighlights();
+ highlight({
+ selector: selector.join(''),
+ annotation: frame.id,
+ // spotlight: true,
+ });
+ } catch (error) {
+ // console.error(error);
+ }
+ return;
+ }
+
  // this debounces the mouseEnter callback in unison with mouseLeave
  // we ensure the pointer remains over the target element before dispatching state events in order to minimize unnecessary renders
  // this helps during scrolling or mouse move events which would otherwise fire in rapid succession slowing down our app

static/app/utils/replays/hooks/useReplayData.tsx

@@ -3,6 +3,7 @@ import * as Sentry from '@sentry/react';
 
 import {Client} from 'sentry/api';
 import parseLinkHeader, {ParsedHeader} from 'sentry/utils/parseLinkHeader';
+import {MOCK_A11Y_DATA} from 'sentry/utils/replays/hooks/mockA11yData';
 import {mapResponseToReplayRecord} from 'sentry/utils/replays/replayDataUtils';
 import RequestError from 'sentry/utils/requestError/requestError';
 import useApi from 'sentry/utils/useApi';
@@ -49,6 +50,7 @@ type Options = {
 };
 
 interface Result {
+ accessibilityFrames: unknown[];
  attachments: unknown[];
  errors: ReplayError[];
  fetchError: undefined | RequestError;
@@ -63,6 +65,7 @@ const INITIAL_STATE: State = Object.freeze({
  fetchingAttachments: true,
  fetchingErrors: true,
  fetchingReplay: true,
+ fetchingAccessibilityFrames: true,
 });
 
 /**
@@ -103,6 +106,7 @@ function useReplayData({
  const [attachments, setAttachments] = useState<unknown[]>([]);
  const attachmentMap = useRef<Map<string, unknown[]>>(new Map()); // Map keys are always iterated by insertion order
  const [errors, setErrors] = useState<ReplayError[]>([]);
+ const [accessibilityFrames, setAccessibilityFrames] = useState<unknown[]>([]);
  const [replayRecord, setReplayRecord] = useState<ReplayRecord>();
 
  const projectSlug = useMemo(() => {
@@ -184,6 +188,31 @@ function useReplayData({
  setState(prev => ({...prev, fetchingErrors: false}));
  }, [api, orgSlug, replayRecord, errorsPerPage]);
 
+ const fetchAccessibilityFrames = useCallback(async () => {
+ if (!attachments || attachments.length === 0) {
+ return;
+ }
+
+ if (MOCK_A11Y_DATA) {
+ setAccessibilityFrames(MOCK_A11Y_DATA);
+ } else {
+ const atts = JSON.stringify(attachments);
+ const response = await fetch('http://localhost:3000/events', {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ body: atts,
+ });
+
+ const body = await response.json();
+
+ setAccessibilityFrames(body);
+ }
+
+ setState(prev => ({...prev, fetchingAccessibilityFrames: false}));
+ }, [attachments]);
+
  const onError = useCallback(error => {
  Sentry.captureException(error);
  setState(prev => ({...prev, fetchError: error}));
@@ -212,9 +241,17 @@ function useReplayData({
  fetchAttachments().catch(onError);
  }, [state.fetchError, fetchAttachments, onError]);
 
+ useEffect(() => {
+ if (state.fetchError) {
+ return;
+ }
+ fetchAccessibilityFrames().catch(onError);
+ }, [state.fetchError, fetchAccessibilityFrames, onError]);
+
  return {
  attachments,
  errors,
+ accessibilityFrames,
  fetchError: state.fetchError,
  fetching: state.fetchingAttachments || state.fetchingErrors || state.fetchingReplay,
  onRetry: loadData,

static/app/utils/replays/hooks/useReplayReader.tsx

@@ -11,14 +11,15 @@ type Props = {
 export default function useReplayReader({orgSlug, replaySlug}: Props) {
  const replayId = parseReplayId(replaySlug);
 
- const {attachments, errors, replayRecord, ...replayData} = useReplayData({
- orgSlug,
- replayId,
- });
+ const {attachments, accessibilityFrames, errors, replayRecord, ...replayData} =
+ useReplayData({
+ orgSlug,
+ replayId,
+ });
 
  const replay = useMemo(
- () => ReplayReader.factory({attachments, errors, replayRecord}),
- [attachments, errors, replayRecord]
+ () => ReplayReader.factory({attachments, errors, replayRecord, accessibilityFrames}),
+ [attachments, errors, replayRecord, accessibilityFrames]
  );
 
  return {
@@ -28,6 +29,7 @@ export default function useReplayReader({orgSlug, replaySlug}: Props) {
  replay,
  replayId,
  replayRecord,
+ accessibilityFrames,
  };
 }
 

static/app/utils/replays/replayReader.tsx

@@ -33,6 +33,8 @@ import {
 import type {ReplayError, ReplayRecord} from 'sentry/views/replays/types';
 
 interface ReplayReaderParams {
+ accessibilityFrames: unknown;
+
  /**
  * Loaded segment data
  *
@@ -88,13 +90,18 @@ function removeDuplicateClicks(frames: BreadcrumbFrame[]) {
 }
 
 export default class ReplayReader {
- static factory({attachments, errors, replayRecord}: ReplayReaderParams) {
- if (!attachments || !replayRecord || !errors) {
+ static factory({
+ attachments,
+ errors,
+ replayRecord,
+ accessibilityFrames,
+ }: ReplayReaderParams) {
+ if (!attachments || !replayRecord || !errors || !accessibilityFrames) {
  return null;
  }
 
  try {
- return new ReplayReader({attachments, errors, replayRecord});
+ return new ReplayReader({attachments, errors, replayRecord, accessibilityFrames});
  } catch (err) {
  Sentry.captureException(err);
 
@@ -113,6 +120,7 @@ export default class ReplayReader {
  private constructor({
  attachments,
  errors,
+ accessibilityFrames,
  replayRecord,
  }: RequiredNotNull<ReplayReaderParams>) {
  this._cacheKey = domId('replayReader-');
@@ -161,6 +169,8 @@ export default class ReplayReader {
  this._sortedSpanFrames = hydrateSpans(replayRecord, spanFrames).sort(sortFrames);
  this._optionFrame = optionFrame;
 
+ this._accessibilityFrames = accessibilityFrames;
+
  // Insert extra records to satisfy minimum requirements for the UI
  this._sortedBreadcrumbFrames.push(replayInitBreadcrumb(replayRecord));
  this._sortedRRWebEvents.unshift(recordingStartFrame(replayRecord));
@@ -173,6 +183,7 @@ export default class ReplayReader {
  private _errors: ErrorFrame[];
  private _optionFrame: undefined | OptionFrame;
  private _replayRecord: ReplayRecord;
+ private _accessibilityFrames: unknown;
  private _sortedBreadcrumbFrames: BreadcrumbFrame[];
  private _sortedRRWebEvents: RecordingFrame[];
  private _sortedSpanFrames: SpanFrame[];
@@ -194,6 +205,8 @@ export default class ReplayReader {
 
  getErrorFrames = () => this._errors;
 
+ getAccessibilityFrames = () => this._accessibilityFrames;
+
  getConsoleFrames = memoize(() =>
  this._sortedBreadcrumbFrames.filter(
  frame =>

static/app/utils/replays/resourceFrame.tsx

@@ -40,3 +40,13 @@ export function getResponseBodySize(frame: SpanFrame) {
  }
  return undefined;
 }
+
+export function getFrameType(frame: SpanFrame) {
+ return frame.id ?? 'unknown';
+}
+export function getFrameImpact(frame: SpanFrame) {
+ return frame.impact ?? 'unknown';
+}
+export function getFramePath(frame: SpanFrame) {
+ return frame.element ?? 'unknown';
+}