From fcb1812c547db8f9c8ccb42d5a3adc17154cd469 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 24 May 2026 01:52:23 -0400 Subject: [PATCH 1/2] feat: add absolute source paths and file links --- client/src/app/AppShell.tsx | 21 +- client/src/app/uiState.test.ts | 30 + client/src/app/uiState.ts | 12 +- .../accessibility/AccessibilityInspector.tsx | 55 +- .../AccessibilityOverlay.test.ts | 26 + .../accessibility/AccessibilityOverlay.tsx | 189 +++- .../accessibility/accessibilityTree.test.ts | 187 ++++ .../accessibility/accessibilityTree.ts | 101 +- client/src/styles/components.css | 12 +- packages/flutter-inspector/README.md | 7 +- .../lib/simdeck_flutter_inspector.dart | 57 +- packages/nativescript-inspector/README.md | 8 +- .../nativescript-inspector/package-lock.json | 4 +- packages/nativescript-inspector/package.json | 2 +- packages/nativescript-inspector/src/index.ts | 860 +++++++++++++++++- packages/react-native-inspector/README.md | 9 +- packages/react-native-inspector/src/auto.ts | 2 + packages/react-native-inspector/src/index.ts | 50 +- server/src/api/routes.rs | 35 +- 19 files changed, 1538 insertions(+), 129 deletions(-) create mode 100644 client/src/features/accessibility/AccessibilityOverlay.test.ts diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index 747bc52d..aac5b0f9 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -114,6 +114,7 @@ import { isMoveControlMessage } from "./controlMessages"; const ACCESSIBILITY_REFRESH_MS = 1500; const REACT_NATIVE_ACCESSIBILITY_REFRESH_MS = 500; const FLUTTER_ACCESSIBILITY_REFRESH_MS = 1000; +const ACCESSIBILITY_BACKGROUND_REFRESH_MS = 3000; const ANDROID_METADATA_REFRESH_MS = 1000; const DEFAULT_ACCESSIBILITY_MAX_DEPTH = 10; const LOGICAL_INSPECTOR_MAX_DEPTH = 80; @@ -1429,18 +1430,16 @@ export function AppShell({ ); useEffect(() => { - if (!hierarchyVisible) { - return; - } - const refreshMs = - accessibilityPreferredSource === "react-native" || - accessibilitySource === "react-native" - ? REACT_NATIVE_ACCESSIBILITY_REFRESH_MS - : accessibilityPreferredSource === "flutter" || - accessibilitySource === "flutter" - ? FLUTTER_ACCESSIBILITY_REFRESH_MS - : ACCESSIBILITY_REFRESH_MS; + hierarchyVisible + ? accessibilityPreferredSource === "react-native" || + accessibilitySource === "react-native" + ? REACT_NATIVE_ACCESSIBILITY_REFRESH_MS + : accessibilityPreferredSource === "flutter" || + accessibilitySource === "flutter" + ? FLUTTER_ACCESSIBILITY_REFRESH_MS + : ACCESSIBILITY_REFRESH_MS + : ACCESSIBILITY_BACKGROUND_REFRESH_MS; let disposed = false; let timeout: number | null = null; const refreshLoop = async () => { diff --git a/client/src/app/uiState.test.ts b/client/src/app/uiState.test.ts index b5bdc75b..86b168a6 100644 --- a/client/src/app/uiState.test.ts +++ b/client/src/app/uiState.test.ts @@ -118,6 +118,36 @@ describe("uiState", () => { ).toBe(false); }); + it("retains the current NativeScript tree during empty or fallback refreshes", () => { + expect( + shouldRetainAccessibilityTreeDuringRefresh( + "auto", + "nativescript", + "nativescript", + 0, + 3, + ), + ).toBe(true); + expect( + shouldRetainAccessibilityTreeDuringRefresh( + "nativescript", + "nativescript", + "native-ax", + 12, + 3, + ), + ).toBe(true); + expect( + shouldRetainAccessibilityTreeDuringRefresh( + "auto", + "nativescript", + "nativescript", + 4, + 3, + ), + ).toBe(false); + }); + it("sanitizes persisted viewport state and falls back to defaults", () => { const sanitized = sanitizePersistedUiState({ bundleIDValue: 123 as unknown as string, diff --git a/client/src/app/uiState.ts b/client/src/app/uiState.ts index 6c22a277..ee98fd00 100644 --- a/client/src/app/uiState.ts +++ b/client/src/app/uiState.ts @@ -173,7 +173,10 @@ export function shouldRetainAccessibilityTreeDuringRefresh( } const retainedSource = currentPreference === "auto" ? currentSource : currentPreference; - if (retainedSource !== "flutter" && retainedSource !== "react-native") { + if ( + !isAccessibilitySource(retainedSource) || + !retainableRichAccessibilitySources.has(retainedSource) + ) { return false; } if (currentSource !== retainedSource) { @@ -182,6 +185,13 @@ export function shouldRetainAccessibilityTreeDuringRefresh( return snapshotSource !== retainedSource || nextRootCount === 0; } +const retainableRichAccessibilitySources = new Set([ + "nativescript", + "react-native", + "flutter", + "swiftui", +]); + export function isAccessibilitySource( value: unknown, ): value is AccessibilitySource { diff --git a/client/src/features/accessibility/AccessibilityInspector.tsx b/client/src/features/accessibility/AccessibilityInspector.tsx index e37e62ed..f9b43b26 100644 --- a/client/src/features/accessibility/AccessibilityInspector.tsx +++ b/client/src/features/accessibility/AccessibilityInspector.tsx @@ -1,5 +1,5 @@ import { useEffect, useRef, useState } from "react"; -import type { CSSProperties, FormEvent } from "react"; +import type { CSSProperties, FormEvent, ReactNode } from "react"; import { sendInspectorRequest } from "../../api/simulators"; import type { @@ -42,6 +42,7 @@ interface AccessibilityInspectorProps { } type InspectorTab = "console" | "inspector" | "performance"; +type DetailValue = ReactNode; export function AccessibilityInspector({ availableSources, @@ -317,6 +318,7 @@ export function AccessibilityInspector({ const kind = accessibilityKind(item.node); const label = hierarchyNodeLabel(item.node, kind); const sourceBadge = sourceLocationBadgeText(item.node); + const sourceHref = sourceLocationHref(item.node); const sourceTitle = sourceLocationText(item.node); const chainBadge = compactedChainBadgeText(item.chain.length); const chainPath = compactedChainPathText(item); @@ -377,15 +379,25 @@ export function AccessibilityInspector({ {label ? ( {label} ) : null} - {sourceBadge ? ( + + {sourceBadge ? ( + sourceHref ? ( + + {sourceBadge} + + ) : ( {sourceBadge} - ) : null} - + ) + ) : null} ); }) @@ -500,10 +512,21 @@ function NodeDetails({ selectedSimulator: SimulatorMetadata | null; }) { const isAndroid = isAndroidSimulator(selectedSimulator); - const details = [ + const sourceText = sourceLocationText(node); + const sourceHref = sourceLocationHref(node); + const details = ([ ["Type", accessibilityKind(node)], ["Label", primaryAccessibilityText(node)], - ["Source", sourceLocationText(node)], + [ + "Source", + sourceHref ? ( + + {sourceText} + + ) : ( + sourceText + ), + ], [ isAndroid ? "Resource ID" : "Identifier", isAndroid @@ -539,7 +562,7 @@ function NodeDetails({ ["PID", node.pid == null ? "" : String(node.pid)], ["Actions", node.custom_actions?.join(", ") ?? ""], ["Help", node.help ?? ""], - ].filter(([, value]) => value); + ] as Array<[string, DetailValue]>).filter(([, value]) => value); return (
@@ -673,6 +696,24 @@ function sourceLocationText(node: AccessibilityNode): string { return `${location.file}:${line}:${column}`; } +function sourceLocationHref(node: AccessibilityNode): string { + const location = primarySourceLocation(node); + return location?.file ? fileUrl(location.file) : ""; +} + +function fileUrl(file: string): string { + if (file.startsWith("file://")) { + return file; + } + if (!file.startsWith("/")) { + return ""; + } + return `file://${file + .split("/") + .map((part, index) => (index === 0 ? "" : encodeURIComponent(part))) + .join("/")}`; +} + function sourceLocationBadgeText(node: AccessibilityNode): string { const location = primarySourceLocation(node); if (!location?.file) { diff --git a/client/src/features/accessibility/AccessibilityOverlay.test.ts b/client/src/features/accessibility/AccessibilityOverlay.test.ts new file mode 100644 index 00000000..9189cc3a --- /dev/null +++ b/client/src/features/accessibility/AccessibilityOverlay.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; + +import { accessibilityDomTagName } from "./AccessibilityOverlay"; + +describe("accessibilityDomTagName", () => { + it("uses source and component names for annotator-friendly custom tags", () => { + expect( + accessibilityDomTagName({ + source: "nativescript", + type: "TabItem", + }), + ).toBe("simdeck-tab-item"); + expect( + accessibilityDomTagName({ + source: "nativescript", + type: "Label", + }), + ).toBe("simdeck-label"); + expect( + accessibilityDomTagName({ + source: "react-native", + type: "RangeAndFilterBar", + }), + ).toBe("simdeck-range-and-filter-bar"); + }); +}); diff --git a/client/src/features/accessibility/AccessibilityOverlay.tsx b/client/src/features/accessibility/AccessibilityOverlay.tsx index 6057349a..8a427cfc 100644 --- a/client/src/features/accessibility/AccessibilityOverlay.tsx +++ b/client/src/features/accessibility/AccessibilityOverlay.tsx @@ -1,4 +1,4 @@ -import type { AriaRole, CSSProperties } from "react"; +import { createElement, type AriaRole, type CSSProperties } from "react"; import type { AccessibilityNode } from "../../api/types"; import { @@ -7,7 +7,8 @@ import { accessibilityRootFrame, buildAccessibilityTree, findAccessibilityItem, - flattenAccessibilityTree, + isAccessibilityHitTestCandidate, + paintOrderedAccessibilityItems, primaryAccessibilityText, validFrame, } from "./accessibilityTree"; @@ -26,8 +27,8 @@ export function AccessibilityOverlay({ const rootFrame = accessibilityRootFrame(roots); const tree = buildAccessibilityTree(roots); const overlayItems = rootFrame - ? flattenAccessibilityTree(tree).filter((item) => - validFrame(item.node.frame), + ? paintOrderedAccessibilityItems(tree).filter( + isAccessibilityHitTestCandidate, ) : []; const selected = selectedId @@ -141,31 +142,40 @@ function AccessibilityDomNode({ } const label = accessibilityDomLabel(node); + const metadata = accessibilityDomMetadata(node, id); const kind = accessibilityKind(node); const role = accessibilityDomRole(kind); + const tagName = accessibilityDomTagName(node); - return ( -
- ); + : undefined, + "aria-label": label, + "aria-level": depth + 1, + "aria-selected": node.selected ?? undefined, + className: "accessibility-dom-node", + "data-testid": `simdeck-accessibility-${id}`, + "data-simdeck-accessibility-id": id, + "data-simdeck-accessibility-component": kind, + "data-simdeck-accessibility-identifier": + accessibilityIdentifier(node) || undefined, + "data-simdeck-accessibility-kind": kind, + "data-simdeck-accessibility-label": primaryAccessibilityText(node), + "data-simdeck-accessibility-image": metadata.imageName, + "data-simdeck-accessibility-source-file": metadata.sourceFile, + "data-simdeck-accessibility-source-line": metadata.sourceLine, + "data-simdeck-accessibility-source-column": metadata.sourceColumn, + "data-simdeck-accessibility-source": node.source || undefined, + "data-simdeck-accessibility-state": metadata.state, + "data-simdeck-accessibility-value": metadata.value, + "data-simdeck-inspector-id": node.inspectorId || undefined, + "data-simdeck-uikit-id": node.uikitId || undefined, + title: label, + role, + style: frameStyle(node.frame, rootFrame), + }); } function frameStyle( @@ -184,10 +194,33 @@ function accessibilityDomLabel(node: AccessibilityNode): string { const text = primaryAccessibilityText(node); const identifier = accessibilityIdentifier(node); const kind = accessibilityKind(node); - if (text && identifier && text !== identifier) { - return `${kind}: ${text} (${identifier})`; + const parts = [`SimDeck accessibility element`, kind]; + if (text) { + parts.push(`label "${text}"`); + } + if (identifier && identifier !== text) { + parts.push(`identifier ${identifier}`); + } + const metadata = accessibilityDomMetadata(node); + if (metadata.value && metadata.value !== text) { + parts.push(`value "${metadata.value}"`); + } + if (metadata.placeholder && metadata.placeholder !== text) { + parts.push(`placeholder "${metadata.placeholder}"`); + } + if (metadata.imageName && metadata.imageName !== text) { + parts.push(`image ${metadata.imageName}`); } - return text || identifier || kind; + if (node.source) { + parts.push(`source ${node.source}`); + } + if (metadata.sourceLocation) { + parts.push(`defined at ${metadata.sourceLocation}`); + } + if (metadata.state) { + parts.push(metadata.state); + } + return parts.join("; "); } function accessibilityDomRole(kind: string): AriaRole { @@ -224,3 +257,105 @@ function accessibilityDomRole(kind: string): AriaRole { } return "group"; } + +export function accessibilityDomTagName(node: AccessibilityNode): string { + const kind = accessibilityKind(node); + const component = cleanTagPart(kind) ?? "element"; + return `simdeck-${component}`; +} + +function cleanTagPart(value: string | null | undefined): string | null { + const kebab = value + ?.trim() + .replace(/([a-z0-9])([A-Z])/g, "$1-$2") + .replace(/[^A-Za-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .toLowerCase(); + return kebab || null; +} + +function accessibilityDomMetadata(node: AccessibilityNode, id?: string) { + const sourceLocation = primarySourceLocation(node); + return { + imageName: cleanAccessibilityText(node.imageName), + placeholder: cleanAccessibilityText(node.placeholder), + sourceFile: sourceLocation.file || undefined, + sourceColumn: + typeof sourceLocation.column === "number" + ? String(sourceLocation.column) + : undefined, + sourceLine: + typeof sourceLocation.line === "number" + ? String(sourceLocation.line) + : undefined, + sourceLocation: formatSourceLocation(sourceLocation), + state: accessibilityStateSummary(node, id), + value: cleanAccessibilityText(node.AXValue), + }; +} + +function primarySourceLocation(node: AccessibilityNode): { + column: number | null; + file: string; + line: number | null; +} { + const location = + node.sourceLocation ?? + node.sourceLocations?.find((location) => location?.file) ?? + null; + const file = + cleanAccessibilityText(location?.file) ?? + cleanAccessibilityText(node.sourceFile) ?? + ""; + const line = + typeof location?.line === "number" + ? location.line + : typeof node.sourceLine === "number" + ? node.sourceLine + : null; + const column = + typeof location?.column === "number" + ? location.column + : typeof node.sourceColumn === "number" + ? node.sourceColumn + : null; + return { column, file, line }; +} + +function formatSourceLocation(location: { + column: number | null; + file: string; + line: number | null; +}): string { + if (!location.file) { + return ""; + } + return location.line == null + ? location.file + : location.column == null + ? `${location.file}:${location.line}` + : `${location.file}:${location.line}:${location.column}`; +} + +function accessibilityStateSummary( + node: AccessibilityNode, + id: string | undefined, +): string { + const state = [ + id ? `tree id ${id}` : "", + node.enabled === false ? "disabled" : "", + node.focused === true ? "focused" : "", + node.selected === true ? "selected" : "", + node.checked === true ? "checked" : "", + node.checked === false ? "unchecked" : "", + node.clickable === true ? "clickable" : "", + node.scrollable === true ? "scrollable" : "", + ].filter(Boolean); + return state.join(", "); +} + +function cleanAccessibilityText( + value: string | null | undefined, +): string | null { + return value?.trim() || null; +} diff --git a/client/src/features/accessibility/accessibilityTree.test.ts b/client/src/features/accessibility/accessibilityTree.test.ts index aa3dd0fb..3907d9f0 100644 --- a/client/src/features/accessibility/accessibilityTree.test.ts +++ b/client/src/features/accessibility/accessibilityTree.test.ts @@ -4,6 +4,9 @@ import type { AccessibilityNode } from "../../api/types"; import { buildAccessibilityTree, findAccessibilityItemAtPoint, + isAccessibilityHitTestCandidate, + paintOrderedAccessibilityItems, + primaryAccessibilityText, } from "./accessibilityTree"; describe("buildAccessibilityTree", () => { @@ -162,6 +165,19 @@ describe("buildAccessibilityTree", () => { }); }); +describe("primaryAccessibilityText", () => { + it("uses image source fallback instead of generated NativeScript class names", () => { + expect( + primaryAccessibilityText({ + source: "nativescript", + type: "_ImageCacheIt", + title: "_ImageCacheIt", + imageName: "~/assets/album-midnight.jpg", + }), + ).toBe("~/assets/album-midnight.jpg"); + }); +}); + describe("findAccessibilityItemAtPoint", () => { it("descends through frameless wrapper nodes", () => { const roots: AccessibilityNode[] = [ @@ -360,4 +376,175 @@ describe("findAccessibilityItemAtPoint", () => { expect(item?.node.type).toBe("Text"); expect(item?.id).toBe("0.0"); }); + + it("prefers the first application root over stale overlapping roots", () => { + const roots: AccessibilityNode[] = [ + { + type: "CurrentApp", + frame: { x: 0, y: 0, width: 400, height: 800 }, + children: [ + { + type: "Button", + AXLabel: "Current continue", + frame: { x: 100, y: 300, width: 200, height: 60 }, + }, + ], + }, + { + type: "PreviousApp", + frame: { x: 0, y: 0, width: 400, height: 800 }, + children: [ + { + type: "Button", + AXLabel: "Previous continue", + frame: { x: 100, y: 300, width: 200, height: 60 }, + }, + ], + }, + ]; + + const item = findAccessibilityItemAtPoint(roots, { x: 0.5, y: 0.4125 }); + + expect(item?.node.AXLabel).toBe("Current continue"); + expect(item?.id).toBe("0.0"); + }); + + it("does not select full-screen NativeScript containers as fallback targets", () => { + const roots: AccessibilityNode[] = [ + { + source: "nativescript", + type: "Frame", + title: "Frame", + frame: { x: 0, y: 0, width: 402, height: 874 }, + children: [ + { + source: "nativescript", + type: "Label", + title: "Now Playing", + frame: { x: 24, y: 120, width: 180, height: 36 }, + }, + ], + }, + ]; + + expect(findAccessibilityItemAtPoint(roots, { x: 0.5, y: 0.5 })).toBeNull(); + expect( + findAccessibilityItemAtPoint(roots, { x: 0.2, y: 0.158 })?.node.type, + ).toBe("Label"); + }); + + it("selects synthetic NativeScript tab items over content under the tab bar", () => { + const roots: AccessibilityNode[] = [ + { + source: "nativescript", + type: "TabView", + title: "TabView", + frame: { x: 0, y: 0, width: 402, height: 874 }, + children: [ + { + source: "nativescript", + type: "Label", + title: "Album title underneath", + frame: { x: 0, y: 810, width: 402, height: 44 }, + }, + { + source: "nativescript", + type: "TabItem", + title: "Home", + frame: { x: 20, y: 791, width: 70, height: 83 }, + }, + ], + }, + ]; + + const item = findAccessibilityItemAtPoint(roots, { x: 0.14, y: 0.94 }); + + expect(item?.node.type).toBe("TabItem"); + expect(item?.node.title).toBe("Home"); + }); + + it("descends through NativeScript tab accessory wrappers", () => { + const roots: AccessibilityNode[] = [ + { + source: "nativescript", + type: "TabView", + frame: { x: 0, y: 0, width: 402, height: 874 }, + children: [ + { + source: "nativescript", + type: "TabAccessory", + title: "Tab accessory", + frame: { x: 21, y: 735, width: 360, height: 48 }, + children: [ + { + source: "nativescript", + type: "Label", + title: "Neon Pulse", + frame: { x: 85, y: 742, width: 212, height: 18 }, + }, + ], + }, + ], + }, + ]; + + const item = findAccessibilityItemAtPoint(roots, { + x: 85 / 402, + y: 742 / 874, + }); + + expect(item?.node.type).toBe("Label"); + expect(item?.node.title).toBe("Neon Pulse"); + }); +}); + +describe("paintOrderedAccessibilityItems", () => { + it("paints later roots first so the preferred root is hit-tested on top", () => { + const tree = buildAccessibilityTree([ + { + type: "CurrentApp", + frame: { x: 0, y: 0, width: 400, height: 800 }, + children: [ + { + type: "Button", + AXLabel: "Current continue", + frame: { x: 100, y: 300, width: 200, height: 60 }, + }, + ], + }, + { + type: "PreviousApp", + frame: { x: 0, y: 0, width: 400, height: 800 }, + }, + ]); + + expect(paintOrderedAccessibilityItems(tree).map((item) => item.id)).toEqual( + ["1", "0", "0.0"], + ); + }); + + it("keeps transparent NativeScript containers out of annotatable DOM targets", () => { + const tree = buildAccessibilityTree([ + { + source: "nativescript", + type: "Frame", + title: "Frame", + frame: { x: 0, y: 0, width: 402, height: 874 }, + children: [ + { + source: "nativescript", + type: "Label", + title: "Albums", + frame: { x: 24, y: 120, width: 120, height: 36 }, + }, + ], + }, + ]); + + expect( + paintOrderedAccessibilityItems(tree) + .filter(isAccessibilityHitTestCandidate) + .map((item) => item.node.type), + ).toEqual(["Label"]); + }); }); diff --git a/client/src/features/accessibility/accessibilityTree.ts b/client/src/features/accessibility/accessibilityTree.ts index 9492315b..572e7af0 100644 --- a/client/src/features/accessibility/accessibilityTree.ts +++ b/client/src/features/accessibility/accessibilityTree.ts @@ -23,6 +23,20 @@ export function flattenAccessibilityTree( ]); } +export function paintOrderedAccessibilityItems( + items: AccessibilityTreeItem[], +): AccessibilityTreeItem[] { + return [...items] + .reverse() + .flatMap((item) => flattenAccessibilityTree([item])); +} + +export function isAccessibilityHitTestCandidate( + item: AccessibilityTreeItem, +): boolean { + return validFrame(item.node.frame) && !isTransparentHitTestBlocker(item); +} + export function visibleAccessibilityTreeItems( items: AccessibilityTreeItem[], expandedIds: Set, @@ -64,7 +78,7 @@ export function findAccessibilityItemAtPoint( x: rootFrame.x + normalizedPoint.x * rootFrame.width, y: rootFrame.y + normalizedPoint.y * rootFrame.height, }; - return findContainingItem(buildAccessibilityTree(roots), point); + return findContainingRootItem(buildAccessibilityTree(roots), point); } export function defaultExpandedAccessibilityIds( @@ -88,13 +102,20 @@ export function ancestorAccessibilityIds(id: string): string[] { } export function primaryAccessibilityText(node: AccessibilityNode): string { + const generatedNames = generatedNodeNames(node); return ( - cleanText(node.AXLabel) ?? - cleanText(node.title) ?? - cleanText(node.AXUniqueId) ?? - cleanText(node.AXIdentifier) ?? - cleanText(node.AXValue) ?? - "" + [ + node.AXLabel, + node.text, + node.title, + node.AXUniqueId, + node.AXIdentifier, + node.AXValue, + node.placeholder, + node.imageName, + ] + .map(cleanText) + .find((text) => text && !generatedNames.has(text)) ?? "" ); } @@ -215,7 +236,27 @@ function findContainingItem( if ( frameContainsPoint(item.node.frame, point) && - !isTransparentHitTestBlocker(item) + isAccessibilityHitTestCandidate(item) + ) { + return item; + } + } + return null; +} + +function findContainingRootItem( + items: AccessibilityTreeItem[], + point: { x: number; y: number }, +): AccessibilityTreeItem | null { + for (const item of items) { + const childMatch = findContainingItem(item.children, point); + if (childMatch) { + return childMatch; + } + + if ( + frameContainsPoint(item.node.frame, point) && + isAccessibilityHitTestCandidate(item) ) { return item; } @@ -235,6 +276,13 @@ function isTransparentHitTestBlocker(item: AccessibilityTreeItem): boolean { ); } + if (node.source === "nativescript") { + return ( + !hasMeaningfulNodeContent(node) && + isNativeScriptTransparentContainerType(cleanText(node.type)) + ); + } + if (node.source !== "in-app-inspector" || node.nativeScript) { return false; } @@ -251,6 +299,11 @@ function isTransparentHitTestBlocker(item: AccessibilityTreeItem): boolean { return !hasMeaningfulNodeContent(node); } +function isNativeScriptTransparentContainerType(value: string | null): boolean { + const type = unqualifiedClassName(value); + return Boolean(type && nativeScriptTransparentContainerTypes.has(type)); +} + function isTransparentContainerClass(value: string | null): boolean { const className = unqualifiedClassName(value); return ( @@ -418,12 +471,27 @@ const flutterFrameworkContainerTypes = new Set([ "WidgetsApp", ]); +const nativeScriptTransparentContainerTypes = new Set([ + "AbsoluteLayout", + "ActionBar", + "ContentView", + "DockLayout", + "FlexboxLayout", + "Frame", + "GridLayout", + "HtmlView", + "Page", + "Placeholder", + "ProxyViewContainer", + "RootLayout", + "StackLayout", + "TabAccessory", + "TabView", + "WrapLayout", +]); + function hasMeaningfulNodeContent(node: AccessibilityNode): boolean { - const generatedNames = new Set( - [node.type, node.className, node.role, "UIView", "UIKit View"] - .map(cleanText) - .filter(Boolean), - ); + const generatedNames = generatedNodeNames(node); return [ node.AXLabel, node.text, @@ -441,6 +509,13 @@ function hasMeaningfulNodeContent(node: AccessibilityNode): boolean { }); } +function generatedNodeNames(node: AccessibilityNode): Set { + const names = [node.type, node.className, node.role, "UIView", "UIKit View"] + .map(cleanText) + .filter((text): text is string => Boolean(text)); + return new Set([...names, ...names.map((name) => `_${name}`)]); +} + function primarySourceLocationFile(node: AccessibilityNode): string { return ( cleanText(node.sourceLocation?.file) ?? diff --git a/client/src/styles/components.css b/client/src/styles/components.css index e9af6007..499bbcd6 100644 --- a/client/src/styles/components.css +++ b/client/src/styles/components.css @@ -1274,9 +1274,15 @@ color: var(--text-muted); font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; font-size: 10px; + text-decoration: none; white-space: nowrap; } +a.hierarchy-node-source:hover, +.hierarchy-detail-link:hover { + text-decoration: underline; +} + .hierarchy-node.selected .hierarchy-node-source { color: color-mix(in srgb, var(--accent-text) 72%, transparent); } @@ -1336,6 +1342,11 @@ color: var(--text); } +.hierarchy-detail-link { + color: inherit; + text-decoration: none; +} + .uikit-script { margin-top: 14px; padding-top: 12px; @@ -2542,7 +2553,6 @@ min-height: 1px; overflow: hidden; border: 0; - opacity: 0; pointer-events: auto; } diff --git a/packages/flutter-inspector/README.md b/packages/flutter-inspector/README.md index 4cda9fbc..085fdbfa 100644 --- a/packages/flutter-inspector/README.md +++ b/packages/flutter-inspector/README.md @@ -23,7 +23,10 @@ void main() { WidgetsFlutterBinding.ensureInitialized(); if (kDebugMode) { - startSimDeckFlutterInspector(port: 4310); + startSimDeckFlutterInspector( + port: 4310, + sourceRoot: '/absolute/path/to/your/app', + ); } runApp(const App()); @@ -42,7 +45,7 @@ ws://127.0.0.1:4310/api/inspector/connect - RenderObject screen frames in logical screen points. - Widget diagnostics properties and state type metadata. - Semantics labels, values, hints, identifiers, flags, roles, and actions. -- Source locations from Flutter's widget creation tracking in debug builds. +- Source locations from Flutter's widget creation tracking in debug builds. Pass `sourceRoot` to publish absolute `sourceLocation.file` values when Flutter reports relative paths. - `View.hitTest`, `View.describeAtPoint`, `View.getProperties`, `View.listActions`, and `View.perform`. `View.perform` supports best-effort `tap`, `longPress`, `focus`, `resignFirstResponder`, `setText`, `scrollBy`, `scrollTo`, `increase`, and `decrease` actions when the selected widget exposes the matching Flutter semantics, text, focus, or scroll API. diff --git a/packages/flutter-inspector/lib/simdeck_flutter_inspector.dart b/packages/flutter-inspector/lib/simdeck_flutter_inspector.dart index a67d8f8c..21cebe0e 100644 --- a/packages/flutter-inspector/lib/simdeck_flutter_inspector.dart +++ b/packages/flutter-inspector/lib/simdeck_flutter_inspector.dart @@ -22,6 +22,7 @@ SimDeckFlutterInspector startSimDeckFlutterInspector({ int port = 4310, bool reconnect = true, bool secure = false, + String sourceRoot = '', }) { if (_sharedInspector != null) { return _sharedInspector!; @@ -33,6 +34,7 @@ SimDeckFlutterInspector startSimDeckFlutterInspector({ port: port, reconnect: reconnect, secure: secure, + sourceRoot: sourceRoot, ), ); _sharedInspector = inspector; @@ -53,6 +55,7 @@ class SimDeckFlutterInspectorOptions { this.port = 4310, this.reconnect = true, this.secure = false, + this.sourceRoot = '', }); final String host; @@ -60,11 +63,12 @@ class SimDeckFlutterInspectorOptions { final int port; final bool reconnect; final bool secure; + final String sourceRoot; } class SimDeckFlutterInspector { SimDeckFlutterInspector([SimDeckFlutterInspectorOptions? options]) - : options = options ?? const SimDeckFlutterInspectorOptions(); + : options = options ?? const SimDeckFlutterInspectorOptions(); final SimDeckFlutterInspectorOptions options; final Expando _ids = Expando('simdeckFlutterInspectorId'); @@ -301,6 +305,9 @@ class SimDeckFlutterInspector { 'displayScale': metadata['displayScale'], 'screenBounds': metadata['screenBounds'], 'coordinateSpace': 'screen-points', + 'sourceRoot': _normalizeSourceRoot(options.sourceRoot).isEmpty + ? null + : _normalizeSourceRoot(options.sourceRoot), 'methods': [ 'Runtime.ping', 'Inspector.getInfo', @@ -320,8 +327,8 @@ class SimDeckFlutterInspector { }, 'flutter': { 'available': true, - 'widgetCreationTracked': - WidgetInspectorService.instance.isWidgetCreationTracked(), + 'widgetCreationTracked': WidgetInspectorService.instance + .isWidgetCreationTracked(), }, 'uikit': {'available': false, 'propertyEditing': false}, }; @@ -332,8 +339,9 @@ class SimDeckFlutterInspector { return _metadata!; } final view = _firstFlutterView(); - final logicalSize = - view == null ? Size.zero : view.physicalSize / view.devicePixelRatio; + final logicalSize = view == null + ? Size.zero + : view.physicalSize / view.devicePixelRatio; final fallback = { 'processIdentifier': io.pid, 'bundleIdentifier': io.Platform.resolvedExecutable, @@ -788,10 +796,14 @@ class SimDeckFlutterInspector { 'protocolVersion': _protocolVersion, 'processIdentifier': metadata['processIdentifier'] ?? io.pid, 'bundleIdentifier': metadata['bundleIdentifier'], - 'displayScale': metadata['displayScale'] ?? + 'displayScale': + metadata['displayScale'] ?? _firstFlutterView()?.devicePixelRatio ?? 1.0, 'coordinateSpace': 'screen-points', + 'sourceRoot': _normalizeSourceRoot(options.sourceRoot).isEmpty + ? null + : _normalizeSourceRoot(options.sourceRoot), }; } @@ -869,18 +881,18 @@ class SimDeckFlutterInspector { } try { final json = element.toDiagnosticsNode().toJsonMap( - InspectorSerializationDelegate( - service: WidgetInspectorService.instance, - subtreeDepth: 0, - ), - ); + InspectorSerializationDelegate( + service: WidgetInspectorService.instance, + subtreeDepth: 0, + ), + ); final location = _objectMap(json['creationLocation']); if (location == null) { _sourceLocations[element] = _noSourceLocation; return null; } final sourceLocation = { - 'file': location['file'], + 'file': _absoluteSourceFile(_stringValue(location['file'])), 'line': location['line'], 'column': location['column'], 'name': location['name'], @@ -894,6 +906,14 @@ class SimDeckFlutterInspector { } } + String _absoluteSourceFile(String file) { + final root = _normalizeSourceRoot(options.sourceRoot); + if (file.isEmpty || _isAbsoluteSourceFile(file) || root.isEmpty) { + return file; + } + return '$root/${file.replaceFirst(RegExp(r'^\./?'), '')}'; + } + bool _shouldReadSourceLocation( Element element, Map? semantics, @@ -1070,7 +1090,7 @@ class SimDeckFlutterInspectorFailure implements Exception { class _TraversalContext { _TraversalContext() - : deadline = DateTime.now().add(const Duration(seconds: 3)); + : deadline = DateTime.now().add(const Duration(seconds: 3)); final DateTime deadline; int remainingNodes = 3500; @@ -1325,6 +1345,17 @@ double? _optionalDouble(Object? value) { String _stringValue(Object? value) => value == null ? '' : value.toString(); +String _normalizeSourceRoot(String sourceRoot) { + final root = sourceRoot.trim().replaceFirst(RegExp(r'/+$'), ''); + return _isAbsoluteSourceFile(root) ? root : ''; +} + +bool _isAbsoluteSourceFile(String file) { + return file.startsWith('/') || + RegExp(r'^[a-zA-Z][a-zA-Z0-9+.-]*:').hasMatch(file) || + RegExp(r'^[A-Za-z]:[\\/]').hasMatch(file); +} + String? _firstString(Iterable values) { for (final value in values) { final text = value?.toString().trim(); diff --git a/packages/nativescript-inspector/README.md b/packages/nativescript-inspector/README.md index 3dffa0ea..0037f2df 100644 --- a/packages/nativescript-inspector/README.md +++ b/packages/nativescript-inspector/README.md @@ -10,7 +10,10 @@ npm install @nativescript/simdeck-inspector import { startSimDeckInspector } from "@nativescript/simdeck-inspector"; if (__DEV__) { - startSimDeckInspector({ port: 4310 }); + startSimDeckInspector({ + port: 4310, + sourceRoot: "/absolute/path/to/your/app", + }); } ``` @@ -36,3 +39,6 @@ back to raw UIKit when called with `{ "source": "uikit" }`. For Angular NativeScript apps, call `startSimDeckInspector()` before `runNativeScriptAngularApp()`. The package installs a small compatibility shim for Angular 20 dev-mode template source locations on NativeScript views. +Pass `sourceRoot` when your framework reports project-relative source paths; +SimDeck will publish absolute `sourceLocation.file` values for Codex context +and local `file://` links in the inspector panel. diff --git a/packages/nativescript-inspector/package-lock.json b/packages/nativescript-inspector/package-lock.json index 9278306c..aa969d25 100644 --- a/packages/nativescript-inspector/package-lock.json +++ b/packages/nativescript-inspector/package-lock.json @@ -1,12 +1,12 @@ { "name": "@nativescript/simdeck-inspector", - "version": "0.1.3", + "version": "0.1.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@nativescript/simdeck-inspector", - "version": "0.1.3", + "version": "0.1.4", "license": "Apache-2.0", "devDependencies": { "@nativescript/core": "~9.0.0", diff --git a/packages/nativescript-inspector/package.json b/packages/nativescript-inspector/package.json index a62f996c..991adab5 100644 --- a/packages/nativescript-inspector/package.json +++ b/packages/nativescript-inspector/package.json @@ -1,6 +1,6 @@ { "name": "@nativescript/simdeck-inspector", - "version": "0.1.3", + "version": "0.1.4", "description": "Debug-only NativeScript runtime inspector for simdeck", "license": "Apache-2.0", "repository": { diff --git a/packages/nativescript-inspector/src/index.ts b/packages/nativescript-inspector/src/index.ts index 8b8f4a45..c76216d8 100644 --- a/packages/nativescript-inspector/src/index.ts +++ b/packages/nativescript-inspector/src/index.ts @@ -25,6 +25,7 @@ declare const CGPointMake: any; declare const CGSizeMake: any; declare const UIEdgeInsetsMake: any; declare const WebSocket: any | undefined; +declare const global: any; type JSONObject = Record; @@ -34,6 +35,7 @@ export interface SimDeckInspectorOptions { port?: number; reconnect?: boolean; secure?: boolean; + sourceRoot?: string; } interface InspectorRequest { @@ -249,9 +251,11 @@ export class SimDeckNativeScriptInspector { private readonly options: Required; private socket: InspectorSocket | null = null; private pollTimer: ReturnType | null = null; + private infoRefreshTimer: ReturnType | null = null; private polling = false; private reconnectTimer: ReturnType | null = null; private nextObjectId = 1; + private lastAnnouncedInfoKey = ""; private readonly ids = new WeakMap(); private readonly nativeScriptViewsByNativeView = new WeakMap(); private readonly objects = new Map(); @@ -264,6 +268,7 @@ export class SimDeckNativeScriptInspector { port: options.port ?? 4310, reconnect: options.reconnect ?? true, secure: options.secure ?? false, + sourceRoot: normalizeSourceRoot(options.sourceRoot), }; } @@ -271,11 +276,11 @@ export class SimDeckNativeScriptInspector { this.stop(); const scheme = this.options.secure ? "wss" : "ws"; const url = `${scheme}://${this.options.host}:${this.options.port}${this.options.path}`; - let announced = false; const socket = createInspectorSocket(url, { onClose: () => { if (this.socket === socket) { this.socket = null; + this.clearInfoRefresh(); } if (this.options.reconnect) { this.scheduleReconnect(); @@ -294,27 +299,14 @@ export class SimDeckNativeScriptInspector { }); }, onOpen: () => { - if (announced) { - return; - } - announced = true; - socket.send( - JSON.stringify({ - method: "Inspector.ready", - params: this.info(), - }), - ); + this.announceInfo(true); + this.scheduleInfoRefresh(250); }, }); this.socket = socket; if (socket.readyState === 1) { - socket.send( - JSON.stringify({ - method: "Inspector.ready", - params: this.info(), - }), - ); - announced = true; + this.announceInfo(true); + this.scheduleInfoRefresh(250); } this.startPolling(); } @@ -328,7 +320,9 @@ export class SimDeckNativeScriptInspector { clearTimeout(this.pollTimer); this.pollTimer = null; } + this.clearInfoRefresh(); this.polling = false; + this.lastAnnouncedInfoKey = ""; const socket = this.socket; this.socket = null; if (socket) { @@ -467,6 +461,48 @@ export class SimDeckNativeScriptInspector { this.socket.send(JSON.stringify(payload)); } + private announceInfo(force = false): void { + const info = this.info(); + const key = infoAnnouncementKey(info); + if (!force && key === this.lastAnnouncedInfoKey) { + return; + } + this.lastAnnouncedInfoKey = key; + this.send({ + method: "Inspector.ready", + params: info, + }); + } + + private scheduleInfoRefresh(delay: number): void { + if (!this.socket || this.socket.readyState !== 1) { + return; + } + this.clearInfoRefresh(); + this.infoRefreshTimer = setTimeout(() => { + this.infoRefreshTimer = null; + this.refreshInfoAnnouncement(); + }, delay); + } + + private clearInfoRefresh(): void { + if (!this.infoRefreshTimer) { + return; + } + clearTimeout(this.infoRefreshTimer); + this.infoRefreshTimer = null; + } + + private refreshInfoAnnouncement(): void { + if (!this.socket || this.socket.readyState !== 1) { + return; + } + this.announceInfo(false); + if (!nativeScriptAvailable(this.info())) { + this.scheduleInfoRefresh(250); + } + } + private dispatch(method: string, params: JSONObject): unknown { switch (method) { case "Runtime.ping": @@ -509,6 +545,7 @@ export class SimDeckNativeScriptInspector { displayScale: numberValue(UIScreen.mainScreen.scale, 1), screenBounds: rectValue(UIScreen.mainScreen.bounds), coordinateSpace: "screen-points", + sourceRoot: this.options.sourceRoot || undefined, methods: [ "Runtime.ping", "Inspector.getInfo", @@ -691,6 +728,8 @@ export class SimDeckNativeScriptInspector { depth: number, ): JSONObject { const nativeView = nativeViewFor(view); + const nativeScriptType = + stringValue((view as any).typeName) || constructorName(view); const children: JSONObject[] = []; if (maxDepth == null || depth < maxDepth) { safeCall(() => { @@ -703,34 +742,335 @@ export class SimDeckNativeScriptInspector { return true; }); }, undefined); + if (nativeScriptType === "TabView") { + const tabAccessoryNode = this.nativeScriptTabAccessoryNode( + view, + nativeView, + includeHidden, + maxDepth, + depth + 1, + ); + if (tabAccessoryNode) { + children.push(tabAccessoryNode); + } + const tabBarNodes = this.nativeScriptTabBarNodes( + view, + nativeView, + children, + ); + children.push(...tabBarNodes); + } } const id = this.objectId("ns", view); const uikitId = nativeView ? this.objectId("view", nativeView) : null; + const nativeAccessibility = nativeView + ? accessibilityInfo(nativeView) + : null; + const nativeControl = nativeView ? controlInfo(nativeView) : null; + const nativeScroll = nativeView ? scrollInfo(nativeView) : null; + const nativeActions = nativeView ? actionsFor(nativeView) : []; + const sourceLocation = sourceLocationForView(view, this.options.sourceRoot); + const nativeScriptData = nativeScriptInfo( + view, + nativeScriptType, + this.options.sourceRoot, + ); + const imageSource = nativeScriptImageSource(view, nativeScriptType); return { id, inspectorId: id, - type: stringValue((view as any).typeName) || constructorName(view), - title: nativeScriptTitle(view), + type: nativeScriptType, + displayName: nativeScriptType, + title: nativeScriptTitle(view, nativeScriptType), + AXIdentifier: nativeAccessibility?.identifier ?? "", + AXLabel: nativeAccessibility?.label ?? "", + AXValue: nativeAccessibility?.value ?? "", + help: nativeAccessibility?.hint ?? "", + text: nativeView ? textValue(nativeView) : "", + imageName: imageSource, + placeholder: nativeView + ? stringValue(read(nativeView, "placeholder")) + : "", + enabled: nativeView + ? isKindOf(nativeView, "UIControl") + ? Boolean(read(nativeView, "enabled")) + : Boolean(read(nativeView, "userInteractionEnabled") ?? true) + : null, + clickable: nativeActions.includes("tap"), + scrollable: Boolean(nativeScroll), source: "nativescript", - sourceLocation: sourceLocationForView(view), + sourceLocation, + sourceLocations: sourceLocation ? [sourceLocation] : [], + sourceFile: sourceString(sourceLocation, "file"), + sourceLine: sourceNumber(sourceLocation, "line"), + sourceColumn: sourceNumber(sourceLocation, "column"), frame: nativeView ? frameInScreen(nativeView) : null, - nativeScript: { - id: stringValue((view as any).id), - className: stringValue((view as any).className), - }, + nativeScript: nativeScriptData, uikit: nativeView ? { id: uikitId, className: className(nativeView), script: this.uikitScriptFor(uikitId, nativeView), + accessibility: nativeAccessibility, + actions: nativeActions, } : null, uikitId, + control: nativeControl, + scroll: nativeScroll, children, }; } + private nativeScriptTabAccessoryNode( + tabView: View, + nativeView: any | null, + includeHidden: boolean, + maxDepth: number | null, + depth: number, + ): JSONObject | null { + const accessoryView = nativeScriptTabAccessoryView(tabView); + if (!accessoryView) { + return null; + } + const accessoryNativeView = nativeViewFor(accessoryView); + const uikitAccessory = this.findUIKitTabAccessory(nativeView); + const accessoryFrame = + accessoryNativeView && hasUsableFrame(frameInScreen(accessoryNativeView)) + ? frameInScreen(accessoryNativeView) + : uikitAccessory + ? frameInScreen(uikitAccessory) + : null; + let child = + maxDepth == null || depth < maxDepth + ? this.nativeScriptNode(accessoryView, includeHidden, maxDepth, depth + 1) + : null; + if (child && uikitAccessory) { + child = patchNativeScriptFramesFromUIKit(child, uikitAccessory); + } + const sourceLocation = + sourceLocationForView(accessoryView, this.options.sourceRoot) || + firstSourceLocationFromNodes(child ? [child] : []); + const id = `${this.objectId("ns", tabView)}:accessory`; + return { + id, + inspectorId: id, + type: "TabAccessory", + displayName: "TabAccessory", + title: "Tab accessory", + AXIdentifier: "", + AXLabel: "Tab accessory", + AXValue: "", + help: "", + text: "", + imageName: "", + placeholder: "", + enabled: true, + clickable: false, + scrollable: false, + source: "nativescript", + sourceLocation, + sourceLocations: sourceLocation ? [sourceLocation] : [], + sourceFile: sourceString(sourceLocation, "file"), + sourceLine: sourceNumber(sourceLocation, "line"), + sourceColumn: sourceNumber(sourceLocation, "column"), + frame: accessoryFrame, + nativeScript: { + type: "TabAccessory", + sourceLocation, + sourceFile: sourceString(sourceLocation, "file"), + sourceLine: sourceNumber(sourceLocation, "line"), + sourceColumn: sourceNumber(sourceLocation, "column"), + }, + uikit: null, + uikitId: null, + control: null, + scroll: null, + children: child ? [child] : [], + }; + } + + private findUIKitTabAccessory(nativeView: any | null): any | null { + const root = nativeView || this.windows()[0] || null; + return root + ? findSubview(root, (view) => /TabAccessory/.test(className(view))) + : null; + } + + private nativeScriptTabBarNodes( + tabView: View, + nativeView: any | null, + nativeScriptChildrenNodes: JSONObject[], + ): JSONObject[] { + const tabBar = + (nativeView + ? findSubview(nativeView, (view) => isKindOf(view, "UITabBar")) + : null) || this.findUIKitTabBar(); + if (!tabBar) { + return this.fallbackNativeScriptTabBarNodes( + tabView, + null, + nativeScriptChildrenNodes, + ); + } + const selectedIndex = numberValue(read(tabView, "selectedIndex"), -1); + const tabItems = nativeScriptTabItems( + tabView, + nativeScriptChildrenNodes, + this.options.sourceRoot, + ); + const controls = tabBarControls(tabBar); + if (controls.length === 0) { + return this.fallbackNativeScriptTabBarNodes( + tabView, + frameInScreen(tabBar), + nativeScriptChildrenNodes, + ); + } + return controls.map((control, index) => { + const item = tabItems[index] ?? {}; + const title = + stringValue(item.title) || + tabBarControlLabel(control) || + `Tab ${index + 1}`; + const sourceLocation = tabItemSourceLocation( + tabView, + item, + index, + this.options.sourceRoot, + ); + const id = this.objectId("view", control); + const nativeActions = actionsFor(control); + return { + id, + inspectorId: id, + type: "TabItem", + displayName: "TabItem", + title, + AXIdentifier: stringValue(read(control, "accessibilityIdentifier")), + AXLabel: title, + AXValue: index === selectedIndex ? "selected" : "", + help: stringValue(read(control, "accessibilityHint")), + text: title, + imageName: stringValue(item.iconSource), + placeholder: "", + enabled: Boolean(read(control, "enabled") ?? true), + clickable: nativeActions.includes("tap"), + scrollable: false, + selected: index === selectedIndex, + source: "nativescript", + sourceLocation, + sourceLocations: sourceLocation ? [sourceLocation] : [], + sourceFile: sourceString(sourceLocation, "file"), + sourceLine: sourceNumber(sourceLocation, "line"), + sourceColumn: sourceNumber(sourceLocation, "column"), + frame: frameInScreen(control), + nativeScript: { + type: "TabItem", + title, + iconSource: stringValue(item.iconSource), + index, + selected: index === selectedIndex, + sourceLocation, + sourceFile: sourceString(sourceLocation, "file"), + sourceLine: sourceNumber(sourceLocation, "line"), + sourceColumn: sourceNumber(sourceLocation, "column"), + }, + uikit: { + id, + className: className(control), + script: this.uikitScriptFor(id, control), + accessibility: accessibilityInfo(control), + actions: nativeActions, + }, + uikitId: id, + control: controlInfo(control), + scroll: null, + children: [], + }; + }); + } + + private fallbackNativeScriptTabBarNodes( + tabView: View, + tabBarFrame: JSONObject | null, + nativeScriptChildrenNodes: JSONObject[] = [], + ): JSONObject[] { + const selectedIndex = numberValue(read(tabView, "selectedIndex"), -1); + const tabItems = nativeScriptTabItems( + tabView, + nativeScriptChildrenNodes, + this.options.sourceRoot, + ); + if (tabItems.length === 0) { + return []; + } + const parentId = this.objectId("ns", tabView); + const frame = tabBarFrame ?? fallbackTabBarFrame(); + return tabItems.map((item, index) => { + const title = stringValue(item.title) || `Tab ${index + 1}`; + const sourceLocation = tabItemSourceLocation( + tabView, + item, + index, + this.options.sourceRoot, + ); + return { + id: `${parentId}:tab:${index}`, + inspectorId: `${parentId}:tab:${index}`, + type: "TabItem", + displayName: "TabItem", + title, + AXIdentifier: "", + AXLabel: title, + AXValue: index === selectedIndex ? "selected" : "", + help: "", + text: title, + imageName: stringValue(item.iconSource), + placeholder: "", + enabled: true, + clickable: false, + scrollable: false, + selected: index === selectedIndex, + source: "nativescript", + sourceLocation, + sourceLocations: sourceLocation ? [sourceLocation] : [], + sourceFile: sourceString(sourceLocation, "file"), + sourceLine: sourceNumber(sourceLocation, "line"), + sourceColumn: sourceNumber(sourceLocation, "column"), + frame: tabItemFallbackFrame(frame, tabItems.length, index), + nativeScript: { + type: "TabItem", + title, + iconSource: stringValue(item.iconSource), + index, + selected: index === selectedIndex, + synthetic: true, + sourceLocation, + sourceFile: sourceString(sourceLocation, "file"), + sourceLine: sourceNumber(sourceLocation, "line"), + sourceColumn: sourceNumber(sourceLocation, "column"), + }, + uikit: null, + uikitId: null, + control: null, + scroll: null, + children: [], + }; + }); + } + + private findUIKitTabBar(): any | null { + for (const window of this.windows()) { + const tabBar = findSubview(window, (view) => isKindOf(view, "UITabBar")); + if (tabBar) { + return tabBar; + } + } + return null; + } + private uikitNode( view: any, includeHidden: boolean, @@ -761,7 +1101,7 @@ export class SimDeckNativeScriptInspector { debugDescription: stringValue(view), uikitScript: this.uikitScriptInfo(id, view), sourceLocation: nativeScriptView - ? sourceLocationForView(nativeScriptView) + ? sourceLocationForView(nativeScriptView, this.options.sourceRoot) : null, frame: rectValue(read(view, "frame")), bounds: rectValue(read(view, "bounds")), @@ -831,6 +1171,7 @@ export class SimDeckNativeScriptInspector { displayScale: numberValue(UIScreen.mainScreen.scale, 1), coordinateSpace: "screen-points", source, + sourceRoot: this.options.sourceRoot || undefined, }; } @@ -1090,6 +1431,335 @@ function isNativeScriptView(value: any): value is View { return Boolean(value && typeof value.eachChildView === "function"); } +function nativeScriptTabAccessoryView(tabView: View): View | null { + return ( + (read(tabView, "iosBottomAccessory") as View | null) || + (read(tabView, "_bottomAccessoryNsView") as View | null) || + null + ); +} + +function nativeScriptTabItems( + tabView: View, + nativeScriptChildrenNodes: JSONObject[] = [], + sourceRoot = "", +): JSONObject[] { + const rawItems = nsArray( + read(tabView, "items") || read(tabView, "_items") || read(tabView, "_tabItems"), + ); + const children = nativeScriptChildren(tabView); + const count = Math.max( + rawItems.length, + children.length, + nativeScriptChildrenNodes.length, + ); + return Array.from({ length: count }, (_, index) => { + const child = children[index] ?? null; + const childNode = nativeScriptChildrenNodes[index] ?? {}; + const item = + rawItems[index] || + read(child, "tabItem") || + read(child, "_tabItem") || + {}; + return { + title: stringValue(read(item, "title")) || stringValue(read(child, "title")), + iconSource: + stringValue(read(item, "iconSource")) || + stringValue(read(child, "iconSource")), + view: read(item, "view") || child, + sourceLocation: + sourceLocationForView(item, sourceRoot) || + sourceLocationForView(child, sourceRoot) || + sourceLocationFromNode(childNode), + }; + }); +} + +function tabItemSourceLocation( + tabView: View, + item: JSONObject, + index: number, + sourceRoot = "", +): JSONObject | null { + const direct = item.sourceLocation as JSONObject | null | undefined; + return ( + direct || + sourceLocationForView(item.view, sourceRoot) || + sourceLocationForView(nativeScriptChildAt(tabView, index), sourceRoot) + ); +} + +function nativeScriptChildAt(view: View, targetIndex: number): View | null { + return nativeScriptChildren(view)[targetIndex] ?? null; +} + +function nativeScriptChildren(view: View): View[] { + const result: View[] = []; + safeCall(() => { + view.eachChildView((child: View) => { + result.push(child); + return true; + }); + }, undefined); + return result; +} + +function sourceLocationFromNode(node: JSONObject): JSONObject | null { + const location = objectValue(node.sourceLocation); + if (location.file) { + return location; + } + const file = stringValue(node.sourceFile); + if (!file) { + return null; + } + const locationFromFields: JSONObject = { file }; + if (typeof node.sourceLine === "number") { + locationFromFields.line = node.sourceLine; + } + if (typeof node.sourceColumn === "number") { + locationFromFields.column = node.sourceColumn; + } + return locationFromFields; +} + +function firstSourceLocationFromNodes(nodes: JSONObject[]): JSONObject | null { + for (const node of nodes) { + const location = sourceLocationFromNode(node); + if (location) { + return location; + } + const childLocation = firstSourceLocationFromNodes( + Array.isArray(node.children) ? (node.children as JSONObject[]) : [], + ); + if (childLocation) { + return childLocation; + } + } + return null; +} + +function patchNativeScriptFramesFromUIKit( + node: JSONObject, + uikitRoot: any, +): JSONObject { + const candidates = collectSubviews( + uikitRoot, + (view) => Boolean(uikitFrameLabel(view)) && hasUsableFrame(frameInScreen(view)), + ).map((view) => ({ + frame: frameInScreen(view), + label: uikitFrameLabel(view), + })); + const used = new Set(); + return patchNodeFramesFromUIKitCandidates(node, candidates, used); +} + +function patchNodeFramesFromUIKitCandidates( + node: JSONObject, + candidates: { frame: JSONObject | null; label: string }[], + used: Set, +): JSONObject { + const patched: JSONObject = { ...node }; + const label = nodeFrameLabel(patched); + if (!hasUsableFrame(patched.frame as JSONObject | null) && label) { + const matchIndex = candidates.findIndex( + (candidate, index) => !used.has(index) && candidate.label === label, + ); + if (matchIndex >= 0) { + used.add(matchIndex); + patched.frame = candidates[matchIndex].frame; + } + } + if (Array.isArray(patched.children)) { + patched.children = (patched.children as JSONObject[]).map((child) => + patchNodeFramesFromUIKitCandidates(child, candidates, used), + ); + } + return patched; +} + +function nodeFrameLabel(node: JSONObject): string { + return ( + stringValue(node.text) || + stringValue(node.title) || + stringValue(node.AXLabel) + ); +} + +function uikitFrameLabel(view: any): string { + const accessibility = accessibilityInfo(view); + return ( + textValue(view) || + stringValue(accessibility.label) || + stringValue(read(view, "accessibilityIdentifier")) + ); +} + +function findSubview( + view: any, + predicate: (view: any) => boolean, +): any | null { + if (!view) { + return null; + } + if (predicate(view)) { + return view; + } + for (const child of nsArray(read(view, "subviews"))) { + const match = findSubview(child, predicate); + if (match) { + return match; + } + } + return null; +} + +function collectSubviews( + view: any, + predicate: (view: any) => boolean, + result: any[] = [], +): any[] { + for (const child of nsArray(read(view, "subviews"))) { + if (predicate(child)) { + result.push(child); + } + collectSubviews(child, predicate, result); + } + return result; +} + +function tabBarControls(tabBar: any): any[] { + const controls = collectSubviews( + tabBar, + (view) => + isKindOf(view, "UIControl") && + isVisible(view) && + hasUsableFrame(frameInScreen(view)), + ); + const byFrame = new Map(); + for (const control of controls) { + const frame = frameInScreen(control); + if (!frame) { + continue; + } + byFrame.set(rectKey(frame), control); + } + return preferLargestNonOverlappingControls([...byFrame.values()]).sort((left, right) => { + const leftFrame = frameInScreen(left); + const rightFrame = frameInScreen(right); + return rectNumber(leftFrame, "x") - rectNumber(rightFrame, "x"); + }); +} + +function preferLargestNonOverlappingControls(controls: any[]): any[] { + const accepted: any[] = []; + for (const control of [...controls].sort( + (left, right) => rectArea(frameInScreen(right)) - rectArea(frameInScreen(left)), + )) { + const frame = frameInScreen(control); + if ( + !frame || + accepted.some((other) => substantiallyOverlaps(frame, frameInScreen(other))) + ) { + continue; + } + accepted.push(control); + } + return accepted; +} + +function substantiallyOverlaps( + frame: JSONObject, + otherFrame: JSONObject | null, +): boolean { + if (!otherFrame) { + return false; + } + const overlap = rectOverlapArea(frame, otherFrame); + const smallerArea = Math.min(rectArea(frame), rectArea(otherFrame)); + return smallerArea > 0 && overlap / smallerArea > 0.6; +} + +function rectArea(frame: JSONObject | null | undefined): number { + return rectNumber(frame, "width") * rectNumber(frame, "height"); +} + +function rectOverlapArea(left: JSONObject, right: JSONObject): number { + const x1 = Math.max(rectNumber(left, "x"), rectNumber(right, "x")); + const y1 = Math.max(rectNumber(left, "y"), rectNumber(right, "y")); + const x2 = Math.min( + rectNumber(left, "x") + rectNumber(left, "width"), + rectNumber(right, "x") + rectNumber(right, "width"), + ); + const y2 = Math.min( + rectNumber(left, "y") + rectNumber(left, "height"), + rectNumber(right, "y") + rectNumber(right, "height"), + ); + return Math.max(0, x2 - x1) * Math.max(0, y2 - y1); +} + +function tabBarControlLabel(control: any): string { + const accessibility = accessibilityInfo(control); + const label = stringValue(accessibility.label); + if (label) { + return label; + } + const text = textValue(control); + if (text) { + return text; + } + const labelView = findSubview( + control, + (view) => isKindOf(view, "UILabel") && Boolean(textValue(view)), + ); + return labelView ? textValue(labelView) : ""; +} + +function hasUsableFrame(frame: JSONObject | null): boolean { + return rectNumber(frame, "width") > 1 && rectNumber(frame, "height") > 1; +} + +function rectKey(frame: JSONObject): string { + return ["x", "y", "width", "height"] + .map((key) => Math.round(rectNumber(frame, key))) + .join(","); +} + +function rectNumber(frame: JSONObject | null | undefined, key: string): number { + const value = frame?.[key]; + return typeof value === "number" && Number.isFinite(value) ? value : 0; +} + +function fallbackTabBarFrame(): JSONObject { + const screen = rectValue(UIScreen.mainScreen.bounds) ?? { + x: 0, + y: 0, + width: 0, + height: 0, + }; + const height = Math.min(83, Math.max(49, rectNumber(screen, "height") * 0.095)); + return { + x: rectNumber(screen, "x"), + y: rectNumber(screen, "y") + rectNumber(screen, "height") - height, + width: rectNumber(screen, "width"), + height, + }; +} + +function tabItemFallbackFrame( + tabBarFrame: JSONObject, + count: number, + index: number, +): JSONObject { + const width = count > 0 ? rectNumber(tabBarFrame, "width") / count : 0; + return { + x: rectNumber(tabBarFrame, "x") + width * index, + y: rectNumber(tabBarFrame, "y"), + width, + height: rectNumber(tabBarFrame, "height"), + }; +} + function isNativeScriptVisible(view: any): boolean { return read(view, "visibility") !== "collapse"; } @@ -1283,6 +1953,28 @@ function moduleName(name: string): string | null { return name.includes(".") ? name.split(".")[0] : null; } +function infoAnnouncementKey(info: JSONObject): string { + const appHierarchy = objectValue(info.appHierarchy); + const nativeScript = objectValue(info.nativeScript); + return JSON.stringify({ + processIdentifier: info.processIdentifier, + bundleIdentifier: info.bundleIdentifier, + sourceRoot: info.sourceRoot, + appHierarchyAvailable: Boolean(appHierarchy.available), + nativeScriptAvailable: Boolean(nativeScript.available), + }); +} + +function nativeScriptAvailable(info: JSONObject): boolean { + const appHierarchy = objectValue(info.appHierarchy); + const nativeScript = objectValue(info.nativeScript); + return Boolean(appHierarchy.available || nativeScript.available); +} + +function objectValue(value: unknown): JSONObject { + return value && typeof value === "object" ? (value as JSONObject) : {}; +} + function stringValue(value: unknown): string { if (value == null) { return ""; @@ -1523,7 +2215,28 @@ function nativeScriptViewType(view: View): string { return stringValue((view as any).typeName) || constructorName(view); } -function sourceLocationForView(view: any): JSONObject | null { +function normalizeSourceRoot(sourceRoot: string | undefined): string { + const root = stringValue(sourceRoot).replace(/\/+$/, ""); + return isAbsoluteSourceFile(root) ? root : ""; +} + +function absoluteSourceFile(file: string, sourceRoot: string): string { + const cleanFile = stringValue(file); + if (!cleanFile || isAbsoluteSourceFile(cleanFile) || !sourceRoot) { + return cleanFile; + } + return `${sourceRoot}/${cleanFile.replace(/^\.?\//, "")}`; +} + +function isAbsoluteSourceFile(file: string): boolean { + return ( + file.startsWith("/") || + /^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(file) || + /^[A-Za-z]:[\\/]/.test(file) + ); +} + +function sourceLocationForView(view: any, sourceRoot = ""): JSONObject | null { const raw = safeCall( () => view.getAttribute?.(angularSourceLocationAttribute), null, @@ -1534,9 +2247,11 @@ function sourceLocationForView(view: any): JSONObject | null { } const separator = value.lastIndexOf("@"); if (separator <= 0) { - return { file: value }; + return { file: absoluteSourceFile(value, sourceRoot) }; } - const location: JSONObject = { file: value.slice(0, separator) }; + const location: JSONObject = { + file: absoluteSourceFile(value.slice(0, separator), sourceRoot), + }; for (const part of value.slice(separator + 1).split(",")) { const [key, rawNumber] = part.split(":"); const parsed = Number(rawNumber); @@ -1554,17 +2269,94 @@ function sourceLocationForView(view: any): JSONObject | null { return location; } -function nativeScriptTitle(view: View): string { +function nativeScriptInfo( + view: View, + type: string, + sourceRoot = "", +): JSONObject { + const anyView = view as any; + const sourceLocation = sourceLocationForView(view, sourceRoot); + return { + id: stringValue(anyView.id), + className: stringValue(anyView.className), + type, + sourceLocation, + sourceFile: sourceString(sourceLocation, "file"), + sourceLine: sourceNumber(sourceLocation, "line"), + sourceColumn: sourceNumber(sourceLocation, "column"), + text: cleanGeneratedNativeScriptText(stringValue(anyView.text), type), + title: cleanGeneratedNativeScriptText(stringValue(anyView.title), type), + src: nativeScriptImageSource(view, type), + testID: + stringValue(read(view, "testID")) || + stringValue(read(view, "testId")) || + stringValue(read(view, "automationText")), + accessibilityLabel: stringValue(read(view, "accessibilityLabel")), + accessibilityHint: stringValue(read(view, "accessibilityHint")), + accessibilityValue: stringValue(read(view, "accessibilityValue")), + }; +} + +function nativeScriptTitle(view: View, type: string): string { const anyView = view as any; return ( - stringValue(anyView.text) || - stringValue(anyView.title) || + cleanGeneratedNativeScriptText(stringValue(anyView.text), type) || + cleanGeneratedNativeScriptText(stringValue(anyView.title), type) || stringValue(anyView.id) || - stringValue(anyView.typeName) || - constructorName(view) + imageLabelFromSource(nativeScriptImageSource(view, type)) ); } +function nativeScriptImageSource(view: View, type: string): string { + if (!isNativeScriptImageType(type)) { + return ""; + } + return ( + stringValue(read(view, "src")) || stringValue(read(view, "imageSource")) + ); +} + +function isNativeScriptImageType(type: string): boolean { + return /image/i.test(type); +} + +function imageLabelFromSource(source: string): string { + if (!source) { + return ""; + } + const fileName = source.split(/[\\/]/).pop() ?? source; + return fileName.replace(/\.[A-Za-z0-9]+$/, ""); +} + +function cleanGeneratedNativeScriptText(value: string, type: string): string { + const trimmed = value.trim(); + if (!trimmed) { + return ""; + } + const generated = new Set([ + type, + constructorName({ constructor: { name: type } }), + `_${type}`, + ]); + return generated.has(trimmed) ? "" : trimmed; +} + +function sourceString( + location: JSONObject | null | undefined, + key: string, +): string | null { + const value = location?.[key]; + return typeof value === "string" && value ? value : null; +} + +function sourceNumber( + location: JSONObject | null | undefined, + key: string, +): number | null { + const value = location?.[key]; + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + function tap(target: any): void { if (isKindOf(target, "UISwitch")) { target.setOnAnimated(!Boolean(read(target, "on")), true); diff --git a/packages/react-native-inspector/README.md b/packages/react-native-inspector/README.md index 5446fa73..c4f830ef 100644 --- a/packages/react-native-inspector/README.md +++ b/packages/react-native-inspector/README.md @@ -28,7 +28,7 @@ import App from "./App"; AppRegistry.registerComponent("Example", () => App); ``` -The auto entrypoint no-ops outside development, reads `EXPO_PUBLIC_SIMDECK_PORT` when present, and otherwise scans common SimDeck daemon ports. +The auto entrypoint no-ops outside development, reads `EXPO_PUBLIC_SIMDECK_PORT` when present, and otherwise scans common SimDeck daemon ports. Set `EXPO_PUBLIC_SIMDECK_SOURCE_ROOT` when Metro reports project-relative source paths. If you need explicit options, call the manual API before registering the app: @@ -38,7 +38,10 @@ import { startSimDeckReactNativeInspector } from "react-native-simdeck"; import App from "./App"; if (__DEV__) { - startSimDeckReactNativeInspector({ port: 4310 }); + startSimDeckReactNativeInspector({ + port: 4310, + sourceRoot: "/absolute/path/to/your/app", + }); } AppRegistry.registerComponent("Example", () => App); @@ -47,7 +50,7 @@ AppRegistry.registerComponent("Example", () => App); ## What It Exposes - React component hierarchy from the React Fiber tree. -- `sourceLocation` from React dev-mode `_debugSource` metadata when Metro includes it. +- `sourceLocation` from React dev-mode `_debugSource` metadata when Metro includes it. Pass `sourceRoot` to publish absolute `sourceLocation.file` values. - Host component tags and measured screen frames when React Native can resolve them. - Best-effort `View.getProperties`, `View.setProperty`, and `View.evaluateScript` for debug sessions. diff --git a/packages/react-native-inspector/src/auto.ts b/packages/react-native-inspector/src/auto.ts index 0fcfd0ba..0b9c88bf 100644 --- a/packages/react-native-inspector/src/auto.ts +++ b/packages/react-native-inspector/src/auto.ts @@ -7,10 +7,12 @@ const defaultPorts = Array.from({ length: 11 }, (_, index) => 4310 + index); if (typeof __DEV__ !== "undefined" && __DEV__) { const envPort = Number(process?.env?.EXPO_PUBLIC_SIMDECK_PORT); + const sourceRoot = process?.env?.EXPO_PUBLIC_SIMDECK_SOURCE_ROOT; startSimDeckReactNativeInspector({ ports: Number.isInteger(envPort) && envPort > 0 ? [envPort, ...defaultPorts] : defaultPorts, + sourceRoot, }); } diff --git a/packages/react-native-inspector/src/index.ts b/packages/react-native-inspector/src/index.ts index f69bdb4a..ad183e36 100644 --- a/packages/react-native-inspector/src/index.ts +++ b/packages/react-native-inspector/src/index.ts @@ -11,6 +11,7 @@ export interface SimDeckReactNativeInspectorOptions { ports?: number[]; reconnect?: boolean; secure?: boolean; + sourceRoot?: string; } type NormalizedSimDeckReactNativeInspectorOptions = Required< @@ -158,6 +159,7 @@ export class SimDeckReactNativeInspector { ports, reconnect: options.reconnect ?? true, secure: options.secure ?? false, + sourceRoot: normalizeSourceRoot(options.sourceRoot), }; } @@ -370,6 +372,7 @@ export class SimDeckReactNativeInspector { displayScale: metadata.displayScale, screenBounds: metadata.screenBounds, coordinateSpace: "screen-points", + sourceRoot: this.options.sourceRoot || undefined, methods: [ "Runtime.ping", "Inspector.getInfo", @@ -712,7 +715,10 @@ export class SimDeckReactNativeInspector { if (this.sourceLocationCache.has(key)) { return Promise.resolve(this.sourceLocationCache.get(key) ?? null); } - const immediate = immediateSourceLocationForFiber(fiber); + const immediate = normalizeSourceLocation( + immediateSourceLocationForFiber(fiber), + this.options.sourceRoot, + ); if (immediate) { this.sourceLocationCache.set(key, immediate); return Promise.resolve(immediate); @@ -721,8 +727,12 @@ export class SimDeckReactNativeInspector { this.pendingSourceLocations.add(key); void resolveSourceLocationForFiber(fiber).then((location) => { this.pendingSourceLocations.delete(key); - if (location) { - this.sourceLocationCache.set(key, location); + const normalized = normalizeSourceLocation( + location, + this.options.sourceRoot, + ); + if (normalized) { + this.sourceLocationCache.set(key, normalized); } }); } @@ -779,6 +789,7 @@ export class SimDeckReactNativeInspector { bundleIdentifier: metadata.bundleIdentifier, displayScale: metadata.displayScale, coordinateSpace: "screen-points", + sourceRoot: this.options.sourceRoot || undefined, }; } @@ -1056,6 +1067,39 @@ function fiberDisplayName(fiber: Fiber): string { return fiber.tag === 6 ? "Text" : "Component"; } +function normalizeSourceRoot(sourceRoot: string | undefined): string { + const root = stringOrNull(sourceRoot)?.replace(/\/+$/, "") ?? ""; + return isAbsoluteSourceFile(root) ? root : ""; +} + +function normalizeSourceLocation( + location: JSONObject | null | undefined, + sourceRoot: string, +): JSONObject | null { + if (!location?.file) { + return null; + } + return { + ...location, + file: absoluteSourceFile(String(location.file), sourceRoot), + }; +} + +function absoluteSourceFile(file: string, sourceRoot: string): string { + if (!file || isAbsoluteSourceFile(file) || !sourceRoot) { + return file; + } + return `${sourceRoot}/${file.replace(/^\.?\//, "")}`; +} + +function isAbsoluteSourceFile(file: string): boolean { + return ( + file.startsWith("/") || + /^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(file) || + /^[A-Za-z]:[\\/]/.test(file) + ); +} + async function resolveSourceLocationForFiber( fiber: Fiber, ): Promise { diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index 1b7f0b19..6e74fe80 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -5159,6 +5159,7 @@ fn inspector_metadata( "port": port, "processIdentifier": process_identifier, "protocolVersion": info.get("protocolVersion").cloned().unwrap_or(Value::Null), + "sourceRoot": hierarchy.get("sourceRoot").cloned().unwrap_or_else(|| info.get("sourceRoot").cloned().unwrap_or(Value::Null)), "transport": transport_name, }) } @@ -5417,18 +5418,24 @@ fn normalize_inspector_node(node: &Value, pid: Option) -> Value { let display_name = object_string(object, "displayName") .or_else(|| object_string(object, "type")) .unwrap_or_else(|| class_name.clone()); + let source = object_string(object, "source").unwrap_or_else(|| "in-app-inspector".to_owned()); let inspector_id = object_string(object, "id"); - let accessibility_label = nested_string(accessibility, "label"); + let accessibility_label = + object_string(object, "AXLabel").or_else(|| nested_string(accessibility, "label")); let text = object_string(object, "text"); let placeholder = object_string(object, "placeholder"); let swiftui_tag = nested_string(swiftui, "tag"); let view_controller_title = nested_string(view_controller, "title"); - let image_name = object_string(object, "imageName"); + let image_name = object_string(object, "imageName").filter(|_| { + source != "nativescript" || display_name.to_ascii_lowercase().contains("image") + }); + let object_title = object_string(object, "title").filter(|title| title != &display_name); let title = first_non_empty_string([ swiftui_tag.clone(), + object_title, text.clone(), view_controller_title, - image_name, + image_name.clone(), Some(display_name.clone()), ]); let role = if nested_bool(swiftui, "isProbe").unwrap_or(false) { @@ -5456,10 +5463,7 @@ fn normalize_inspector_node(node: &Value, pid: Option) -> Value { normalized.insert("role".to_owned(), Value::String(role.to_owned())); normalized.insert("title".to_owned(), Value::String(title)); normalized.insert("children".to_owned(), Value::Array(children)); - normalized.insert( - "source".to_owned(), - Value::String("in-app-inspector".to_owned()), - ); + normalized.insert("source".to_owned(), Value::String(source)); if let Some(value) = inspector_id { normalized.insert("AXUniqueId".to_owned(), Value::String(value.clone())); @@ -5473,10 +5477,15 @@ fn normalize_inspector_node(node: &Value, pid: Option) -> Value { if let Some(value) = accessibility_label.or(text.clone()) { normalized.insert("AXLabel".to_owned(), Value::String(value)); } - if let Some(value) = nested_string(accessibility, "value").or(placeholder.clone()) { + if let Some(value) = object_string(object, "AXValue") + .or_else(|| nested_string(accessibility, "value")) + .or(placeholder.clone()) + { normalized.insert("AXValue".to_owned(), Value::String(value)); } - if let Some(value) = nested_string(accessibility, "hint") { + if let Some(value) = + object_string(object, "help").or_else(|| nested_string(accessibility, "hint")) + { normalized.insert("help".to_owned(), Value::String(value)); } if let Some(frame) = object @@ -5519,9 +5528,15 @@ fn normalize_inspector_node(node: &Value, pid: Option) -> Value { copy_optional_field(object, &mut normalized, "nativeScript"); copy_optional_field(object, &mut normalized, "uikitScript"); copy_optional_field(object, &mut normalized, "sourceLocation"); + copy_optional_field(object, &mut normalized, "sourceLocations"); + copy_optional_field(object, &mut normalized, "sourceFile"); + copy_optional_field(object, &mut normalized, "sourceLine"); + copy_optional_field(object, &mut normalized, "sourceColumn"); copy_optional_field(object, &mut normalized, "text"); copy_optional_field(object, &mut normalized, "placeholder"); - copy_optional_field(object, &mut normalized, "imageName"); + if let Some(image_name) = image_name { + normalized.insert("imageName".to_owned(), Value::String(image_name)); + } if let Some(enabled) = view_enabled(object) { normalized.insert("enabled".to_owned(), Value::Bool(enabled)); From bf35622f3db138527a57f10fd7c6f5fce80b0cfe Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 24 May 2026 01:58:03 -0400 Subject: [PATCH 2/2] style: format accessibility inspector changes --- client/src/app/AppShell.tsx | 19 ++-- .../accessibility/AccessibilityInspector.tsx | 98 ++++++++++--------- .../accessibility/AccessibilityOverlay.tsx | 56 +++++------ packages/nativescript-inspector/src/index.ts | 46 ++++++--- 4 files changed, 117 insertions(+), 102 deletions(-) diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index aac5b0f9..29fd9775 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -1430,16 +1430,15 @@ export function AppShell({ ); useEffect(() => { - const refreshMs = - hierarchyVisible - ? accessibilityPreferredSource === "react-native" || - accessibilitySource === "react-native" - ? REACT_NATIVE_ACCESSIBILITY_REFRESH_MS - : accessibilityPreferredSource === "flutter" || - accessibilitySource === "flutter" - ? FLUTTER_ACCESSIBILITY_REFRESH_MS - : ACCESSIBILITY_REFRESH_MS - : ACCESSIBILITY_BACKGROUND_REFRESH_MS; + const refreshMs = hierarchyVisible + ? accessibilityPreferredSource === "react-native" || + accessibilitySource === "react-native" + ? REACT_NATIVE_ACCESSIBILITY_REFRESH_MS + : accessibilityPreferredSource === "flutter" || + accessibilitySource === "flutter" + ? FLUTTER_ACCESSIBILITY_REFRESH_MS + : ACCESSIBILITY_REFRESH_MS + : ACCESSIBILITY_BACKGROUND_REFRESH_MS; let disposed = false; let timeout: number | null = null; const refreshLoop = async () => { diff --git a/client/src/features/accessibility/AccessibilityInspector.tsx b/client/src/features/accessibility/AccessibilityInspector.tsx index f9b43b26..e4a91f42 100644 --- a/client/src/features/accessibility/AccessibilityInspector.tsx +++ b/client/src/features/accessibility/AccessibilityInspector.tsx @@ -514,55 +514,57 @@ function NodeDetails({ const isAndroid = isAndroidSimulator(selectedSimulator); const sourceText = sourceLocationText(node); const sourceHref = sourceLocationHref(node); - const details = ([ - ["Type", accessibilityKind(node)], - ["Label", primaryAccessibilityText(node)], + const details = ( [ - "Source", - sourceHref ? ( - - {sourceText} - - ) : ( - sourceText - ), - ], - [ - isAndroid ? "Resource ID" : "Identifier", - isAndroid - ? (node.androidResourceId ?? "") - : accessibilityIdentifier(node), - ], - ["Inspector ID", node.inspectorId ?? ""], - ["Module", node.moduleName ?? ""], - ["NativeScript", nativeScriptDescription(node.nativeScript)], - ["React Native", reactNativeDescription(node.reactNative)], - ["Flutter", flutterDescription(node.flutter)], - [isAndroid ? "Android Class" : "UIKit Class", node.className ?? ""], - ["Package", isAndroid ? (node.androidPackage ?? "") : ""], - ["Last JS", lastUIKitScriptText(node)], - ["Value", node.AXValue ?? ""], - ["Role", node.role ?? ""], - ["Role Description", node.role_description ?? ""], - ["View Controller", objectClassName(node.viewController)], - ["SwiftUI", swiftUIDescription(node.swiftUI)], - ["Enabled", node.enabled == null ? "" : node.enabled ? "true" : "false"], - ["Hidden", node.isHidden == null ? "" : node.isHidden ? "true" : "false"], - ["Clickable", boolDetail(isAndroid, node.clickable)], - ["Long Clickable", boolDetail(isAndroid, node.longClickable)], - ["Focusable", boolDetail(isAndroid, node.focusable)], - ["Focused", boolDetail(isAndroid, node.focused)], - ["Scrollable", boolDetail(isAndroid, node.scrollable)], - ["Checkable", boolDetail(isAndroid, node.checkable)], - ["Checked", boolDetail(isAndroid, node.checked)], - ["Selected", boolDetail(isAndroid, node.selected)], - ["Password", boolDetail(isAndroid, node.password)], - ["Alpha", node.alpha == null ? "" : String(round(node.alpha))], - ["Frame", validFrame(node.frame) ? frameText(node.frame) : ""], - ["PID", node.pid == null ? "" : String(node.pid)], - ["Actions", node.custom_actions?.join(", ") ?? ""], - ["Help", node.help ?? ""], - ] as Array<[string, DetailValue]>).filter(([, value]) => value); + ["Type", accessibilityKind(node)], + ["Label", primaryAccessibilityText(node)], + [ + "Source", + sourceHref ? ( + + {sourceText} + + ) : ( + sourceText + ), + ], + [ + isAndroid ? "Resource ID" : "Identifier", + isAndroid + ? (node.androidResourceId ?? "") + : accessibilityIdentifier(node), + ], + ["Inspector ID", node.inspectorId ?? ""], + ["Module", node.moduleName ?? ""], + ["NativeScript", nativeScriptDescription(node.nativeScript)], + ["React Native", reactNativeDescription(node.reactNative)], + ["Flutter", flutterDescription(node.flutter)], + [isAndroid ? "Android Class" : "UIKit Class", node.className ?? ""], + ["Package", isAndroid ? (node.androidPackage ?? "") : ""], + ["Last JS", lastUIKitScriptText(node)], + ["Value", node.AXValue ?? ""], + ["Role", node.role ?? ""], + ["Role Description", node.role_description ?? ""], + ["View Controller", objectClassName(node.viewController)], + ["SwiftUI", swiftUIDescription(node.swiftUI)], + ["Enabled", node.enabled == null ? "" : node.enabled ? "true" : "false"], + ["Hidden", node.isHidden == null ? "" : node.isHidden ? "true" : "false"], + ["Clickable", boolDetail(isAndroid, node.clickable)], + ["Long Clickable", boolDetail(isAndroid, node.longClickable)], + ["Focusable", boolDetail(isAndroid, node.focusable)], + ["Focused", boolDetail(isAndroid, node.focused)], + ["Scrollable", boolDetail(isAndroid, node.scrollable)], + ["Checkable", boolDetail(isAndroid, node.checkable)], + ["Checked", boolDetail(isAndroid, node.checked)], + ["Selected", boolDetail(isAndroid, node.selected)], + ["Password", boolDetail(isAndroid, node.password)], + ["Alpha", node.alpha == null ? "" : String(round(node.alpha))], + ["Frame", validFrame(node.frame) ? frameText(node.frame) : ""], + ["PID", node.pid == null ? "" : String(node.pid)], + ["Actions", node.custom_actions?.join(", ") ?? ""], + ["Help", node.help ?? ""], + ] as Array<[string, DetailValue]> + ).filter(([, value]) => value); return (
diff --git a/client/src/features/accessibility/AccessibilityOverlay.tsx b/client/src/features/accessibility/AccessibilityOverlay.tsx index 8a427cfc..70e0ab30 100644 --- a/client/src/features/accessibility/AccessibilityOverlay.tsx +++ b/client/src/features/accessibility/AccessibilityOverlay.tsx @@ -148,34 +148,34 @@ function AccessibilityDomNode({ const tagName = accessibilityDomTagName(node); return createElement(tagName, { - "aria-checked": - role === "checkbox" || role === "switch" - ? (node.checked ?? undefined) - : undefined, - "aria-label": label, - "aria-level": depth + 1, - "aria-selected": node.selected ?? undefined, - className: "accessibility-dom-node", - "data-testid": `simdeck-accessibility-${id}`, - "data-simdeck-accessibility-id": id, - "data-simdeck-accessibility-component": kind, - "data-simdeck-accessibility-identifier": - accessibilityIdentifier(node) || undefined, - "data-simdeck-accessibility-kind": kind, - "data-simdeck-accessibility-label": primaryAccessibilityText(node), - "data-simdeck-accessibility-image": metadata.imageName, - "data-simdeck-accessibility-source-file": metadata.sourceFile, - "data-simdeck-accessibility-source-line": metadata.sourceLine, - "data-simdeck-accessibility-source-column": metadata.sourceColumn, - "data-simdeck-accessibility-source": node.source || undefined, - "data-simdeck-accessibility-state": metadata.state, - "data-simdeck-accessibility-value": metadata.value, - "data-simdeck-inspector-id": node.inspectorId || undefined, - "data-simdeck-uikit-id": node.uikitId || undefined, - title: label, - role, - style: frameStyle(node.frame, rootFrame), - }); + "aria-checked": + role === "checkbox" || role === "switch" + ? (node.checked ?? undefined) + : undefined, + "aria-label": label, + "aria-level": depth + 1, + "aria-selected": node.selected ?? undefined, + className: "accessibility-dom-node", + "data-testid": `simdeck-accessibility-${id}`, + "data-simdeck-accessibility-id": id, + "data-simdeck-accessibility-component": kind, + "data-simdeck-accessibility-identifier": + accessibilityIdentifier(node) || undefined, + "data-simdeck-accessibility-kind": kind, + "data-simdeck-accessibility-label": primaryAccessibilityText(node), + "data-simdeck-accessibility-image": metadata.imageName, + "data-simdeck-accessibility-source-file": metadata.sourceFile, + "data-simdeck-accessibility-source-line": metadata.sourceLine, + "data-simdeck-accessibility-source-column": metadata.sourceColumn, + "data-simdeck-accessibility-source": node.source || undefined, + "data-simdeck-accessibility-state": metadata.state, + "data-simdeck-accessibility-value": metadata.value, + "data-simdeck-inspector-id": node.inspectorId || undefined, + "data-simdeck-uikit-id": node.uikitId || undefined, + title: label, + role, + style: frameStyle(node.frame, rootFrame), + }); } function frameStyle( diff --git a/packages/nativescript-inspector/src/index.ts b/packages/nativescript-inspector/src/index.ts index c76216d8..b56cf36d 100644 --- a/packages/nativescript-inspector/src/index.ts +++ b/packages/nativescript-inspector/src/index.ts @@ -844,7 +844,12 @@ export class SimDeckNativeScriptInspector { : null; let child = maxDepth == null || depth < maxDepth - ? this.nativeScriptNode(accessoryView, includeHidden, maxDepth, depth + 1) + ? this.nativeScriptNode( + accessoryView, + includeHidden, + maxDepth, + depth + 1, + ) : null; if (child && uikitAccessory) { child = patchNativeScriptFramesFromUIKit(child, uikitAccessory); @@ -1445,7 +1450,9 @@ function nativeScriptTabItems( sourceRoot = "", ): JSONObject[] { const rawItems = nsArray( - read(tabView, "items") || read(tabView, "_items") || read(tabView, "_tabItems"), + read(tabView, "items") || + read(tabView, "_items") || + read(tabView, "_tabItems"), ); const children = nativeScriptChildren(tabView); const count = Math.max( @@ -1462,7 +1469,8 @@ function nativeScriptTabItems( read(child, "_tabItem") || {}; return { - title: stringValue(read(item, "title")) || stringValue(read(child, "title")), + title: + stringValue(read(item, "title")) || stringValue(read(child, "title")), iconSource: stringValue(read(item, "iconSource")) || stringValue(read(child, "iconSource")), @@ -1545,7 +1553,8 @@ function patchNativeScriptFramesFromUIKit( ): JSONObject { const candidates = collectSubviews( uikitRoot, - (view) => Boolean(uikitFrameLabel(view)) && hasUsableFrame(frameInScreen(view)), + (view) => + Boolean(uikitFrameLabel(view)) && hasUsableFrame(frameInScreen(view)), ).map((view) => ({ frame: frameInScreen(view), label: uikitFrameLabel(view), @@ -1595,10 +1604,7 @@ function uikitFrameLabel(view: any): string { ); } -function findSubview( - view: any, - predicate: (view: any) => boolean, -): any | null { +function findSubview(view: any, predicate: (view: any) => boolean): any | null { if (!view) { return null; } @@ -1644,22 +1650,27 @@ function tabBarControls(tabBar: any): any[] { } byFrame.set(rectKey(frame), control); } - return preferLargestNonOverlappingControls([...byFrame.values()]).sort((left, right) => { - const leftFrame = frameInScreen(left); - const rightFrame = frameInScreen(right); - return rectNumber(leftFrame, "x") - rectNumber(rightFrame, "x"); - }); + return preferLargestNonOverlappingControls([...byFrame.values()]).sort( + (left, right) => { + const leftFrame = frameInScreen(left); + const rightFrame = frameInScreen(right); + return rectNumber(leftFrame, "x") - rectNumber(rightFrame, "x"); + }, + ); } function preferLargestNonOverlappingControls(controls: any[]): any[] { const accepted: any[] = []; for (const control of [...controls].sort( - (left, right) => rectArea(frameInScreen(right)) - rectArea(frameInScreen(left)), + (left, right) => + rectArea(frameInScreen(right)) - rectArea(frameInScreen(left)), )) { const frame = frameInScreen(control); if ( !frame || - accepted.some((other) => substantiallyOverlaps(frame, frameInScreen(other))) + accepted.some((other) => + substantiallyOverlaps(frame, frameInScreen(other)), + ) ) { continue; } @@ -1737,7 +1748,10 @@ function fallbackTabBarFrame(): JSONObject { width: 0, height: 0, }; - const height = Math.min(83, Math.max(49, rectNumber(screen, "height") * 0.095)); + const height = Math.min( + 83, + Math.max(49, rectNumber(screen, "height") * 0.095), + ); return { x: rectNumber(screen, "x"), y: rectNumber(screen, "y") + rectNumber(screen, "height") - height,