Skip to content

Commit

Permalink
iOS a11y dialogues and popups integration (#1091)
Browse files Browse the repository at this point in the history
## Proposed Changes

Integrate popup and dialogues with a11y.
Add `setEscapeEventListener` to `ComposeSceneLayer` for semantic escape,
aka "get me out of here if possible"-action incoming from
accessibility-based input.

## Testing

Test: N/A

---------

Co-authored-by: dima.avdeev <dima.avdeev@jetbrains.com>
  • Loading branch information
elijah-semyonov and dima-avdeev-jb committed Feb 13, 2024
1 parent 7c83de9 commit 8fd92b6
Show file tree
Hide file tree
Showing 5 changed files with 120 additions and 25 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ NS_ASSUME_NONNULL_BEGIN

- (BOOL)accessibilityScroll:(UIAccessibilityScrollDirection)direction CMP_MUST_BE_OVERRIDED;

- (BOOL)accessibilityPerformEscape;

@end

NS_ASSUME_NONNULL_END
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,10 @@ - (BOOL)accessibilityScroll:(UIAccessibilityScrollDirection)direction {
CMP_MUST_BE_OVERRIDED_INVARIANT_VIOLATION
}

- (BOOL)accessibilityPerformEscape {
return [super accessibilityPerformEscape];
}

- (void)accessibilityElementDidBecomeFocused {
[super accessibilityElementDidBecomeFocused];
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ import platform.UIKit.UIAccessibilityFocusedElement
import platform.UIKit.UIAccessibilityIsVoiceOverRunning
import platform.UIKit.UIAccessibilityLayoutChangedNotification
import platform.UIKit.UIAccessibilityPostNotification
import platform.UIKit.UIAccessibilityScreenChangedNotification
import platform.UIKit.UIAccessibilityScrollDirection
import platform.UIKit.UIAccessibilityScrollDirectionDown
import platform.UIKit.UIAccessibilityScrollDirectionLeft
Expand Down Expand Up @@ -473,9 +474,12 @@ private class AccessibilityElement(
if (config.contains(SemanticsProperties.InvisibleToUser)) {
false
} else {
// TODO: investigate if it can it be a traversal group _and_ contain properties that should
// TODO: investigate if it can it be one of those _and_ contain properties that should
// be communicated to iOS?
if (config.getOrNull(SemanticsProperties.IsTraversalGroup) == true) {
if (config.getOrNull(SemanticsProperties.IsTraversalGroup) == true
|| config.contains(SemanticsProperties.IsPopup)
|| config.contains(SemanticsProperties.IsDialog)
) {
false
} else {
config.containsImportantForAccessibility()
Expand Down Expand Up @@ -572,6 +576,21 @@ private class AccessibilityElement(
mediator.convertRectToWindowSpaceCGRect(semanticsNode.boundsInWindow)
}


override fun accessibilityPerformEscape(): Boolean {
if (!isAlive) {
mediator.debugLogger?.log("accessibilityPerformEscape() called after $semanticsNodeId was removed from the tree")
return false
}

if (mediator.performEscape()) {
UIAccessibilityPostNotification(UIAccessibilityScreenChangedNotification, null)
return true
} else {
return super.accessibilityPerformEscape()
}
}

// TODO: check the reference/value semantics for SemanticsNode, perhaps it doesn't need
// recreation at all
fun updateWithNewSemanticsNode(newSemanticsNode: SemanticsNode) {
Expand Down Expand Up @@ -828,15 +847,35 @@ sealed class AccessibilitySyncOptions(
class Always(debugLogger: AccessibilityDebugLogger?): AccessibilitySyncOptions(debugLogger = debugLogger)
}

@OptIn(ExperimentalComposeApi::class)
private val AccessibilitySyncOptions.shouldPerformSync
get() =
when (this) {
is AccessibilitySyncOptions.Never -> false
is AccessibilitySyncOptions.WhenRequiredByAccessibilityServices -> UIAccessibilityIsVoiceOverRunning()
is AccessibilitySyncOptions.Always -> true
}

@OptIn(ExperimentalComposeApi::class)
private val AccessibilitySyncOptions.debugLoggerIfEnabled: AccessibilityDebugLogger?
get() =
if (shouldPerformSync) {
debugLogger
} else {
null
}


/**
* A class responsible for mediating between the tree of specific SemanticsOwner and the iOS accessibility tree.
*/
@OptIn(ExperimentalComposeApi::class)
internal class AccessibilityMediator constructor(
internal class AccessibilityMediator(
val view: UIView,
private val owner: SemanticsOwner,
coroutineContext: CoroutineContext,
private val getAccessibilitySyncOptions: () -> AccessibilitySyncOptions,
val performEscape: () -> Boolean
) {
/**
* Indicates that this mediator was just created and the accessibility focus should be set on the
Expand All @@ -845,6 +884,10 @@ internal class AccessibilityMediator constructor(
private var needsInitialRefocusing = true
private var isAlive = true

/**
* Remembered [AccessibilityDebugLogger] after last sync, if logging is enabled according to
* [AccessibilitySyncOptions].
*/
var debugLogger: AccessibilityDebugLogger? = null
private set

Expand Down Expand Up @@ -873,6 +916,8 @@ internal class AccessibilityMediator constructor(
private val accessibilityElementsMap = mutableMapOf<Int, AccessibilityElement>()

init {
getAccessibilitySyncOptions().debugLoggerIfEnabled?.log("AccessibilityMediator for ${view} created")

val updateIntervalMillis = 50L
// TODO: this approach was copied from desktop implementation, obviously it has a [updateIntervalMillis] lag
// between the actual change in the semantics tree and the change in the accessibility tree.
Expand All @@ -883,13 +928,7 @@ internal class AccessibilityMediator constructor(

val syncOptions = getAccessibilitySyncOptions()

val shouldPerformSync = when (syncOptions) {
is AccessibilitySyncOptions.Never -> false
is AccessibilitySyncOptions.WhenRequiredByAccessibilityServices -> {
UIAccessibilityIsVoiceOverRunning()
}
is AccessibilitySyncOptions.Always -> true
}
val shouldPerformSync = syncOptions.shouldPerformSync

debugLogger = if (shouldPerformSync) {
syncOptions.debugLogger
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,9 @@ import kotlinx.cinterop.ObjCAction
import kotlinx.cinterop.readValue
import kotlinx.cinterop.useContents
import org.jetbrains.skia.Canvas
import org.jetbrains.skiko.SkikoKey
import org.jetbrains.skiko.SkikoKeyboardEvent
import org.jetbrains.skiko.SkikoKeyboardEventKind
import platform.CoreGraphics.CGAffineTransformIdentity
import platform.CoreGraphics.CGAffineTransformInvert
import platform.CoreGraphics.CGPoint
Expand Down Expand Up @@ -103,11 +105,20 @@ internal sealed interface SceneLayout {
class Bounds(val rect: IntRect) : SceneLayout
}

/**
* iOS specific-implementation of [PlatformContext.SemanticsOwnerListener] used to track changes in [SemanticsOwner].
*
* @property container The UI container associated with the semantics owner.
* @property coroutineContext The coroutine context to use for handling semantics changes.
* @property getAccessibilitySyncOptions A lambda function to retrieve the latest accessibility synchronization options.
* @property performEscape A lambda to delegate accessibility escape operation. Returns true if the escape was handled, false otherwise.
*/
@OptIn(ExperimentalComposeApi::class)
private class SemanticsOwnerListenerImpl(
private val container: UIView,
private val coroutineContext: CoroutineContext,
private val getAccessibilitySyncOptions: () -> AccessibilitySyncOptions
private val getAccessibilitySyncOptions: () -> AccessibilitySyncOptions,
private val performEscape: () -> Boolean
) : PlatformContext.SemanticsOwnerListener {
var current: Pair<SemanticsOwner, AccessibilityMediator>? = null

Expand All @@ -117,7 +128,8 @@ private class SemanticsOwnerListenerImpl(
container,
semanticsOwner,
coroutineContext,
getAccessibilitySyncOptions
getAccessibilitySyncOptions,
performEscape
)
}
}
Expand Down Expand Up @@ -197,7 +209,7 @@ internal class ComposeSceneMediator(
invalidate: () -> Unit,
platformContext: PlatformContext,
coroutineContext: CoroutineContext
) -> ComposeScene,
) -> ComposeScene
) {
private val focusable: Boolean get() = focusStack != null
private val keyboardOverlapHeightState: MutableState<Float> = mutableStateOf(0f)
Expand Down Expand Up @@ -262,9 +274,36 @@ internal class ComposeSceneMediator(

@OptIn(ExperimentalComposeApi::class)
private val semanticsOwnerListener by lazy {
SemanticsOwnerListenerImpl(rootView, coroutineContext, getAccessibilitySyncOptions = {
configuration.accessibilitySyncOptions
})
SemanticsOwnerListenerImpl(
rootView,
coroutineContext,
getAccessibilitySyncOptions = {
configuration.accessibilitySyncOptions
},
performEscape = {
val down = onKeyboardEvent(
KeyEvent(
SkikoKeyboardEvent(
SkikoKey.KEY_ESCAPE,
kind = SkikoKeyboardEventKind.DOWN,
platform = null
)
)
)

val up = onKeyboardEvent(
KeyEvent(
SkikoKeyboardEvent(
SkikoKey.KEY_ESCAPE,
kind = SkikoKeyboardEventKind.UP,
platform = null
)
)
)

down || up
}
)
}

private val platformContext: PlatformContext by lazy {
Expand All @@ -291,10 +330,7 @@ internal class ComposeSceneMediator(
private val keyboardEventHandler: KeyboardEventHandler by lazy {
object : KeyboardEventHandler {
override fun onKeyboardEvent(event: SkikoKeyboardEvent) {
val composeEvent = KeyEvent(event)
if (!uiKitTextInputService.onPreviewKeyEvent(composeEvent)) {
scene.sendKeyEvent(composeEvent)
}
onKeyboardEvent(KeyEvent(event))
}
}
}
Expand Down Expand Up @@ -355,6 +391,9 @@ internal class ComposeSceneMediator(
)
}

var density by scene::density
var layoutDirection by scene::layoutDirection

private var onAttachedToWindow: (() -> Unit)? = null
private fun runOnceViewAttached(block: () -> Unit) {
if (renderingView.window == null) {
Expand Down Expand Up @@ -623,9 +662,21 @@ internal class ComposeSceneMediator(
size.height
}

var density by scene::density
var layoutDirection by scene::layoutDirection
private var _onPreviewKeyEvent: (KeyEvent) -> Boolean = { false }
private var _onKeyEvent: (KeyEvent) -> Boolean = { false }
fun setKeyEventListener(
onPreviewKeyEvent: ((KeyEvent) -> Boolean)?,
onKeyEvent: ((KeyEvent) -> Boolean)?
) {
this._onPreviewKeyEvent = onPreviewKeyEvent ?: { false }
this._onKeyEvent = onKeyEvent ?: { false }
}

private fun onKeyboardEvent(keyEvent: KeyEvent): Boolean =
uiKitTextInputService.onPreviewKeyEvent(keyEvent) // TODO: fix redundant call
|| _onPreviewKeyEvent(keyEvent)
|| scene.sendKeyEvent(keyEvent)
|| _onKeyEvent(keyEvent)
}

internal fun getConstraintsToFillParent(view: UIView, parent: UIView) =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ internal class UIViewComposeSceneLayer(
windowContext = windowContext,
coroutineContext = compositionContext.effectCoroutineContext,
renderingUIViewFactory = ::createSkikoUIView,
composeSceneFactory = ::createComposeScene,
composeSceneFactory = ::createComposeScene
).also {
it.compositionLocalContext = compositionLocalContext
}
Expand Down Expand Up @@ -187,8 +187,7 @@ internal class UIViewComposeSceneLayer(
onPreviewKeyEvent: ((KeyEvent) -> Boolean)?,
onKeyEvent: ((KeyEvent) -> Boolean)?
) {
//todo It needs to handle dismiss key, like Esc. But on iOS it is very rare case.
// But also it is exposed to public in Popup.skiko.kt
mediator.setKeyEventListener(onPreviewKeyEvent, onKeyEvent)
}

override fun setOutsidePointerEventListener(
Expand Down

0 comments on commit 8fd92b6

Please sign in to comment.