Semantic matching basics #4788 (#4793)

* Semantic matching basics #4788

* 100% threshold pass with new tests

* Semantic Types update

* Detox changes:
* moved semanticTypes.js to src/matchers/
* changed invoker from type(x) to semanticType(x)
* re-indexed some of the classes with its matching semantic types (eg. added RCTTouchable...)
* unit test updates

* update of some changes

* Updated code with refactoring and new functionality

* updates done based on Yarik's comments

* rawType update

* semantic tests updated

* revert the old MatchersScreen

* Revert "Reverting Yarik's fix and updating iOS simulators to 18.5"

This reverts commit b74550245d.

* updated from PR

* update config simulators

* adding scrollview to the main screen

* created icon for the example app on iOS

* Revert "created icon for the example app on iOS"

This reverts commit c29f6bcf1def479f80cc352f5b662d4bdb9f7cca.

---------

Co-authored-by: Yaroslav Serhieiev <[email protected]>

detox/detox.d.ts

@@ -1156,6 +1156,8 @@ declare global {
         interface NativeElement extends NativeElementActions {
         }
 
+        type SemanticMatchingTypes = 'image' | 'input-field' | 'text' | 'button' | 'scrollview' | 'list' | 'switch' | 'slider' | 'picker' | 'activity-indicator' | 'progress';
+
         interface ByFacade {
             /**
              * by.id will match an id that is given to the view via testID prop.
@@ -1185,10 +1187,19 @@ declare global {
             label(label: string | RegExp): NativeMatcher;
 
             /**
-             * Find an element by native view type.
-             * @example await element(by.type('RCTImageView'));
+             * Find an element by native view type OR semantic type.
+             * Automatically detects if the input is a semantic type or regular class name.
+             * @example 
+             * // Semantic types (cross-platform):
+             * await element(by.type('image'));
+             * await element(by.type('button'));
+             * await element(by.type('input-field'));
+             * 
+             * // Native class names (platform-specific):
+             * await element(by.type('RCTImageView'));
+             * await element(by.type('android.widget.Button'));
              */
-            type(nativeViewType: string): NativeMatcher;
+            type(typeOrSemanticType: SemanticMatchingTypes | string): NativeMatcher;
 
             /**
              * Find an element with an accessibility trait. (iOS only)

detox/ios/Detox/Invocation/Predicate.swift

@@ -147,6 +147,15 @@ class Predicate : CustomStringConvertible, CustomDebugStringConvertible {
           compoundPredicate = AndCompoundPredicate(predicates: innerPredicates, modifiers: modifiers)
         }
         return compoundPredicate
+      case Kind.or:
+        let predicatesDictionaryRepresentation = dictionaryRepresentation[Keys.predicates] as! [[String: Any]]
+        let innerPredicates = predicatesDictionaryRepresentation.compactMap { try? Predicate.with(dictionaryRepresentation: $0) }
+        guard !innerPredicates.isEmpty else {
+          fatalError("No valid predicates found in OR compound predicate")
+        }
+        return innerPredicates.count == 1
+          ? innerPredicates.first!
+          : OrCompoundPredicate(predicates: innerPredicates, modifiers: modifiers)
       default:
         fatalError("Unknown predicate type \(kind)")
     }

detox/src/android/matchers/native.js

@@ -1,9 +1,16 @@
 const DetoxRuntimeError = require('../../errors/DetoxRuntimeError');
 const invoke = require('../../invoke');
+const semanticTypes = require('../../matchers/semanticTypes');
 const { isRegExp } = require('../../utils/isRegExp');
 const { NativeMatcher } = require('../core/NativeMatcher');
 const DetoxMatcherApi = require('../espressoapi/DetoxMatcher');
 
+const createClassMatcher = (className) =>
+  new NativeMatcher(invoke.callDirectly(DetoxMatcherApi.matcherForClass(className)));
+
+const combineWithOr = (matchers) =>
+  matchers.reduce((acc, matcher) => acc.or(matcher));
+
 class LabelMatcher extends NativeMatcher {
   constructor(value) {
     super();
@@ -29,9 +36,23 @@ class IdMatcher extends NativeMatcher {
 }
 
 class TypeMatcher extends NativeMatcher {
-  constructor(value) {
+  constructor(typeOrSemanticType) {
     super();
-    this._call = invoke.callDirectly(DetoxMatcherApi.matcherForClass(value));
+
+    const descriptors = semanticTypes.getClasses(typeOrSemanticType, 'android');
+    if (!descriptors.length) {
+      throw new DetoxRuntimeError(`No class names found for: ${typeOrSemanticType}`);
+    }
+
+    const matchers = descriptors.map(({ className, excludes }) => {
+      const includeMatcher = createClassMatcher(className);
+      return excludes.length
+        ? includeMatcher.and(combineWithOr(excludes.map(createClassMatcher)).not)
+        : includeMatcher;
+    });
+
+    const combinedMatcher = combineWithOr(matchers);
+    this._call = { ...combinedMatcher._call, rawType: typeOrSemanticType };
   }
 }
 

detox/src/android/matchers/native.test.js

@@ -0,0 +1,98 @@
+// @ts-nocheck
+jest.mock('../../matchers/semanticTypes', () => ({
+  getTypes: jest.fn(),
+  getClasses: jest.fn(),
+  includes: jest.fn()
+}));
+
+
+const semanticTypes = require('../../matchers/semanticTypes');
+
+const { TypeMatcher } = require('./native');
+
+describe('Native Matchers', () => {
+  describe('TypeMatcher', () => {
+    beforeEach(() => {
+      jest.clearAllMocks();
+    });
+
+    it('should handle regular class names', () => {
+      semanticTypes.includes.mockReturnValue(false);
+      semanticTypes.getClasses.mockReturnValue([
+        { className: 'com.example.CustomView', excludes: [] }
+      ]);
+
+      expect(() => {
+        new TypeMatcher('com.example.CustomView');
+      }).not.toThrow();
+    });
+
+    it('should handle semantic types automatically', () => {
+      semanticTypes.includes.mockReturnValue(true);
+      semanticTypes.getClasses.mockReturnValue([
+        { className: 'android.widget.ImageView', excludes: [] },
+        { className: 'com.facebook.react.views.image.ReactImageView', excludes: [] }
+      ]);
+
+      expect(() => {
+        new TypeMatcher('image');
+      }).not.toThrow();
+    });
+
+    it('should handle exclusion objects for semantic types', () => {
+      semanticTypes.includes.mockReturnValue(true);
+      semanticTypes.getClasses.mockReturnValue([
+        {
+          className: 'android.widget.ProgressBar',
+          excludes: ['android.widget.AbsSeekBar']
+        },
+        {
+          className: 'androidx.core.widget.ContentLoadingProgressBar',
+          excludes: ['android.widget.AbsSeekBar']
+        }
+      ]);
+
+      expect(() => {
+        new TypeMatcher('activity-indicator');
+      }).not.toThrow();
+    });
+
+    it('should handle mixed string and exclusion objects', () => {
+      semanticTypes.includes.mockReturnValue(true);
+      semanticTypes.getClasses.mockReturnValue([
+        {
+          className: 'android.widget.ProgressBar',
+          excludes: ['android.widget.AbsSeekBar']
+        },
+        {
+          className: 'androidx.core.widget.ContentLoadingProgressBar',
+          excludes: ['android.widget.AbsSeekBar']
+        }
+      ]);
+
+      expect(() => {
+        new TypeMatcher('progress');
+      }).not.toThrow();
+    });
+
+    it('should handle regular class names when not semantic types', () => {
+      semanticTypes.includes.mockReturnValue(false);
+      semanticTypes.getClasses.mockReturnValue([
+        { className: 'android.widget.ImageView', excludes: [] }
+      ]);
+
+      expect(() => {
+        new TypeMatcher('android.widget.ImageView');
+      }).not.toThrow();
+    });
+
+    it('should throw error when no class names are found', () => {
+      semanticTypes.includes.mockReturnValue(false);
+      semanticTypes.getClasses.mockReturnValue([]);
+
+      expect(() => {
+        new TypeMatcher('empty-type');
+      }).toThrow('No class names found for: empty-type');
+    });
+  });
+});

detox/src/ios/expectTwo.js

@@ -4,7 +4,7 @@ const path = require('path');
 const fs = require('fs-extra');
 const _ = require('lodash');
 
-
+const semanticTypes = require('../matchers/semanticTypes');
 const { assertTraceDescription, assertEnum, assertNormalized } = require('../utils/assertArgument');
 const { removeMilliseconds } = require('../utils/dateUtils');
 const { actionDescription, expectDescription } = require('../utils/invocationTraceDescriptions');
@@ -17,6 +17,18 @@ const traceInvocationCall = require('../utils/traceInvocationCall').bind(null, l
 const { systemElement, systemMatcher, systemExpect, isSystemElement } = require('./system');
 const { webElement, webMatcher, webExpect, isWebElement } = require('./web');
 
+const createTypePredicate = (className) => ({ type: 'type', value: className });
+const createOrPredicate = (predicates) => ({ type: 'or', predicates });
+const createExclusionPredicate = (className, excludes) => ({
+  type: 'and',
+  predicates: [
+    createTypePredicate(className),
+    {
+      type: 'not',
+      predicate: createOrPredicate(excludes.map(createTypePredicate))
+    }
+  ]
+});
 
 const assertDirection = assertEnum(['left', 'right', 'up', 'down']);
 const assertSpeed = assertEnum(['fast', 'slow']);
@@ -440,9 +452,16 @@ class Matcher {
     return this;
   }
 
-  type(type) {
-    if (typeof type !== 'string') throw new Error('type should be a string, but got ' + (type + (' (' + (typeof type + ')'))));
-    this.predicate = { type: 'type', value: type };
+    type(typeOrSemanticType) {
+    if (typeof typeOrSemanticType !== 'string') throw new Error('type should be a string, but got ' + (typeOrSemanticType + (' (' + (typeof typeOrSemanticType + ')'))));
+
+    const descriptors = semanticTypes.getClasses(typeOrSemanticType, 'ios');
+    const predicates = descriptors.map(({ className, excludes }) =>
+      excludes.length ? createExclusionPredicate(className, excludes) : createTypePredicate(className)
+    );
+
+    this.predicate = predicates.length > 1 ? createOrPredicate(predicates) : predicates[0];
+    this.predicate.rawType = typeOrSemanticType;
     return this;
   }
 

detox/src/ios/expectTwo.test.js

@@ -793,6 +793,89 @@ describe('expectTwo', () => {
     });
   });
 
+  describe('semantic types', () => {
+    it(`should parse correct JSON for regular class name using by.type()`, async () => {
+      const testCall = await e.element(e.by.type('CustomUIView')).tap();
+      const jsonOutput = {
+        invocation: {
+          type: 'action',
+          action: 'tap',
+          predicate: {
+            type: 'type',
+            value: 'CustomUIView',
+            rawType: 'CustomUIView'
+          }
+        }
+      };
+
+      expect(testCall).toDeepEqual(jsonOutput);
+    });
+
+    it(`should parse correct JSON for semantic type 'image' using by.type()`, async () => {
+      const testCall = await e.element(e.by.type('image')).tap();
+      const jsonOutput = {
+        invocation: {
+          type: 'action',
+          action: 'tap',
+          predicate: {
+            type: 'or',
+            predicates: [
+              {
+                type: 'type',
+                value: 'RCTImageView'
+              },
+              {
+                type: 'type',
+                value: 'RCTImageComponentView'
+              },
+              {
+                type: 'type',
+                value: 'UIImageView'
+              },
+            ],
+            rawType: 'image'
+          }
+        }
+      };
+
+      expect(testCall).toDeepEqual(jsonOutput);
+    });
+
+    it(`should parse correct JSON for semantic type with exclusions using by.type()`, async () => {
+      const semanticTypes = require('../matchers/semanticTypes');
+      const originalGetClasses = semanticTypes.getClasses;
+
+      semanticTypes.getClasses = jest.fn().mockReturnValue([
+        { className: 'UIActivityIndicatorView', excludes: ['UIProgressView'] }
+      ]);
+
+      const testCall = await e.element(e.by.type('mock-type-with-exclusion')).tap();
+      const jsonOutput = {
+        invocation: {
+          type: 'action',
+          action: 'tap',
+          predicate: {
+            type: 'and',
+            predicates: [
+              { type: 'type', value: 'UIActivityIndicatorView' },
+              {
+                type: 'not',
+                predicate: {
+                  type: 'or',
+                  predicates: [{ type: 'type', value: 'UIProgressView' }]
+                }
+              }
+            ],
+            rawType: 'mock-type-with-exclusion'
+          }
+        }
+      };
+
+      expect(testCall).toDeepEqual(jsonOutput);
+      semanticTypes.getClasses = originalGetClasses;
+    });
+  });
+
   describe('web views', () => {
     it(`should parse expect(web(by.id('webViewId').element(web(by.label('tapMe')))).toExist()`, async () => {
       const testCall = await e.expect(e.web(e.by.id('webViewId')).atIndex(1).element(e.by.web.label('tapMe')).atIndex(2)).toExist();

detox/src/matchers/semanticTypes.js

@@ -0,0 +1,137 @@
+/**
+ * Semantic type mappings for cross-platform component matching
+ */
+
+// Shared class mappings for aliases
+const ACTIVITY_INDICATOR_CLASSES = {
+  ios: ['UIActivityIndicatorView'],
+  android: [
+    {
+      include: ['android.widget.ProgressBar', 'androidx.core.widget.ContentLoadingProgressBar'],
+      exclude: ['android.widget.AbsSeekBar']
+    }
+  ]
+};
+
+const SEMANTIC_TYPE_MAPPINGS = {
+  // Images
+  'image': {
+    ios: ['RCTImageView', 'RCTImageComponentView', 'UIImageView'],
+    android: ['android.widget.ImageView', 'com.facebook.react.views.image.ReactImageView']
+  },
+
+  // Input fields
+  'input-field': {
+    ios: ['RCTTextInputView', 'RCTMultilineTextInputView', 'UITextField', 'UITextView'],
+    android: ['android.widget.EditText', 'com.facebook.react.views.textinput.ReactEditText']
+  },
+
+  // Text elements (includes both old and new arch classes)
+  'text': {
+    ios: ['RCTText', 'RCTParagraphComponentView', 'UILabel'],
+    android: [
+      {
+        include: ['android.widget.TextView', 'com.facebook.react.views.text.ReactTextView'],
+        exclude: ['android.widget.EditText', 'android.widget.Button']
+      }
+    ]
+  },
+
+  // Button elements (includes both old and new arch classes)
+  'button': {
+    ios: ['UIButton', 'RCTTouchableOpacity', 'RCTTouchableHighlight', 'RCTTouchableWithoutFeedback'],
+    android: ['android.widget.Button', 'android.widget.ImageButton']
+  },
+
+  // Scroll containers - The UITableView inherits from scrollview so it could also show up here...
+  'scrollview': {
+    ios: ['RCTScrollView', 'RCTScrollViewComponentView', 'UIScrollView'],
+    android: ['android.widget.ScrollView', 'androidx.core.widget.NestedScrollView', 'com.facebook.react.views.scroll.ReactScrollView']
+  },
+
+  // Lists
+  'list': {
+    ios: ['UITableView', 'UICollectionView', 'RCTScrollView'],
+    android: ['android.widget.ListView', 'androidx.recyclerview.widget.RecyclerView', 'com.facebook.react.views.scroll.ReactScrollView']
+  },
+
+  // Switches/Toggles
+  'switch': {
+    ios: ['UISwitch'],
+    android: ['android.widget.Switch', 'androidx.appcompat.widget.SwitchCompat', 'com.facebook.react.views.switchview.ReactSwitch']
+  },
+
+  // Sliders
+  'slider': {
+    ios: ['UISlider'],
+    android: ['android.widget.SeekBar']
+  },
+
+  // Picker/Selector
+  'picker': {
+    ios: ['UIPickerView'],
+    android: ['android.widget.Spinner', 'androidx.appcompat.widget.AppCompatSpinner']
+  },
+
+  // Activity indicators/Progress
+  'activity-indicator': ACTIVITY_INDICATOR_CLASSES,
+
+  // Progress (alias for activity-indicator)
+  'progress': ACTIVITY_INDICATOR_CLASSES
+};
+
+/**
+ * Get platform-specific class names for a semantic type
+ * @param {string} semanticType - The semantic type (e.g., 'image', 'input-field')
+ * @param {string} platform - The platform ('ios' or 'android')
+ * @returns {Array<string|object>} Array of class names or matcher objects for the platform
+ */
+function getClasses(semanticType, platform) {
+  const mapping = SEMANTIC_TYPE_MAPPINGS[semanticType];
+  if (!mapping) {
+    return [{ className: semanticType, excludes: [] }];
+  }
+
+  const classNames = mapping[platform];
+  if (!classNames) {
+    throw new Error(`Platform ${platform} not supported for semantic type ${semanticType}`);
+  }
+
+  return classNames.map(item => {
+    if (typeof item === 'string') {
+      return { className: item, excludes: [] };
+    } else if (item.include && item.exclude) {
+      if (Array.isArray(item.include)) {
+        return item.include.map(className => ({
+          className,
+          excludes: item.exclude
+        }));
+      } else {
+        return {
+          className: item.include,
+          excludes: item.exclude
+        };
+      }
+    }
+    return { className: item, excludes: [] };
+  }).flat();
+}
+
+/**
+ * Get all available semantic types
+ * @returns {string[]} Array of available semantic type names
+ */
+function getTypes() {
+  return Object.keys(SEMANTIC_TYPE_MAPPINGS);
+}
+
+function includes(value) {
+  return getTypes().includes(value);
+}
+
+module.exports = {
+  SEMANTIC_TYPE_MAPPINGS,
+  getClasses,
+  getTypes,
+  includes,
+};

detox/src/matchers/semanticTypes.test.js

@@ -0,0 +1,160 @@
+const semanticTypes = require('./semanticTypes');
+
+describe('semanticTypes', () => {
+  const semanticTypeList = semanticTypes.getTypes();
+  const platforms = ['ios', 'android'];
+
+  describe('getClasses', () => {
+    it('should return fallback for unknown semantic type', () => {
+      const classNames = semanticTypes.getClasses('unknown-type', 'ios');
+      expect(classNames).toEqual([{ className: 'unknown-type', excludes: [] }]);
+    });
+
+    it('should throw error for unsupported platform', () => {
+      expect(() => {
+        semanticTypes.getClasses('image', 'windows');
+      }).toThrow('Platform windows not supported for semantic type image');
+    });
+
+    it('should return same classes for progress and activity-indicator', () => {
+      platforms.forEach(platform => {
+        expect(semanticTypes.getClasses('progress', platform)).toEqual(
+          semanticTypes.getClasses('activity-indicator', platform)
+        );
+      });
+    });
+
+    it('should handle exclusion objects correctly', () => {
+      const classNames = semanticTypes.getClasses('activity-indicator', 'android');
+
+      expect(classNames).toHaveLength(2);
+      expect(classNames[0]).toHaveProperty('className', 'android.widget.ProgressBar');
+      expect(classNames[0]).toHaveProperty('excludes', ['android.widget.AbsSeekBar']);
+      expect(classNames[1]).toHaveProperty('className', 'androidx.core.widget.ContentLoadingProgressBar');
+      expect(classNames[1]).toHaveProperty('excludes', ['android.widget.AbsSeekBar']);
+    });
+
+    it('should handle text exclusions to prevent EditText and Button conflicts', () => {
+      const classNames = semanticTypes.getClasses('text', 'android');
+
+      expect(classNames).toHaveLength(2);
+      expect(classNames[0]).toHaveProperty('className', 'android.widget.TextView');
+      expect(classNames[0]).toHaveProperty('excludes', ['android.widget.EditText', 'android.widget.Button']);
+      expect(classNames[1]).toHaveProperty('className', 'com.facebook.react.views.text.ReactTextView');
+      expect(classNames[1]).toHaveProperty('excludes', ['android.widget.EditText', 'android.widget.Button']);
+    });
+
+    it('should handle single string include with exclusions', () => {
+      semanticTypes.SEMANTIC_TYPE_MAPPINGS['test-single'] = {
+        android: [{
+          include: 'android.widget.TestView',
+          exclude: ['android.widget.ExcludedView']
+        }]
+      };
+
+      const classNames = semanticTypes.getClasses('test-single', 'android');
+
+      expect(classNames).toHaveLength(1);
+      expect(classNames[0]).toHaveProperty('className', 'android.widget.TestView');
+      expect(classNames[0]).toHaveProperty('excludes', ['android.widget.ExcludedView']);
+
+      delete semanticTypes.SEMANTIC_TYPE_MAPPINGS['test-single'];
+    });
+
+    it('should handle unexpected item types gracefully', () => {
+      semanticTypes.SEMANTIC_TYPE_MAPPINGS['test-fallback'] = {
+        android: [{ unexpectedProperty: 'someValue' }]
+      };
+
+      const classNames = semanticTypes.getClasses('test-fallback', 'android');
+
+      expect(classNames).toHaveLength(1);
+      expect(classNames[0]).toHaveProperty('className');
+      expect(classNames[0]).toHaveProperty('excludes', []);
+
+      delete semanticTypes.SEMANTIC_TYPE_MAPPINGS['test-fallback'];
+    });
+  });
+
+  describe('getTypes', () => {
+    it('should return all semantic types from mappings', () => {
+      const types = semanticTypes.getTypes();
+      const mappingKeys = Object.keys(semanticTypes.SEMANTIC_TYPE_MAPPINGS);
+      expect(types).toEqual(mappingKeys);
+    });
+
+    it('should return an array', () => {
+      const types = semanticTypes.getTypes();
+      expect(Array.isArray(types)).toBe(true);
+    });
+
+    it('should not be empty', () => {
+      const types = semanticTypes.getTypes();
+      expect(types.length).toBeGreaterThan(0);
+    });
+  });
+
+  describe('includes', () => {
+    it('should return true for valid semantic types', () => {
+      semanticTypeList.forEach(semanticType => {
+        expect(semanticTypes.includes(semanticType)).toBe(true);
+      });
+    });
+
+    it('should return false for invalid semantic types', () => {
+      expect(semanticTypes.includes('unknown-type')).toBe(false);
+      expect(semanticTypes.includes('')).toBe(false);
+
+      const buttonClasses = semanticTypes.getClasses('button', 'ios');
+      const imageClasses = semanticTypes.getClasses('image', 'android');
+
+      buttonClasses.forEach(item => {
+        expect(semanticTypes.includes(item.className)).toBe(false);
+      });
+
+      imageClasses.forEach(item => {
+        expect(semanticTypes.includes(item.className)).toBe(false);
+      });
+    });
+
+    it('should return false for non-string values', () => {
+      expect(semanticTypes.includes(null)).toBe(false);
+      expect(semanticTypes.includes(undefined)).toBe(false);
+      expect(semanticTypes.includes(123)).toBe(false);
+      expect(semanticTypes.includes({})).toBe(false);
+      expect(semanticTypes.includes([])).toBe(false);
+    });
+  });
+
+  describe('cross-platform validation', () => {
+    it('should ensure all semantic types have valid class names for both platforms', () => {
+      const allSemanticTypes = semanticTypes.getTypes();
+      allSemanticTypes.forEach(semanticType => {
+        const androidClassNames = semanticTypes.getClasses(semanticType, 'android');
+        const iosClassNames = semanticTypes.getClasses(semanticType, 'ios');
+
+        expect(androidClassNames.length).toBeGreaterThan(0);
+        expect(iosClassNames.length).toBeGreaterThan(0);
+
+        [...androidClassNames, ...iosClassNames].forEach(item => {
+          expect(item).toHaveProperty('className');
+          expect(item).toHaveProperty('excludes');
+          expect(typeof item.className).toBe('string');
+          expect(item.className.length).toBeGreaterThan(0);
+          expect(Array.isArray(item.excludes)).toBe(true);
+          item.excludes.forEach(excludeClass => {
+            expect(typeof excludeClass).toBe('string');
+            expect(excludeClass.length).toBeGreaterThan(0);
+          });
+        });
+      });
+    });
+
+    it('should differentiate between semantic types and regular class names', () => {
+      expect(semanticTypes.getTypes().includes('image')).toBe(true);
+      expect(semanticTypes.getTypes().includes('button')).toBe(true);
+      expect(semanticTypes.getTypes().includes('com.example.CustomView')).toBe(false);
+      expect(semanticTypes.getTypes().includes('UIButton')).toBe(false);
+    });
+  });
+});

detox/test/e2e/27.semantic-types.test.js

@@ -0,0 +1,52 @@
+const jestExpect = require('expect').default;
+
+describe('Semantic Types', () => {
+  beforeEach(async () => {
+    await device.reloadReactNative();
+    await element(by.text('Semantic Types')).tap();
+  });
+
+  it('should match image elements by semantic type using by.type()', async () => {
+    await expect(element(by.type('image')).atIndex(0)).toBeVisible();
+  });
+
+  it('should match text elements by semantic type using by.type()', async () => {
+    await expect(element(by.type('text')).atIndex(0)).toBeVisible();
+    await expect(element(by.type('text').and(by.text('Text Element')))).toBeVisible();
+  });
+
+  it('should match input field elements by semantic type using by.type()', async () => {
+    await expect(element(by.type('input-field')).atIndex(0)).toBeVisible();
+    await element(by.type('input-field')).atIndex(0).typeText('Test');
+    await expect(element(by.type('input-field')).atIndex(0)).toHaveText('Test');
+  });
+
+  it('should support progress alias for activity-indicator', async () => {
+    await expect(element(by.type('progress'))).toExist();
+    await expect(element(by.type('activity-indicator'))).toExist();
+
+    const progressAttrs = await element(by.type('progress')).getAttributes();
+    const activityIndicatorAttrs = await element(by.type('activity-indicator')).getAttributes();
+    jestExpect(progressAttrs.identifier).toEqual(activityIndicatorAttrs.identifier);
+  });
+
+  it('should match scrollview elements by semantic type', async () => {
+    await expect(element(by.type('scrollview')).atIndex(0)).toExist();
+  });
+
+  it('should match list elements by semantic type', async () => {
+    await expect(element(by.type('list')).atIndex(0)).toExist();
+  });
+
+  it('should match switch elements by semantic type', async () => {
+    await expect(element(by.type('switch'))).toExist();
+  });
+
+  it('should match slider elements by semantic type', async () => {
+    await expect(element(by.type('slider'))).toExist();
+  });
+
+  it('should match picker elements by semantic type', async () => {
+    await expect(element(by.type('picker'))).toExist();
+  });
+});

detox/test/src/Screens/SemanticTypesScreen.js

@@ -0,0 +1,80 @@
+import React from 'react';
+import {
+  View,
+  Text,
+  Image,
+  ScrollView,
+  FlatList,
+  Switch,
+  ActivityIndicator,
+  TextInput,
+} from 'react-native';
+import Slider from '@react-native-community/slider';
+import { Picker } from '@react-native-picker/picker';
+
+export default function SemanticTypesScreen() {
+  return (
+    <ScrollView testID="semanticTypesScreen" style={{ flex: 1, padding: 20, paddingTop: 60 }}>
+      <Text style={{ fontSize: 18, marginBottom: 10 }}>Semantic Types Test</Text>
+      
+      {/* Image */}
+      <Image 
+        testID="semanticImage"
+        source={require('../assets/star.png')} 
+        style={{ width: 50, height: 50, marginBottom: 10 }} 
+      />
+      
+      {/* Text */}
+      <Text testID="semanticText" style={{ marginBottom: 10 }}>Text Element</Text>
+      
+      {/* Input Field */}
+      <TextInput 
+        testID="semanticInput"
+        placeholder="Input field"
+        style={{ borderWidth: 1, padding: 10, marginBottom: 10 }}
+      />
+      
+      {/* Activity Indicator / Progress */}
+      <ActivityIndicator 
+        testID="semanticProgress"
+        size="large" 
+        color="#0000ff" 
+        style={{ marginBottom: 10 }} 
+      />
+      
+      {/* Switch */}
+      <Switch testID="semanticSwitch" style={{ marginBottom: 10 }} />
+      
+      {/* Slider */}
+      <Slider 
+        testID="semanticSlider"
+        style={{ width: 200, marginBottom: 10 }} 
+        minimumValue={0} 
+        maximumValue={1} 
+      />
+      
+      {/* Picker */}
+      <Picker testID="semanticPicker" style={{ width: 200, height: 50, marginBottom: 10 }}>
+        <Picker.Item label="Option 1" value="1" />
+        <Picker.Item label="Option 2" value="2" />
+      </Picker>
+      
+      {/* ScrollView (nested) */}
+      <View style={{ height: 60, marginBottom: 10 }}>
+        <ScrollView testID="semanticScrollView" style={{ backgroundColor: '#eee' }}>
+          <Text>Scrollable content 1</Text>
+          <Text>Scrollable content 2</Text>
+        </ScrollView>
+      </View>
+      
+      {/* List (FlatList) */}
+      <FlatList
+        testID="semanticList"
+        data={[{key: '1', title: 'Item 1'}, {key: '2', title: 'Item 2'}]}
+        renderItem={({item}) => <Text>{item.title}</Text>}
+        style={{ height: 60, backgroundColor: '#ddd' }}
+      />
+    </ScrollView>
+  );
+}
+

detox/test/src/Screens/index.js

@@ -34,6 +34,7 @@ import DragNDropScreen  from './DragNDropScreen';
 import ShapeMatchGameScreen from "./ShapeMatchGameScreen";
 import SystemDialogsScreen from "./SystemDialogsScreen";
 import AlertScreen from "./AlertScreen";
+import SemanticTypesScreen from "./SemanticTypesScreen";
 
 export {
   SanityScreen,
@@ -72,4 +73,5 @@ export {
   DragNDropScreen,
   SystemDialogsScreen,
   AlertScreen,
+  SemanticTypesScreen,
 };

detox/test/src/app.js

@@ -70,7 +70,11 @@ export default class example extends Component {
         </Text>
         {this.renderScreenButton('Language', Screens.LanguageScreen)}
         {this.renderScreenButton('Sanity', Screens.SanityScreen)}
-        {this.renderScreenButton('Matchers', Screens.MatchersScreen)}
+        <View style={{flexDirection: 'row', justifyContent: 'center'}}>
+          {this.renderScreenButton('Matchers', Screens.MatchersScreen)}
+          {this.renderInlineSeparator()}
+          {this.renderScreenButton('Semantic Types', Screens.SemanticTypesScreen)}
+        </View>
         {this.renderScreenButton('Actions', Screens.ActionsScreen)}
         {this.renderScreenButton('Visibility Expectation', Screens.VisibilityExpectationScreen)}
         {isIos && this.renderScreenButton('Visibility Debug Artifacts', Screens.VisibilityScreen)}