feat: accept RegExp and arrays for URL blacklist (setURLBlacklist / detoxURLBlacklistRegex) (#4959)

detox/android/detox/src/full/java/com/wix/detox/reactnative/idlingresources/ReactNativeIdlingResources.kt

@@ -15,6 +15,7 @@ import com.wix.detox.reactnative.idlingresources.looper.MQThreadsReflector
 import com.wix.detox.reactnative.idlingresources.network.NetworkIdlingResource
 import kotlinx.coroutines.runBlocking
 import org.joor.Reflect
+import org.json.JSONArray
 import java.util.concurrent.ConcurrentHashMap
 
 private const val LOG_TAG = "DetoxRNIdleRes"
@@ -170,9 +171,28 @@ class ReactNativeIdlingResources(
     }
 
     private fun toFormattedUrlArray(urlList: String): List<String> {
+        parseUrlBlacklistJsonArray(urlList)?.let {
+            return it
+        }
+
         var formattedUrls = urlList
         formattedUrls = formattedUrls.replace(Regex("""[()"]"""), "")
         formattedUrls = formattedUrls.trim()
         return formattedUrls.split(',')
     }
+
+    private fun parseUrlBlacklistJsonArray(urlList: String): List<String>? {
+        val trimmedUrlList = urlList.trim()
+        if (!trimmedUrlList.startsWith("[")) {
+            return null
+        }
+
+        return try {
+            val jsonArray = JSONArray(trimmedUrlList)
+            List(jsonArray.length()) { index -> jsonArray.getString(index) }
+        } catch (e: Exception) {
+            Log.w(LOG_TAG, "Failed to parse URL blacklist as JSON array, falling back to legacy parser", e)
+            null
+        }
+    }
 }

detox/detox.d.ts

@@ -966,9 +966,13 @@ declare global {
             /**
              * Disable network synchronization mechanism on preferred endpoints. Useful if you want to on skip over synchronizing on certain URLs.
              *
+             * Accepts either string patterns or `RegExp` objects. With `RegExp`, only the `i`, `m` and `s`
+             * flags are portable across iOS and Android; the `g`, `y`, `d`, `u` and `v` flags are rejected.
+             *
              * @example await device.setURLBlacklist(['.*127.0.0.1.*']);
+             * @example await device.setURLBlacklist([/.*127\.0\.0\.1/, /my\.ignored\.endpoint/i]);
              */
-            setURLBlacklist(urls: string[]): Promise<void>;
+            setURLBlacklist(urls: Array<string | RegExp>): Promise<void>;
 
             /**
              * Temporarily disable synchronization (idle/busy monitoring) with the app - namely, stop waiting for the app to go idle before moving forward in the test execution.

detox/ios/Detox/DetoxInit.m

@@ -67,14 +67,43 @@ static void detoxConditionalInit(void)
 		settings[@"enabled"] = @([syncEnabled boolValue]);
 	}
 	
-	NSString *blacklistRegex = [options stringForKey:@"detoxURLBlacklistRegex"];
+	id blacklistRegex = [options objectForKey:@"detoxURLBlacklistRegex"];
 	if (blacklistRegex)
 	{
-	    NSCharacterSet* separatorOrUselessChars = [NSCharacterSet characterSetWithCharactersInString:@"()\", "];
-        NSArray* _blacklistArray = [blacklistRegex componentsSeparatedByCharactersInSet:separatorOrUselessChars];
+		NSPredicate* predicate = [NSPredicate predicateWithBlock:^BOOL(id value, NSDictionary* __unused bindings) {
+			return [value isKindOfClass:NSString.class] && [value length] > 1;
+		}];
+		NSArray* parsedBlacklist = nil;
 
-        NSPredicate* predicate = [NSPredicate predicateWithFormat:@"length>1"];
-        settings[@"blacklistURLs"] = [_blacklistArray filteredArrayUsingPredicate:predicate];
+		if([blacklistRegex isKindOfClass:NSArray.class])
+		{
+			parsedBlacklist = [blacklistRegex filteredArrayUsingPredicate:predicate];
+		}
+		else if([blacklistRegex isKindOfClass:NSString.class])
+		{
+			NSData* jsonData = [blacklistRegex dataUsingEncoding:NSUTF8StringEncoding];
+			NSError* jsonError = nil;
+			id jsonParsed = [NSJSONSerialization JSONObjectWithData:jsonData options:0 error:&jsonError];
+			if(!jsonError && [jsonParsed isKindOfClass:NSArray.class])
+			{
+				parsedBlacklist = [jsonParsed filteredArrayUsingPredicate:predicate];
+			}
+			else
+			{
+				NSCharacterSet* separatorOrUselessChars = [NSCharacterSet characterSetWithCharactersInString:@"()\", "];
+				NSArray* blacklistArray = [blacklistRegex componentsSeparatedByCharactersInSet:separatorOrUselessChars];
+				parsedBlacklist = [blacklistArray filteredArrayUsingPredicate:predicate];
+			}
+		}
+		else
+		{
+			NSLog(@"[DetoxInit] Ignoring unsupported detoxURLBlacklistRegex value type: %@", [blacklistRegex class]);
+		}
+
+		if(parsedBlacklist)
+		{
+			settings[@"blacklistURLs"] = parsedBlacklist;
+		}
 	}
 	
 	NSString* maxTimerWait = [options objectForKey:@"detoxMaxSynchronizedDelay"];

detox/ios/Detox/DetoxManager.swift

@@ -200,7 +200,6 @@ public class DetoxManager : NSObject, WebSocketDelegate {
                     Thread.sleep(forTimeInterval: Double(truncating: value as! NSNumber) / 1000)
                     return
                 case "blacklistURLs":
-                    DTXSyncManager.urlBlacklist = value as! [String]
                     DTXSyncManager.urlBlacklist = value as! [String]
                     return
                 case "enabled":

detox/src/devices/common/drivers/android/tools/instrumentationArgs.js

@@ -2,6 +2,11 @@ const _ = require('lodash');
 
 const { encodeBase64 } = require('../../../../../utils/encoding');
 const { autoEscape } = require('../../../../../utils/shellUtils');
+const {
+  isSerializedURLBlacklistForAndroid,
+  serializeURLBlacklistForAndroid,
+  URL_BLACKLIST_LAUNCH_ARG,
+} = require('../../utils/urlBlacklist');
 
 const reservedInstrumentationArgs = new Set(['class', 'package', 'func', 'unit', 'size', 'perf', 'debug', 'log', 'emma', 'coverageFile']);
 const isReservedInstrumentationArg = (arg) => reservedInstrumentationArgs.has(arg);
@@ -9,7 +14,8 @@ const isReservedInstrumentationArg = (arg) => reservedInstrumentationArgs.has(ar
 function prepareInstrumentationArgs(args) {
   const usedReservedArgs = [];
   const preparedLaunchArgs = _.reduce(args, (result, value, key) => {
-    const valueAsString = _.isString(value) ? value : JSON.stringify(value);
+    const serializedValue = key === URL_BLACKLIST_LAUNCH_ARG ? serializeURLBlacklistForAndroid(value) : value;
+    const valueAsString = _.isString(serializedValue) ? serializedValue : JSON.stringify(serializedValue);
 
     let valueEncoded = valueAsString;
     if (isReservedInstrumentationArg(key)) {
@@ -18,7 +24,7 @@ function prepareInstrumentationArgs(args) {
       valueEncoded = encodeBase64(valueAsString);
     }
 
-    const valueEscaped = hasLegacyIssues(key) ? valueEncoded : autoEscape.shell(valueEncoded);
+    const valueEscaped = hasLegacyIssues(key, valueEncoded) ? valueEncoded : autoEscape.shell(valueEncoded);
     result.push('-e', key, valueEscaped);
     return result;
   }, []);
@@ -29,8 +35,8 @@ function prepareInstrumentationArgs(args) {
   };
 }
 
-function hasLegacyIssues(key) {
-  return key === 'detoxURLBlacklistRegex';
+function hasLegacyIssues(key, value) {
+  return key === URL_BLACKLIST_LAUNCH_ARG && !isSerializedURLBlacklistForAndroid(value);
 }
 
 module.exports = {

detox/src/devices/common/drivers/android/tools/instrumentationArgs.test.js

@@ -1,5 +1,7 @@
 /* eslint-disable no-useless-escape */
 // @ts-nocheck
+const { autoEscape } = require('../../../../../utils/shellUtils');
+
 describe('Instrumentation arguments', () => {
   let uut;
   let encodeBase64;
@@ -99,6 +101,25 @@ describe('Instrumentation arguments', () => {
     expect(result.usedReservedArgs).toEqual(Object.keys(args));
   });
 
+  it('should serialize URL blacklist RegExp arrays for Android instrumentation args', () => {
+    const blacklist = [
+      /https:\/\/x\.com\/a(?:b|c)\/{1,3}/,
+      /^https:\/\/x\.com\/foo bar$/i,
+    ];
+    const expectedValue = JSON.stringify([
+      'https:\\/\\/x\\.com\\/a(?:b|c)\\/{1,3}',
+      '(?i:^https:\\/\\/x\\.com\\/foo bar$)',
+    ]);
+
+    const result = uut.prepareInstrumentationArgs({
+      detoxURLBlacklistRegex: blacklist,
+    });
+
+    expect(result.args).toEqual([
+      ...expectedArgUnencoded('detoxURLBlacklistRegex', autoEscape.shell(expectedValue)),
+    ]);
+  });
+
   it('should whitelist args with \'detox\' prefix with respect to base64 encoding', () => {
     const args = {
       mockMe: 'this',

detox/src/devices/common/drivers/ios/tools/AppleSimUtils.js

@@ -10,6 +10,7 @@ const childProcess = require('../../../../../utils/childProcess');
 const environment = require('../../../../../utils/environment');
 const log = require('../../../../../utils/logger').child({ cat: 'device' });
 const { quote } = require('../../../../../utils/shellQuote');
+const { serializeURLBlacklistForIOS, URL_BLACKLIST_LAUNCH_ARG } = require('../../utils/urlBlacklist');
 
 const PERMISSIONS_VALUES = {
   YES: 'YES',
@@ -530,6 +531,9 @@ class AppleSimUtils {
     const args = {
       ...launchArgs,
     };
+    if (URL_BLACKLIST_LAUNCH_ARG in args) {
+      args[URL_BLACKLIST_LAUNCH_ARG] = serializeURLBlacklistForIOS(args[URL_BLACKLIST_LAUNCH_ARG]);
+    }
 
     if (languageAndLocale) {
       if (languageAndLocale.language) {

detox/src/devices/common/drivers/ios/tools/AppleSimUtils.test.js

@@ -2,6 +2,9 @@
 jest.mock('../../../../../utils/childProcess');
 jest.mock('../../../../../utils/environment');
 
+const { quote } = require('../../../../../utils/shellQuote');
+const { serializeURLBlacklistForIOS } = require('../../utils/urlBlacklist');
+
 describe('AppleSimUtils', () => {
   let AppleSimUtils;
   let uut;
@@ -56,6 +59,20 @@ describe('AppleSimUtils', () => {
       expect(launchCmd).not.toContain('--arch');
     });
 
+    it('should serialize URL blacklist RegExp arrays for iOS launch args', async () => {
+      const blacklist = [
+        /https:\/\/x\.com\/a(?:b|c)\/{1,3}/,
+        /^https:\/\/x\.com\/foo bar$/i,
+      ];
+
+      await uut.launch(udid, bundleId, { detoxURLBlacklistRegex: blacklist }, '');
+
+      const launchCmd = childProcess.execWithRetriesAndLogs.mock.calls[0][0];
+      expect(launchCmd).toContain(
+        quote(['-detoxURLBlacklistRegex', serializeURLBlacklistForIOS(blacklist)])
+      );
+    });
+
     it('should not query device version when arch is undefined', async () => {
       const spy = jest.spyOn(uut, '_isArchArgumentAccessible');
       await uut.launch(udid, bundleId, {}, '');

detox/src/devices/common/drivers/utils/urlBlacklist.js

@@ -0,0 +1,98 @@
+const URL_BLACKLIST_LAUNCH_ARG = 'detoxURLBlacklistRegex';
+
+function isRegExp(value) {
+  return value instanceof RegExp;
+}
+
+function toURLBlacklistArray(value) {
+  if (Array.isArray(value)) {
+    return value.map(toRegexPattern);
+  }
+
+  if (isRegExp(value)) {
+    return [toRegexPattern(value)];
+  }
+
+  return null;
+}
+
+function toRegexPattern(value) {
+  if (isRegExp(value)) {
+    return withPortableFlags(value);
+  }
+
+  if (typeof value === 'string') {
+    return value;
+  }
+
+  throw new TypeError([
+    'detoxURLBlacklistRegex must be a RegExp, string,',
+    `or an array of RegExp/string values, got ${typeof value}`,
+  ].join(' '));
+}
+
+const UNSUPPORTED_FLAGS = ['g', 'y', 'd', 'u', 'v'];
+
+function withPortableFlags(regex) {
+  const unsupported = UNSUPPORTED_FLAGS.filter(f => regex.flags.includes(f));
+  if (unsupported.length > 0) {
+    throw new TypeError(
+      `detoxURLBlacklistRegex: flag(s) [${unsupported.join(', ')}] in /${regex.source}/${regex.flags} ` +
+      `are not portable across iOS and Android — only i, m, s are supported`
+    );
+  }
+
+  const flags = [
+    regex.ignoreCase && 'i',
+    regex.multiline && 'm',
+    regex.dotAll && 's',
+  ].filter(Boolean).join('');
+
+  return flags ? `(?${flags}:${regex.source})` : regex.source;
+}
+
+function serializeURLBlacklistForIOS(value) {
+  const patterns = toURLBlacklistArray(value);
+  if (!patterns) {
+    return value;
+  }
+
+  return JSON.stringify(patterns);
+}
+
+function serializeURLBlacklistForAndroid(value) {
+  const patterns = toURLBlacklistArray(value);
+  if (!patterns) {
+    return value;
+  }
+
+  return JSON.stringify(patterns);
+}
+
+function normalizeURLBlacklist(value) {
+  const patterns = toURLBlacklistArray(value);
+  if (!patterns) {
+    return value;
+  }
+
+  return patterns;
+}
+
+function isSerializedURLBlacklistForAndroid(value) {
+  if (typeof value !== 'string') return false;
+  const trimmed = value.trim();
+  if (!trimmed.startsWith('[')) return false;
+  try {
+    return Array.isArray(JSON.parse(trimmed));
+  } catch {
+    return false;
+  }
+}
+
+module.exports = {
+  URL_BLACKLIST_LAUNCH_ARG,
+  isSerializedURLBlacklistForAndroid,
+  normalizeURLBlacklist,
+  serializeURLBlacklistForAndroid,
+  serializeURLBlacklistForIOS,
+};

detox/src/devices/common/drivers/utils/urlBlacklist.test.js

@@ -0,0 +1,90 @@
+const {
+  serializeURLBlacklistForAndroid,
+  serializeURLBlacklistForIOS,
+  isSerializedURLBlacklistForAndroid,
+  normalizeURLBlacklist,
+} = require('./urlBlacklist');
+
+describe('URL blacklist launch-arg serialization', () => {
+  const blacklist = [
+    /https:\/\/x\.com\/a(?:b|c)\/{1,3}/,
+    /^https:\/\/x\.com\/foo bar$/,
+    /^https:\/\/x\.com\/"quoted"$/,
+    /foo\s+bar/ims,
+  ];
+
+  const expectedPatterns = [
+    'https:\\/\\/x\\.com\\/a(?:b|c)\\/{1,3}',
+    '^https:\\/\\/x\\.com\\/foo bar$',
+    '^https:\\/\\/x\\.com\\/"quoted"$',
+    '(?ims:foo\\s+bar)',
+  ];
+
+  it('serializes RegExp arrays to an iOS JSON array string', () => {
+    expect(serializeURLBlacklistForIOS(blacklist)).toBe(JSON.stringify(expectedPatterns));
+  });
+
+  it('serializes RegExp arrays to an Android JSON array string', () => {
+    expect(serializeURLBlacklistForAndroid(blacklist)).toBe(JSON.stringify(expectedPatterns));
+  });
+
+  it('also accepts string arrays for callers that cannot use RegExp objects', () => {
+    const stringBlacklist = [
+      'https://x.com/a(?:b|c)/{1,3}',
+      '^https://x.com/foo bar$',
+    ];
+
+    expect(serializeURLBlacklistForIOS(stringBlacklist)).toBe(JSON.stringify(stringBlacklist));
+    expect(serializeURLBlacklistForAndroid(stringBlacklist)).toBe(JSON.stringify(stringBlacklist));
+  });
+
+  it('leaves legacy string values untouched', () => {
+    const legacyValue = '(".*frog.wix.com/.*")';
+
+    expect(serializeURLBlacklistForIOS(legacyValue)).toBe(legacyValue);
+    expect(serializeURLBlacklistForAndroid(legacyValue)).toBe(legacyValue);
+  });
+
+  it('does not misidentify a legacy plain-string regex starting with [ as a serialized array', () => {
+    expect(isSerializedURLBlacklistForAndroid('[a-z]+.*')).toBe(false);
+    expect(isSerializedURLBlacklistForAndroid('[invalid json')).toBe(false);
+    expect(isSerializedURLBlacklistForAndroid('["valid","json"]')).toBe(true);
+    expect(isSerializedURLBlacklistForAndroid('.*no-brackets.*')).toBe(false);
+    expect(isSerializedURLBlacklistForAndroid(42)).toBe(false);
+  });
+
+  it('accepts a bare RegExp (not wrapped in an array)', () => {
+    const pattern = /my\.api\.host/i;
+    expect(serializeURLBlacklistForIOS(pattern)).toBe(JSON.stringify(['(?i:my\\.api\\.host)']));
+    expect(serializeURLBlacklistForAndroid(pattern)).toBe(JSON.stringify(['(?i:my\\.api\\.host)']));
+  });
+
+  it('throws for RegExp flags that are not portable across iOS and Android', () => {
+    expect(() => serializeURLBlacklistForAndroid([/pattern/u])).toThrow(/not portable/);
+    expect(() => serializeURLBlacklistForIOS([/pattern/g])).toThrow(/not portable/);
+  });
+
+  it('throws when an array element is neither a string nor a RegExp', () => {
+    expect(() => serializeURLBlacklistForAndroid([42])).toThrow(/detoxURLBlacklistRegex must be a RegExp/);
+    expect(() => serializeURLBlacklistForIOS([null])).toThrow(/detoxURLBlacklistRegex must be a RegExp/);
+  });
+
+  describe('normalizeURLBlacklist (runtime setURLBlacklist)', () => {
+    it('converts a RegExp array to a plain pattern-string array', () => {
+      expect(normalizeURLBlacklist(blacklist)).toEqual(expectedPatterns);
+    });
+
+    it('leaves a plain string array untouched', () => {
+      const stringBlacklist = ['.*127.0.0.1.*', '.*my.ignored.endpoint.*'];
+      expect(normalizeURLBlacklist(stringBlacklist)).toEqual(stringBlacklist);
+    });
+
+    it('passes non-array values (e.g. undefined) through unchanged', () => {
+      expect(normalizeURLBlacklist(undefined)).toBeUndefined();
+    });
+
+    it('throws for non-portable RegExp flags', () => {
+      expect(() => normalizeURLBlacklist([/pattern/g])).toThrow(/not portable/);
+    });
+  });
+});

detox/src/devices/runtime/RuntimeDevice.js

@@ -4,6 +4,7 @@ const log = require('../../utils/logger').child({ cat: 'device' });
 const mapDeviceLongPressArguments = require('../../utils/mapDeviceLongPressArguments');
 const traceMethods = require('../../utils/traceMethods');
 const wrapWithStackTraceCutter = require('../../utils/wrapWithStackTraceCutter');
+const { normalizeURLBlacklist } = require('../common/drivers/utils/urlBlacklist');
 
 const LaunchArgsEditor = require('./utils/LaunchArgsEditor');
 
@@ -331,7 +332,7 @@ class RuntimeDevice {
   }
 
   async setURLBlacklist(urlList) {
-    await this.deviceDriver.setURLBlacklist(urlList);
+    await this.deviceDriver.setURLBlacklist(normalizeURLBlacklist(urlList));
   }
 
   async enableSynchronization() {

docs/api/device.md

@@ -257,7 +257,23 @@ await device.launchApp({
 Launches the app with a URL blacklist to disable network synchronization on certain endpoints.
 Useful if the app makes frequent network calls to blacklisted endpoints upon startup.
 
-> Note that due to the complexity of reg-exps and interoperability concerns, the implementation is fairly sensitive to the format of the string of urls.
+**Recommended — pass a `RegExp` or an array of `RegExp`/strings:**
+
+```js
+await device.launchApp({
+  newInstance: true,
+  launchArgs: { detoxURLBlacklistRegex: [/^http:\/\/192\.168\.1\.253:\d{4}\/.*/, /https:\/\/e\.crashlytics\.com\/spi\/v2\/events/] },
+});
+```
+
+Using the array form (including plain strings) fixes a long-standing bug where patterns containing commas (e.g. `\d{1,3}`) were silently corrupted by comma-splitting.
+
+> **Supported `RegExp` flags:** Only `i`, `m`, and `s` are portable across iOS and Android.
+> The flags `g`, `y`, `d`, `u`, and `v` are not supported and will throw a `TypeError` at launch time.
+
+**Legacy string form (still supported):**
+
+> Note that due to the complexity of reg-exps and interoperability concerns, the legacy string form is fairly sensitive to its format.
 > Please do your best to follow the example below:
 
 ```js
@@ -405,7 +421,13 @@ await device.enableSynchronization();
 
 Exclude synchronization with respect to network activity (i.e. don’t wait for network to go idle before moving forward in the test execution) according to **specific** endpoints, denoted as URL reg-exp’s. To disable endpoints at initialization, pass in the black-list as an [app-launch argument](../guide/launch-args.md) named `detoxURLBlacklistRegex` (as explained [here](#11-detoxurlblacklistregexinitialize-the-url-blacklist-at-app-launch)).
 
+Accepts an array of strings or `RegExp` objects (or a mix of both). Only the `i`, `m`, and `s` flags are supported for `RegExp`; using `g`, `y`, `d`, `u`, or `v` will throw a `TypeError`.
+
 ```js
+// Using RegExp objects (recommended):
+await device.setURLBlacklist([/.*127\.0\.0\.1.*/, /.*my\.ignored\.endpoint.*/i]);
+
+// Using plain strings:
 await device.setURLBlacklist(['.*127.0.0.1.*', '.*my.ignored.endpoint.*']);
 ```
 

docs/troubleshooting/synchronization.md

@@ -147,6 +147,8 @@ Sometimes Detox synchronization gets block, waiting for networks call that are i
 
 _This can be solved by [black-listing the URL's in Detox](../api/device.md#deviceseturlblacklisturls), which would exclude it from Detox's synchronization over the network:_
 
+Both `setURLBlacklist` and `detoxURLBlacklistRegex` also accept `RegExp` objects (e.g. `/.*127\.0\.0\.1.*/`).
+
 ```js
 // Black-list (exclude) URL's in mid test-run:
 await device.setURLBlacklist(['.*127.0.0.1.*']);