Fix encoding of CSI keys in Safari (#8993)

.changeset/eighty-starfishes-listen.md

@@ -0,0 +1,6 @@
+---
+"@firebase/firestore": patch
+"@firebase/util": minor
+---
+
+Fix Safari/WebKit cache issues when client-side indexing is used.

common/api-review/util.api.md

@@ -317,6 +317,11 @@ export function isReactNative(): boolean;
 // @public
 export function isSafari(): boolean;
 
+// Warning: (ae-missing-release-tag) "isSafariOrWebkit" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public
+export function isSafariOrWebkit(): boolean;
+
 // Warning: (ae-missing-release-tag) "issuedAtTime" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public

config/karma.base.js

@@ -53,6 +53,7 @@ const config = {
 
   // Doing 65 seconds to allow for the 20 second firestore tests
   browserNoActivityTimeout: 65000,
+  browserDisconnectTimeout: 65000,
 
   // Preprocess matching files before serving them to the browser.
   // Available preprocessors:

packages/firestore/package.json

@@ -36,6 +36,8 @@
     "test:browser:emulator": "karma start --targetBackend=emulator",
     "test:browser:nightly": "karma start --targetBackend=nightly",
     "test:browser:prod": "karma start --targetBackend=prod",
+    "test:webkit:prod": "BROWSERS=WebkitHeadless karma start --targetBackend=prod",
+    "test:webkit:unit": "BROWSERS=WebkitHeadless karma start --unit --targetBackend=prod",
     "test:browser:prod:nameddb": "karma start --targetBackend=prod --databaseId=test-db",
     "test:browser:unit": "karma start --unit",
     "test:browser:debug": "karma start --browsers=Chrome --auto-watch",

packages/firestore/src/index/index_entry.ts

@@ -15,15 +15,19 @@
  * limitations under the License.
  */
 
+import { isSafariOrWebkit } from '@firebase/util';
+
+import { DbIndexEntry } from '../local/indexeddb_schema';
+import { DbIndexEntryKey, KeySafeBytes } from '../local/indexeddb_sentinels';
 import { DocumentKey } from '../model/document_key';
 
 /** Represents an index entry saved by the SDK in persisted storage. */
 export class IndexEntry {
   constructor(
-    readonly indexId: number,
-    readonly documentKey: DocumentKey,
-    readonly arrayValue: Uint8Array,
-    readonly directionalValue: Uint8Array
+    readonly _indexId: number,
+    readonly _documentKey: DocumentKey,
+    readonly _arrayValue: Uint8Array,
+    readonly _directionalValue: Uint8Array
   ) {}
 
   /**
@@ -31,49 +35,82 @@ export class IndexEntry {
    * directional value.
    */
   successor(): IndexEntry {
-    const currentLength = this.directionalValue.length;
+    const currentLength = this._directionalValue.length;
     const newLength =
-      currentLength === 0 || this.directionalValue[currentLength - 1] === 255
+      currentLength === 0 || this._directionalValue[currentLength - 1] === 255
         ? currentLength + 1
         : currentLength;
 
     const successor = new Uint8Array(newLength);
-    successor.set(this.directionalValue, 0);
+    successor.set(this._directionalValue, 0);
     if (newLength !== currentLength) {
-      successor.set([0], this.directionalValue.length);
+      successor.set([0], this._directionalValue.length);
     } else {
       ++successor[successor.length - 1];
     }
 
     return new IndexEntry(
-      this.indexId,
-      this.documentKey,
-      this.arrayValue,
+      this._indexId,
+      this._documentKey,
+      this._arrayValue,
       successor
     );
   }
+
+  // Create a representation of the Index Entry as a DbIndexEntry
+  dbIndexEntry(
+    uid: string,
+    orderedDocumentKey: Uint8Array,
+    documentKey: DocumentKey
+  ): DbIndexEntry {
+    return {
+      indexId: this._indexId,
+      uid,
+      arrayValue: encodeKeySafeBytes(this._arrayValue),
+      directionalValue: encodeKeySafeBytes(this._directionalValue),
+      orderedDocumentKey: encodeKeySafeBytes(orderedDocumentKey),
+      documentKey: documentKey.path.toArray()
+    };
+  }
+
+  // Create a representation of the Index Entry as a DbIndexEntryKey
+  dbIndexEntryKey(
+    uid: string,
+    orderedDocumentKey: Uint8Array,
+    documentKey: DocumentKey
+  ): DbIndexEntryKey {
+    const entry = this.dbIndexEntry(uid, orderedDocumentKey, documentKey);
+    return [
+      entry.indexId,
+      entry.uid,
+      entry.arrayValue,
+      entry.directionalValue,
+      entry.orderedDocumentKey,
+      entry.documentKey
+    ];
+  }
 }
 
 export function indexEntryComparator(
   left: IndexEntry,
   right: IndexEntry
 ): number {
-  let cmp = left.indexId - right.indexId;
+  let cmp = left._indexId - right._indexId;
   if (cmp !== 0) {
     return cmp;
   }
 
-  cmp = compareByteArrays(left.arrayValue, right.arrayValue);
+  cmp = compareByteArrays(left._arrayValue, right._arrayValue);
   if (cmp !== 0) {
     return cmp;
   }
 
-  cmp = compareByteArrays(left.directionalValue, right.directionalValue);
+  cmp = compareByteArrays(left._directionalValue, right._directionalValue);
   if (cmp !== 0) {
     return cmp;
   }
 
-  return DocumentKey.comparator(left.documentKey, right.documentKey);
+  return DocumentKey.comparator(left._documentKey, right._documentKey);
 }
 
 export function compareByteArrays(left: Uint8Array, right: Uint8Array): number {
@@ -85,3 +122,57 @@ export function compareByteArrays(left: Uint8Array, right: Uint8Array): number {
   }
   return left.length - right.length;
 }
+
+/**
+ * Workaround for WebKit bug: https://bugs.webkit.org/show_bug.cgi?id=292721
+ * Create a key safe representation of Uint8Array values.
+ * If the browser is detected as Safari or WebKit, then
+ * the input array will be converted to "sortable byte string".
+ * Otherwise, the input array will be returned in its original type.
+ */
+export function encodeKeySafeBytes(array: Uint8Array): KeySafeBytes {
+  if (isSafariOrWebkit()) {
+    return encodeUint8ArrayToSortableString(array);
+  }
+  return array;
+}
+
+/**
+ * Reverts the key safe representation of Uint8Array (created by
+ * encodeKeySafeBytes) to a normal Uint8Array.
+ */
+export function decodeKeySafeBytes(input: KeySafeBytes): Uint8Array {
+  if (typeof input !== 'string') {
+    return input;
+  }
+  return decodeSortableStringToUint8Array(input);
+}
+
+/**
+ * Encodes a Uint8Array into a "sortable byte string".
+ * A "sortable byte string" sorts in the same order as the Uint8Array.
+ * This works because JS string comparison sorts strings based on code points.
+ */
+function encodeUint8ArrayToSortableString(array: Uint8Array): string {
+  let byteString = '';
+  for (let i = 0; i < array.length; i++) {
+    byteString += String.fromCharCode(array[i]);
+  }
+
+  return byteString;
+}
+
+/**
+ * Decodes a "sortable byte string" back into a Uint8Array.
+ * A "sortable byte string" is assumed to be created where each character's
+ * Unicode code point directly corresponds to a single byte value (0-255).
+ */
+function decodeSortableStringToUint8Array(byteString: string): Uint8Array {
+  const uint8array = new Uint8Array(byteString.length);
+
+  for (let i = 0; i < byteString.length; i++) {
+    uint8array[i] = byteString.charCodeAt(i);
+  }
+
+  return uint8array;
+}

packages/firestore/src/local/indexeddb_index_manager.ts

@@ -39,7 +39,12 @@ import {
 } from '../core/target';
 import { FirestoreIndexValueWriter } from '../index/firestore_index_value_writer';
 import { IndexByteEncoder } from '../index/index_byte_encoder';
-import { IndexEntry, indexEntryComparator } from '../index/index_entry';
+import {
+  IndexEntry,
+  indexEntryComparator,
+  encodeKeySafeBytes,
+  decodeKeySafeBytes
+} from '../index/index_entry';
 import { documentKeySet, DocumentMap } from '../model/collections';
 import { Document } from '../model/document';
 import { DocumentKey } from '../model/document_key';
@@ -817,14 +822,13 @@ export class IndexedDbIndexManager implements IndexManager {
     indexEntry: IndexEntry
   ): PersistencePromise<void> {
     const indexEntries = indexEntriesStore(transaction);
-    return indexEntries.put({
-      indexId: indexEntry.indexId,
-      uid: this.uid,
-      arrayValue: indexEntry.arrayValue,
-      directionalValue: indexEntry.directionalValue,
-      orderedDocumentKey: this.encodeDirectionalKey(fieldIndex, document.key),
-      documentKey: document.key.path.toArray()
-    });
+    return indexEntries.put(
+      indexEntry.dbIndexEntry(
+        this.uid,
+        this.encodeDirectionalKey(fieldIndex, document.key),
+        document.key
+      )
+    );
   }
 
   private deleteIndexEntry(
@@ -834,14 +838,13 @@ export class IndexedDbIndexManager implements IndexManager {
     indexEntry: IndexEntry
   ): PersistencePromise<void> {
     const indexEntries = indexEntriesStore(transaction);
-    return indexEntries.delete([
-      indexEntry.indexId,
-      this.uid,
-      indexEntry.arrayValue,
-      indexEntry.directionalValue,
-      this.encodeDirectionalKey(fieldIndex, document.key),
-      document.key.path.toArray()
-    ]);
+    return indexEntries.delete(
+      indexEntry.dbIndexEntryKey(
+        this.uid,
+        this.encodeDirectionalKey(fieldIndex, document.key),
+        document.key
+      )
+    );
   }
 
   private getExistingIndexEntries(
@@ -858,7 +861,9 @@ export class IndexedDbIndexManager implements IndexManager {
           range: IDBKeyRange.only([
             fieldIndex.indexId,
             this.uid,
-            this.encodeDirectionalKey(fieldIndex, documentKey)
+            encodeKeySafeBytes(
+              this.encodeDirectionalKey(fieldIndex, documentKey)
+            )
           ])
         },
         (_, entry) => {
@@ -866,8 +871,8 @@ export class IndexedDbIndexManager implements IndexManager {
             new IndexEntry(
               fieldIndex.indexId,
               documentKey,
-              entry.arrayValue,
-              entry.directionalValue
+              decodeKeySafeBytes(entry.arrayValue),
+              decodeKeySafeBytes(entry.directionalValue)
             )
           );
         }
@@ -1020,24 +1025,16 @@ export class IndexedDbIndexManager implements IndexManager {
         return [];
       }
 
-      const lowerBound = [
-        bounds[i].indexId,
+      const lowerBound = bounds[i].dbIndexEntryKey(
         this.uid,
-        bounds[i].arrayValue,
-        bounds[i].directionalValue,
         EMPTY_VALUE,
-        []
-      ] as DbIndexEntryKey;
-
-      const upperBound = [
-        bounds[i + 1].indexId,
+        DocumentKey.empty()
+      );
+      const upperBound = bounds[i + 1].dbIndexEntryKey(
         this.uid,
-        bounds[i + 1].arrayValue,
-        bounds[i + 1].directionalValue,
         EMPTY_VALUE,
-        []
-      ] as DbIndexEntryKey;
-
+        DocumentKey.empty()
+      );
       ranges.push(IDBKeyRange.bound(lowerBound, upperBound));
     }
     return ranges;

packages/firestore/src/local/indexeddb_schema.ts

@@ -26,7 +26,7 @@ import {
 } from '../protos/firestore_proto_api';
 
 import { EncodedResourcePath } from './encoded_resource_path';
-import { DbTimestampKey } from './indexeddb_sentinels';
+import { DbTimestampKey, KeySafeBytes } from './indexeddb_sentinels';
 
 /**
  * Schema Version for the Web client:
@@ -52,9 +52,11 @@ import { DbTimestampKey } from './indexeddb_sentinels';
  * 14. Add overlays.
  * 15. Add indexing support.
  * 16. Parse timestamp strings before creating index entries.
+ * 17. TODO(tomandersen)
+ * 18. Encode key safe representations of IndexEntry in DbIndexEntryStore.
  */
 
-export const SCHEMA_VERSION = 17;
+export const SCHEMA_VERSION = 18;
 
 /**
  * Wrapper class to store timestamps (seconds and nanos) in IndexedDb objects.
@@ -507,14 +509,14 @@ export interface DbIndexEntry {
   /** The user id for this entry. */
   uid: string;
   /** The encoded array index value for this entry. */
-  arrayValue: Uint8Array;
+  arrayValue: KeySafeBytes;
   /** The encoded directional value for equality and inequality filters. */
-  directionalValue: Uint8Array;
+  directionalValue: KeySafeBytes;
   /**
    * The document key this entry points to. This entry is encoded by an ordered
    * encoder to match the key order of the index.
    */
-  orderedDocumentKey: Uint8Array;
+  orderedDocumentKey: KeySafeBytes;
   /** The segments of the document key this entry points to. */
   documentKey: string[];
 }

packages/firestore/src/local/indexeddb_schema_converter.ts

@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+import { isSafariOrWebkit } from '@firebase/util';
+
 import { User } from '../auth/user';
 import { ListenSequence } from '../core/listen_sequence';
 import { SnapshotVersion } from '../core/snapshot_version';
@@ -277,6 +279,22 @@ export class SchemaConverter implements SimpleDbSchemaConverter {
       });
     }
 
+    if (fromVersion < 18 && toVersion >= 18) {
+      // Clear the IndexEntryStores on WebKit and Safari to remove possibly
+      // corrupted index entries
+      if (isSafariOrWebkit()) {
+        p = p
+          .next(() => {
+            const indexStateStore = txn.objectStore(DbIndexStateStore);
+            indexStateStore.clear();
+          })
+          .next(() => {
+            const indexEntryStore = txn.objectStore(DbIndexEntryStore);
+            indexEntryStore.clear();
+          });
+      }
+    }
+
     return p;
   }
 

packages/firestore/src/local/indexeddb_sentinels.ts

@@ -305,6 +305,15 @@ export const DbIndexStateSequenceNumberIndex = 'sequenceNumberIndex';
 
 export const DbIndexStateSequenceNumberIndexPath = ['uid', 'sequenceNumber'];
 
+/**
+ * Representation of a byte array that is safe for
+ * use in an IndexedDb key. The value is either
+ * a "sortable byte string", which is key safe in
+ * Safari/WebKit, or the value is a Uint8Array,
+ * which is key safe in other browsers.
+ */
+export type KeySafeBytes = Uint8Array | string;
+
 /**
  * The key for each index entry consists of the index id and its user id,
  * the encoded array and directional value for the indexed fields as well as
@@ -313,9 +322,9 @@ export const DbIndexStateSequenceNumberIndexPath = ['uid', 'sequenceNumber'];
 export type DbIndexEntryKey = [
   number,
   string,
-  Uint8Array,
-  Uint8Array,
-  Uint8Array,
+  KeySafeBytes,
+  KeySafeBytes,
+  KeySafeBytes,
   string[]
 ];
 
@@ -425,6 +434,7 @@ export const V15_STORES = [
 ];
 export const V16_STORES = V15_STORES;
 export const V17_STORES = [...V15_STORES, DbGlobalsStore];
+export const V18_STORES = V17_STORES;
 
 /**
  * The list of all default IndexedDB stores used throughout the SDK. This is
@@ -435,7 +445,9 @@ export const ALL_STORES = V12_STORES;
 
 /** Returns the object stores for the provided schema. */
 export function getObjectStores(schemaVersion: number): string[] {
-  if (schemaVersion === 17) {
+  if (schemaVersion === 18) {
+    return V18_STORES;
+  } else if (schemaVersion === 17) {
     return V17_STORES;
   } else if (schemaVersion === 16) {
     return V16_STORES;
@@ -450,6 +462,6 @@ export function getObjectStores(schemaVersion: number): string[] {
   } else if (schemaVersion === 11) {
     return V11_STORES;
   } else {
-    fail(0xeb55, 'Only schema version 11 and 12 and 13 are supported');
+    fail(0xeb55, 'Only schema versions >11 are supported');
   }
 }

packages/util/src/environment.ts

@@ -173,6 +173,17 @@ export function isSafari(): boolean {
   );
 }
 
+/** Returns true if we are running in Safari or WebKit */
+export function isSafariOrWebkit(): boolean {
+  return (
+    !isNode() &&
+    !!navigator.userAgent &&
+    (navigator.userAgent.includes('Safari') ||
+      navigator.userAgent.includes('WebKit')) &&
+    !navigator.userAgent.includes('Chrome')
+  );
+}
+
 /**
  * This method checks if indexedDB is supported by current browser/service worker context
  * @return true if indexedDB is supported by current browser/service worker context