Merge pull request #1986 from firebase/next

.github/workflows/test.yml

@@ -21,16 +21,19 @@ jobs:
           node-version: ${{ matrix.node }}
           cache: "npm"
           cache-dependency-path: "**/package-lock.json"
-      - name: NPM INSTALL
+      - name: npm install
         run: npm i
-      - name: build emulator functions
+      - name: Build emulator functions
         run: cd _emulator/functions && npm i && npm run build & cd ../..
-      - name: Install firebase CLI
+      - name: Install Firebase CLI
         uses: nick-invision/retry@v1
         with:
           timeout_minutes: 10
           retry_wait_seconds: 60
           max_attempts: 3
           command: npm i -g firebase-tools@11
+      - name: Setup e2e secrets
+        run: |
+          echo SMTP_PASSWORD=${{ secrets.SENDGRID_API_KEY }} >> _emulator/extensions/firestore-send-email-sendgrid.secret.local
       - name: npm test
         run: npm run test:ci

_emulator/.firebaserc

@@ -2,4 +2,4 @@
   "projects": {
     "default": "demo-test"
   }
-}
+}

_emulator/extensions/firestore-bigquery-export.env.local

@@ -1,4 +1,5 @@
 BIGQUERY_PROJECT_ID=dev-extensions-testing
+DATABASE_ID=(default)
 COLLECTION_PATH=posts
 DATASET_ID=firestore_export
 DATASET_LOCATION=us-central1

_emulator/extensions/firestore-send-email-sendgrid.env.local

@@ -0,0 +1,6 @@
+DEFAULT_FROM=test-assertion@email.com
+firebaseextensions.v1beta.function/location=us-central1
+MAIL_COLLECTION=mail-sg
+SMTP_CONNECTION_URI=smtps://apikey@smtp.sendgrid.net:465
+TTL_EXPIRE_TYPE=never
+TTL_EXPIRE_VALUE=1

_emulator/firebase.json

@@ -4,7 +4,8 @@
     "delete-user-data": "../delete-user-data",
     "storage-resize-images": "../storage-resize-images",
     "firestore-counter": "../firestore-counter",
-    "firestore-bigquery-export": "../firestore-bigquery-export"
+    "firestore-bigquery-export": "../firestore-bigquery-export",
+    "firestore-send-email-sendgrid": "../firestore-send-email"
   },
   "storage": {
     "rules": "storage.rules"

firestore-bigquery-export/CHANGELOG.md

@@ -1,3 +1,13 @@
+## Version 0.1.46
+
+feature - add the ability to select Firestore database instance
+
+fix - specify Cloud Task task retry config in `extension.yaml`
+
+fix - specify the location in the `fsimportexistingdocs` Cloud Task
+
+docs - fix typos, remove mention of BigQuery updating on import
+
 ## Version 0.1.45
 
 feature - bring back the backfill parameter `DO_BACKFILL`

firestore-bigquery-export/PREINSTALL.md

@@ -7,7 +7,7 @@ The extension creates and updates a [dataset](https://cloud.google.com/bigquery/
 
 *Warning*: A BigQuery table corresponding to your configuration will be automatically generated upon installing or updating this extension. Manual table creation may result in discrepancies with your configured settings.
 
-If you create, update, delete, or import a document in the specified collection, this extension sends that update to BigQuery. You can then run queries on this mirrored dataset.
+If you create, update, or delete a document in the specified collection, this extension sends that update to BigQuery. You can then run queries on this mirrored dataset.
 
 Note that this extension only listens for _document_ changes in the collection, but not changes in any _subcollection_. You can, though, install additional instances of this extension to specifically listen to a subcollection or other collections in your database. Or if you have the same subcollection across documents in a given collection, you can use `{wildcard}` notation to listen to all those subcollections (for example: `chats/{chatid}/posts`). 
 

firestore-bigquery-export/README.md

@@ -15,7 +15,7 @@ The extension creates and updates a [dataset](https://cloud.google.com/bigquery/
 
 *Warning*: A BigQuery table corresponding to your configuration will be automatically generated upon installing or updating this extension. Manual table creation may result in discrepancies with your configured settings.
 
-If you create, update, delete, or import a document in the specified collection, this extension sends that update to BigQuery. You can then run queries on this mirrored dataset.
+If you create, update, or delete a document in the specified collection, this extension sends that update to BigQuery. You can then run queries on this mirrored dataset.
 
 Note that this extension only listens for _document_ changes in the collection, but not changes in any _subcollection_. You can, though, install additional instances of this extension to specifically listen to a subcollection or other collections in your database. Or if you have the same subcollection across documents in a given collection, you can use `{wildcard}` notation to listen to all those subcollections (for example: `chats/{chatid}/posts`). 
 
@@ -124,6 +124,8 @@ To install an extension, your project must be on the [Blaze (pay as you go) plan
 
 * BigQuery Project ID: Override the default project for BigQuery instance. This can allow updates to be directed to to a BigQuery instance on another GCP project.
 
+* Database ID: Override the default project Firestore database. Learn more about managing multiple Firestore databases [here](https://cloud.google.com/firestore/docs/manage-databases).
+
 * Collection path: What is the path of the collection that you would like to export? You may use `{wildcard}` notation to match a subcollection of all documents in a collection (for example: `chatrooms/{chatid}/posts`). Parent Firestore Document IDs from `{wildcards}`  can be returned in `path_params` as a JSON formatted string.
 
 * Enable Wildcard Column field with Parent Firestore Document IDs: If enabled, creates a column containing a JSON object of all wildcard ids from a documents path.
@@ -146,7 +148,7 @@ essential for the script to insert data into an already partitioned table.)
 
 * BigQuery SQL table clustering: This parameter will allow you to set up Clustering for the BigQuery Table created by the extension. (for example: `data,document_id,timestamp`- no whitespaces). You can select up to 4 comma separated fields. The order of the specified columns determines the sort order of the data. Available schema extensions table fields for clustering: `document_id, document_name, timestamp, event_id, operation, data`.
 
-* Maximum number of synced documents per second: This parameter will set the maximum number of syncronised documents per second with BQ. Please note, any other external updates to a Big Query table will be included within this quota. Ensure that you have a set a low enough number to componsate. Defaults to 10.
+* Maximum number of synced documents per second: This parameter will set the maximum number of syncronised documents per second with BQ. Please note, any other external updates to a Big Query table will be included within this quota. Ensure that you have a set a low enough number to compensate. Defaults to 10.
 
 * Backup Collection Name: This (optional) parameter will allow you to specify a collection for which failed BigQuery updates will be written to.
 

firestore-bigquery-export/extension.yaml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 name: firestore-bigquery-export
-version: 0.1.45
+version: 0.1.46
 specVersion: v1beta
 
 displayName: Stream Firestore to BigQuery
@@ -58,7 +58,7 @@ resources:
       runtime: nodejs18
       eventTrigger:
         eventType: providers/cloud.firestore/eventTypes/document.write
-        resource: projects/${param:PROJECT_ID}/databases/(default)/documents/${param:COLLECTION_PATH}/{documentId}
+        resource: projects/${param:PROJECT_ID}/databases/${param:DATABASE_ID}/documents/${param:COLLECTION_PATH}/{documentId}
 
   - name: fsimportexistingdocs
     type: firebaseextensions.v1beta.function
@@ -68,7 +68,10 @@ resources:
       `IMPORT` and the timestamp of epoch.
     properties:
       runtime: nodejs18
-      taskQueueTrigger: {}
+      taskQueueTrigger:
+        retryConfig:
+          maxAttempts: 15
+          minBackoffSeconds: 60
 
   - name: syncBigQuery
     type: firebaseextensions.v1beta.function
@@ -90,15 +93,21 @@ resources:
       Runs configuration for sycning with BigQuery
     properties:
       runtime: nodejs18
-      taskQueueTrigger: {}
+      taskQueueTrigger:
+        retryConfig:
+          maxAttempts: 15
+          minBackoffSeconds: 60
 
   - name: setupBigQuerySync
     type: firebaseextensions.v1beta.function
     description: >-
       Runs configuration for sycning with BigQuery
     properties:
       runtime: nodejs18
-      taskQueueTrigger: {}
+      taskQueueTrigger:
+        retryConfig:
+          maxAttempts: 15
+          minBackoffSeconds: 60
 
 params:
   - param: DATASET_LOCATION
@@ -180,6 +189,16 @@ params:
     default: ${PROJECT_ID}
     required: true
 
+  - param: DATABASE_ID
+    label: Database ID
+    description: >-
+      Override the default project Firestore database. Learn more about managing
+      multiple Firestore databases
+      [here](https://cloud.google.com/firestore/docs/manage-databases).
+    type: string
+    default: (default)
+    required: true
+
   - param: COLLECTION_PATH
     label: Collection path
     description: >-
@@ -333,7 +352,7 @@ params:
       This parameter will set the maximum number of syncronised documents per
       second with BQ. Please note, any other external updates to a Big Query
       table will be included within this quota. Ensure that you have a set a low
-      enough number to componsate. Defaults to 10.
+      enough number to compensate. Defaults to 10.
     type: string
     validationRegex: ^([1-9]|[1-9][0-9]|[1-4][0-9]{2}|500)$
     validationErrorMessage: Please select a number between 1 and 500