From 5ec25da9a3a35987d7082137280a4f5ab13a29a5 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Mon, 24 Feb 2025 01:47:32 +0100
Subject: [PATCH 01/52] remote function

---
 definitions/output/reports/reports_dynamic.js | 33 +++++++--
 .../output/reports/tech_report_adoption.js    | 12 +++-
 .../output/reports/tech_report_audits.js      | 12 +++-
 .../output/reports/tech_report_categories.js  | 11 ++-
 .../reports/tech_report_core_web_vitals.js    | 12 +++-
 .../output/reports/tech_report_lighthouse.js  | 12 +++-
 .../output/reports/tech_report_page_weight.js | 12 +++-
 .../reports/tech_report_technologies.js       | 11 ++-
 .../output/reports/tech_report_versions.js    | 11 ++-
 includes/constants.js                         |  6 +-
 infra/dataform-export/index.js                | 67 ++++++++-----------
 infra/tf/dataform.tf                          |  2 +-
 infra/tf/functions.tf                         | 21 ++++++
 workflow_settings.yaml                        |  2 +-
 14 files changed, 168 insertions(+), 56 deletions(-)

diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js
index 006234c7..9309ba88 100644
--- a/definitions/output/reports/reports_dynamic.js
+++ b/definitions/output/reports/reports_dynamic.js
@@ -23,9 +23,20 @@ if (iterations.length === 1) {
       }).preOps(ctx => `
 --DELETE FROM ${ctx.self()}
 --WHERE date = '${params.date}';
-  `).query(ctx => `
-/* {"dataform_trigger": "report_complete", "date": "${params.date}", "name": "${metric.id}", "type": "${sql.type}"} */` +
-sql.query(ctx, params))
+      `).query(
+        ctx => sql.query(ctx, params)
+      ).postOps(ctx => `
+SELECT
+  reports.run_export_job(
+    JSON '''{
+      "dataform_trigger": "report_complete",
+      "date": "${params.date}",
+      "name": "${metric.id}",
+      "type": "${sql.type}",
+      "environment": "${constants.environment}"
+    }'''
+  );
+      `)
     })
   })
 } else {
@@ -38,9 +49,19 @@ sql.query(ctx, params))
 DELETE FROM reports.${metric.id}_${sql.type}
 WHERE date = '${params.date}';
 
-/* {"dataform_trigger": "report_complete", "date": "${params.date}", "name": "${metric.id}", "type": "${sql.type}"} */
-INSERT INTO reports.${metric.id}_${sql.type}` +
-        sql.query(ctx, params))
+INSERT INTO reports.${metric.id}_${sql.type}` + sql.query(ctx, params)
+        ).postOps(ctx => `
+SELECT
+  reports.run_export_job(
+    JSON '''{
+      "dataform_trigger": "report_complete",
+      "date": "${params.date}",
+      "name": "${metric.id}",
+      "type": "${sql.type}",
+      "environment": "${constants.environment}"
+    }'''
+  );
+        `)
       })
     })
   })
diff --git a/definitions/output/reports/tech_report_adoption.js b/definitions/output/reports/tech_report_adoption.js
index d6fd203d..9c5b8a66 100644
--- a/definitions/output/reports/tech_report_adoption.js
+++ b/definitions/output/reports/tech_report_adoption.js
@@ -13,7 +13,6 @@ publish('tech_report_adoption', {
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';
 `).query(ctx => `
-/* {"dataform_trigger": "tech_report_complete", "date": "${pastMonth}", "name": "adoption", "type": "report"} */
 SELECT
   date,
   geo,
@@ -32,4 +31,15 @@ GROUP BY
   rank,
   technology,
   version
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "dataform_trigger": "tech_report_complete",
+        "date": "${pastMonth}",
+        "name": "adoption",
+        "type": "report",
+        "environment": "${constants.environment}"
+      }'''
+    );
 `)
diff --git a/definitions/output/reports/tech_report_audits.js b/definitions/output/reports/tech_report_audits.js
index 87fde697..f7aa4ac4 100644
--- a/definitions/output/reports/tech_report_audits.js
+++ b/definitions/output/reports/tech_report_audits.js
@@ -67,7 +67,6 @@ return Object.keys(auditMap).map(function(key) {
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';
 `).query(ctx => `
-/* {"dataform_trigger": "tech_report_complete", "date": "${pastMonth}", "name": "audits", "type": "report"} */
 SELECT
   date,
   geo,
@@ -86,4 +85,15 @@ GROUP BY
   rank,
   technology,
   version
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "dataform_trigger": "tech_report_complete",
+        "date": "${pastMonth}",
+        "name": "audits",
+        "type": "report",
+        "environment": "${constants.environment}"
+      }'''
+    );
 `)
diff --git a/definitions/output/reports/tech_report_categories.js b/definitions/output/reports/tech_report_categories.js
index d73ba218..605983ae 100644
--- a/definitions/output/reports/tech_report_categories.js
+++ b/definitions/output/reports/tech_report_categories.js
@@ -5,7 +5,6 @@ publish('tech_report_categories', {
   type: 'table',
   tags: ['tech_report']
 }).query(ctx => `
-/* {"dataform_trigger": "tech_report_complete", "name": "categories", "type": "dict"} */
 WITH pages AS (
   SELECT DISTINCT
     client,
@@ -91,4 +90,14 @@ FROM (
   FROM pages
   GROUP BY client
 )
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "dataform_trigger": "tech_report_complete",
+        "name": "categories",
+        "type": "dict",
+        "environment": "${constants.environment}"
+      }'''
+    );
 `)
diff --git a/definitions/output/reports/tech_report_core_web_vitals.js b/definitions/output/reports/tech_report_core_web_vitals.js
index cb663d12..5241ab6a 100644
--- a/definitions/output/reports/tech_report_core_web_vitals.js
+++ b/definitions/output/reports/tech_report_core_web_vitals.js
@@ -68,7 +68,6 @@ return Object.values(vitals)
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';
 `).query(ctx => `
-/* {"dataform_trigger": "tech_report_complete", "date": "${pastMonth}", "name": "core_web_vitals", "type": "report"} */
 SELECT
   date,
   geo,
@@ -100,4 +99,15 @@ GROUP BY
   rank,
   technology,
   version
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "dataform_trigger": "tech_report_complete",
+        "date": "${pastMonth}",
+        "name": "core_web_vitals",
+        "type": "report",
+        "environment": "${constants.environment}"
+      }'''
+    );
 `)
diff --git a/definitions/output/reports/tech_report_lighthouse.js b/definitions/output/reports/tech_report_lighthouse.js
index 8d438ecc..021e317e 100644
--- a/definitions/output/reports/tech_report_lighthouse.js
+++ b/definitions/output/reports/tech_report_lighthouse.js
@@ -54,7 +54,6 @@ return Object.values(lighthouse)
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';
 `).query(ctx => `
-/* {"dataform_trigger": "tech_report_complete", "date": "${pastMonth}", "name": "lighthouse", "type": "report"} */
 SELECT
   date,
   geo,
@@ -76,4 +75,15 @@ GROUP BY
   rank,
   technology,
   version
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "dataform_trigger": "tech_report_complete",
+        "date": "${pastMonth}",
+        "name": "lighthouse",
+        "type": "report",
+        "environment": "${constants.environment}"
+      }'''
+    );
 `)
diff --git a/definitions/output/reports/tech_report_page_weight.js b/definitions/output/reports/tech_report_page_weight.js
index b1150971..4a54438f 100644
--- a/definitions/output/reports/tech_report_page_weight.js
+++ b/definitions/output/reports/tech_report_page_weight.js
@@ -48,7 +48,6 @@ return Object.values(pageWeight)
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';
 `).query(ctx => `
-/* {"dataform_trigger": "tech_report_complete", "date": "${pastMonth}", "name": "page_weight", "type": "report"} */
 SELECT
   date,
   geo,
@@ -69,4 +68,15 @@ GROUP BY
   rank,
   technology,
   version
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "dataform_trigger": "tech_report_complete",
+        "date": "${pastMonth}",
+        "name": "page_weight",
+        "type": "report",
+        "environment": "${constants.environment}"
+      }'''
+    );
 `)
diff --git a/definitions/output/reports/tech_report_technologies.js b/definitions/output/reports/tech_report_technologies.js
index 210e8199..5d481ffc 100644
--- a/definitions/output/reports/tech_report_technologies.js
+++ b/definitions/output/reports/tech_report_technologies.js
@@ -5,7 +5,6 @@ publish('tech_report_technologies', {
   type: 'table',
   tags: ['tech_report']
 }).query(ctx => `
-/* {"dataform_trigger": "tech_report_complete", "name": "technologies", "type": "dict"} */
 WITH pages AS (
   SELECT DISTINCT
     client,
@@ -86,4 +85,14 @@ SELECT
     MAX(IF(client = 'mobile', origins, 0)) AS mobile
   ) AS origins
 FROM total_pages
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "dataform_trigger": "tech_report_complete",
+        "name": "technologies",
+        "type": "dict",
+        "environment": "${constants.environment}"
+      }'''
+    );
 `)
diff --git a/definitions/output/reports/tech_report_versions.js b/definitions/output/reports/tech_report_versions.js
index d11fa8f3..bcf6602d 100644
--- a/definitions/output/reports/tech_report_versions.js
+++ b/definitions/output/reports/tech_report_versions.js
@@ -5,7 +5,6 @@ publish('tech_report_versions', {
   type: 'table',
   tags: ['tech_report']
 }).query(ctx => `
-/* {"dataform_trigger": "tech_report_complete", "name": "versions", "type": "dict"} */
 WITH pages AS (
   SELECT DISTINCT
     client,
@@ -61,4 +60,14 @@ SELECT
   'ALL' AS version,
   origins
 FROM total_origins
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "dataform_trigger": "tech_report_complete",
+        "name": "versions",
+        "type": "dict",
+        "environment": "${constants.environment}"
+      }'''
+    );
 `)
diff --git a/includes/constants.js b/includes/constants.js
index 18c59cf6..6b327d1f 100644
--- a/includes/constants.js
+++ b/includes/constants.js
@@ -1,4 +1,4 @@
-const today = (dataform.projectConfig.vars.today ? dataform.projectConfig.vars.today : new Date().toISOString()).substring(0, 10)
+const today = new Date().toISOString().substring(0, 10)
 const currentMonth = today.substring(0, 8) + '01'
 function fnDateUnderscored (dateStr) {
   return dateStr.replaceAll('-', '_')
@@ -10,10 +10,11 @@ function fnPastMonth (monthISOstring) {
 }
 const clients = ['desktop', 'mobile']
 const booleans = ['FALSE', 'TRUE']
+const environment = dataform.projectConfig.vars.environment
 const [
   devTABLESAMPLE,
   devRankFilter
-] = dataform.projectConfig.vars.env_name === 'dev'
+] = environment === 'dev'
   ? [
       'TABLESAMPLE SYSTEM (0.001 PERCENT)',
       'AND rank <= 10000'
@@ -68,6 +69,7 @@ module.exports = {
   fnDateUnderscored,
   clients,
   booleans,
+  environment,
   devTABLESAMPLE,
   devRankFilter,
   DataformTemplateBuilder
diff --git a/infra/dataform-export/index.js b/infra/dataform-export/index.js
index f3c9dfc8..f215203c 100644
--- a/infra/dataform-export/index.js
+++ b/infra/dataform-export/index.js
@@ -28,6 +28,15 @@ async function callRunJob (payload = {}) {
   console.info(`Job initialized: ${operation.name}`)
 }
 
+function hasRequiredKeys (obj) {
+  const requiredKeys = ['dataform_trigger', 'name', 'type', 'environment']
+  if (obj.type === 'report') {
+    requiredKeys.push('date')
+  }
+
+  return requiredKeys.every(key => Object.hasOwn(obj, key))
+}
+
 /**
  * Handle incoming message and trigger the appropriate action.
  *
@@ -35,50 +44,32 @@ async function callRunJob (payload = {}) {
  * @param {object} res Cloud Function response context.
  */
 functions.http('dataform-export', async (req, res) => {
+  console.log(JSON.stringify(req.body))
   try {
-    const message = req.body.message
-    if (!message) {
-      console.log(`no message received: ${JSON.stringify(req.body)}`)
-      res.status(400).send('Bad Request: no message received')
-    }
-
-    const messageData = (message.data && JSON.parse(Buffer.from(message.data, 'base64').toString('utf-8'))) || message
-    if (!messageData) {
-      console.info(JSON.stringify(message))
-      res.status(400).send('Bad Request: invalid message format')
+    const payload = req.body.calls[0][0]
+    if (!payload) {
+      res.status(400).json({
+        replies: [400],
+        errorMessage: 'Bad Request: no payload received'
+      })
     }
 
-    const query = messageData.protoPayload.serviceData.jobCompletedEvent.job.jobConfiguration.query.query
-    if (!query) {
-      console.log(`no query found: ${JSON.stringify(messageData)}`)
-      res.status(400).send('Bad Request: no query found')
+    if (!hasRequiredKeys(payload)) {
+      res.status(400).json({
+        replies: [400],
+        errorMessage: 'Bad Request: unexpected payload structure'
+      })
     }
 
-    const repoEnvironment = messageData.protoPayload.serviceData.jobCompletedEvent.job.jobConfiguration.labels.dataform_repository_id
-    if (!repoEnvironment) {
-      console.log(`no repo environment found: ${JSON.stringify(messageData)}`)
-      res.status(400).send('Bad Request: no repo environment found')
-    }
-
-    const regex = /\/\* ({"dataform_trigger":.+) \*\//
-    const reportConfig = regex.exec(query)
-    if (!reportConfig) {
-      console.log(`no trigger config found: ${query}`)
-      res.status(400).send('Bad Request: no trigger config found')
-    }
-
-    const eventData = JSON.parse(reportConfig[1])
-    if (!eventData) {
-      console.log(`no event data found: ${reportConfig[1]}`)
-      res.status(400).send('Bad Request: no event data found')
-    }
-    eventData.environment = repoEnvironment === 'crawl-data' ? 'prod' : 'dev'
-    await callRunJob(eventData)
+    await callRunJob(payload)
 
-    res.status(200).send('OK')
+    res.status(200).json({
+      replies: [200]
+    })
   } catch (error) {
-    console.log(JSON.stringify(req.body))
-    console.error(error)
-    res.status(500).send('Internal Server Error')
+    res.status(400).json({
+      replies: [400],
+      errorMessage: error
+    })
   }
 })
diff --git a/infra/tf/dataform.tf b/infra/tf/dataform.tf
index 9712e1fa..d514a6ff 100644
--- a/infra/tf/dataform.tf
+++ b/infra/tf/dataform.tf
@@ -17,7 +17,7 @@ locals {
     "blink_features",
 
     // Reports
-    "core_web_vitals",  // TODO: Remove after tech report migration
+    "core_web_vitals", // TODO: Remove after tech report migration
     "reports",
 
     // Service
diff --git a/infra/tf/functions.tf b/infra/tf/functions.tf
index 4cc6213f..8b422bb2 100644
--- a/infra/tf/functions.tf
+++ b/infra/tf/functions.tf
@@ -17,3 +17,24 @@ resource "google_bigquery_dataset_iam_member" "cloud_function_dataset_reader_rol
   role       = "roles/bigquery.dataViewer"
   member     = "serviceAccount:${local.function_identity}"
 }
+
+resource "google_bigquery_connection" "connection" {
+   connection_id = "my-connection"
+   location      = "US"
+   friendly_name = "👋"
+   description   = "a riveting description"
+   cloud_resource {}
+}
+
+resource "google_bigquery_connection" "procedures" {
+   connection_id = "procedures"
+   location      = "US"
+   spark {
+   }
+}
+
+resource "google_project_iam_member" "bigquery-remote-functions-identity" {
+  project = local.project
+  role    = "roles/run.invoker"
+  member  = "serviceAccount:bqcx-226352634162-1s4t@gcp-sa-bigquery-condel.iam.gserviceaccount.com"
+}
diff --git a/workflow_settings.yaml b/workflow_settings.yaml
index bb899418..6d3e2f6b 100644
--- a/workflow_settings.yaml
+++ b/workflow_settings.yaml
@@ -3,4 +3,4 @@ defaultLocation: US
 defaultAssertionDataset: dataform_assertions
 
 vars:
-  env_name: prod # MUST be equal 'prod' in main branch, enables processing sampled data
+  environment: prod # MUST be equal 'prod' in main branch, enables processing sampled data

From b3297c5583fb6577f92f7b0b3af5452b27be19c6 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Mon, 24 Feb 2025 02:01:05 +0100
Subject: [PATCH 02/52] connection

---
 infra/tf/functions.tf | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/infra/tf/functions.tf b/infra/tf/functions.tf
index 8b422bb2..fde30c05 100644
--- a/infra/tf/functions.tf
+++ b/infra/tf/functions.tf
@@ -18,14 +18,6 @@ resource "google_bigquery_dataset_iam_member" "cloud_function_dataset_reader_rol
   member     = "serviceAccount:${local.function_identity}"
 }
 
-resource "google_bigquery_connection" "connection" {
-   connection_id = "my-connection"
-   location      = "US"
-   friendly_name = "👋"
-   description   = "a riveting description"
-   cloud_resource {}
-}
-
 resource "google_bigquery_connection" "procedures" {
    connection_id = "procedures"
    location      = "US"
@@ -33,7 +25,7 @@ resource "google_bigquery_connection" "procedures" {
    }
 }
 
-resource "google_project_iam_member" "bigquery-remote-functions-identity" {
+resource "google_project_iam_member" "bigquery-functions-identity" {
   project = local.project
   role    = "roles/run.invoker"
   member  = "serviceAccount:bqcx-226352634162-1s4t@gcp-sa-bigquery-condel.iam.gserviceaccount.com"

From 76d39f138b913cd8a5ac8275e06cc9e8fec7084f Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Mon, 24 Feb 2025 19:41:56 +0100
Subject: [PATCH 03/52] masthead update

---
 infra/tf/.terraform.lock.hcl | 52 ++++++++++++++++++------------------
 infra/tf/functions.tf        |  2 +-
 infra/tf/masthead/main.tf    |  2 +-
 3 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/infra/tf/.terraform.lock.hcl b/infra/tf/.terraform.lock.hcl
index 4f090a1d..7f2cd4e8 100644
--- a/infra/tf/.terraform.lock.hcl
+++ b/infra/tf/.terraform.lock.hcl
@@ -21,41 +21,41 @@ provider "registry.terraform.io/hashicorp/archive" {
 }
 
 provider "registry.terraform.io/hashicorp/google" {
-  version     = "6.17.0"
+  version     = "6.21.0"
   constraints = ">= 6.13.0"
   hashes = [
-    "h1:siQ5DPLcE3KCbl55zr9yE1ceecICvbZ3MKkLbIcHZ04=",
-    "zh:2ae1ba33889babf740298f131c3151477c638a6d8dc2d850f207380ae91d5ee0",
-    "zh:2b950b0f4dcb1f79e10ad9611fc1573114028be423af742eb9b5027d1e1127fc",
-    "zh:4557ce5a9ce78e365af99c15c3a2d4d37a246535d0d62182a66cfc1c9de53cbd",
-    "zh:5ced8255a5cd868ebd6a0ba377b5016f578be402daea7479e488c109a74e8339",
-    "zh:6b7666678f6238637c7f78020edb8405669804a18ae580296419fb4179642cf6",
-    "zh:8677c153477daf1b636421a00633f25022b8c33fc803699d6ea6f89b75b4554b",
-    "zh:9f85498e26bf90049c252e6220a5a47cff88a4cd249e08845c59bd4c16aa48f3",
-    "zh:dce93c05d1852f1c692566c2ebf7200cb98aa059301044c2211c10319354c680",
-    "zh:df72b36e76e0721904c63eab34191bc9c4ccf93d067c2a0d455dd8bb39e73b66",
-    "zh:e9a9e8d8ae14ab6e661f3f9b07c5edec60507203dac7d2f187dc716317f4d79c",
+    "h1:M4X6/r6w8QrkzoOu/8+61+fkYy4+kqNt1fbHE7Igvgg=",
+    "zh:1c2462367d92f6f8f6c527115905f7cca78e48cf5d5bc7448d3beeb7c9e895eb",
+    "zh:3644dbd09c3740e6d843e035de34a74ed41ffc32e7ed04a19aecddc4c57334cc",
+    "zh:3a586bbb9a9c6463c975a94ddd4671f2a84992a2c169bfb2f5053c2cea55849c",
+    "zh:4ae96672e6a52a077760a11c95946ec9d3f84f7ab84c0ba3c0cb66c3d3580d59",
+    "zh:9c26b3dbc1f9a594d1d07b6a25ce089f8463e8331324f4ecde73829e9d1d5ee5",
+    "zh:b99a602111d6ca5842c852ac1eff5c009f1d75492e355ea25f3dbd6e008e4d9a",
+    "zh:d45100c41c940c35c07fae2876f6cc654328d405077f01d268e8bd5a25b56c30",
+    "zh:de6e14e85a9ea2322a4fd971fde3b71071e7b6435a12dbcd3b8c5f42765e8b3c",
+    "zh:e22f6b54cfebb0c1a0991d83adc83b3d454ba6d9b5c21574af135799b488ed66",
     "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
-    "zh:fb92287bca4fc7b49666c644ca7789e4acf5b17317acb963f138c0ae6347a289",
+    "zh:f6964268874a15788dfed47a26cb880718f47c94cba7c8a0284b70921fec807b",
+    "zh:ff51b3e83149798ce6c7545688fe3b9703b04d5c0376cd55215a93767144f40e",
   ]
 }
 
 provider "registry.terraform.io/hashicorp/google-beta" {
-  version     = "6.17.0"
+  version     = "6.21.0"
   constraints = ">= 6.13.0"
   hashes = [
-    "h1:/Ld1YLT//KSQ5kOI8i9fjKvLGJLYXux/9ydCk1EE67E=",
-    "zh:41018bb792fbc6eeb389be133ebeb88df5c0c7ab1cdd70cb49ef3b834b5253ef",
-    "zh:464a0432a42a0973a7cdaf40713a0e54adf74a18db2d9390b00ab691a7cbab14",
-    "zh:57d8f8c2f8d2ea2512ba73caf58b80b6643e268e63dd33aa6b3908f8e9c92e8c",
-    "zh:5a7e90f80f6a8fe19597053565565c4d85efd9896cbe28038c8e1f9452acef74",
-    "zh:5c5ad4eed1bc1c42c088555aa90c99e499b2904e4de0009aacf57fff90ebb2de",
-    "zh:6c950ac6dc08c4db26762717907109665989bb3c6faa0be2db8bf65f82112eaa",
-    "zh:846c821a7664b29569626dcba87667416b399a506ca86f045263e3b918dc73c6",
-    "zh:e06a2ac6afa592127e01768bf3b47051ac010e8c7ddc515dbd42b232d2ecfa2e",
-    "zh:ea2eec97f55eff6cf5cc67f41b1d4d4ec4403b1f61cd762dc1c028ba50e3b349",
-    "zh:f0e102bfdb2c70b747e7a439b31fe2c03480b598f46193325287a51ef744d2fa",
+    "h1:emD7GhPVQW+twGaoChbc+fVEgUdRlU3Qvi3R7aj8q2I=",
+    "zh:13945569f2f0859199f21d74395f4263ec576572db29fc7ab0c6af7b2c7611e7",
+    "zh:459e5114343509144397f7114a15e5eb4435e786fe4ab7a1d8809a3def0364f6",
+    "zh:45a363a8f31bfe3b238230949f568b0f591e6f5bebad839bcd13cd5b937ff6df",
+    "zh:86a8b26a4fd45da6561f87b6b01bc5d41ffe0dd05d285f144accc7c97a16a1f3",
+    "zh:aadd5a8828c87f482cf551224cc3ecfaa38d9ba8d6da54850a9dcdb24ffbab3a",
+    "zh:ae16ed6f8b971de85b28bf040c60e72dcd0d310f86288ad8cc52161c2208b461",
+    "zh:bc6c0f0147b78e103cd086acc29b18110ef3f84f970ea0291064c6b3552c133a",
+    "zh:bc796494f601caf538a83662c13fa7f43d118572ef6666bd1e163f8f17ce6b0e",
+    "zh:bce97850c2855eee3b8f94fa540bbe2ad4fe75ada841aa8e672140bb7d179dda",
+    "zh:bd7420d1c03cc72730e4b718d184ee769dc3dd4247606751b567a5ac416705a0",
+    "zh:f157138eecd0fdb1080994641521c50c7ab8fff0a5f3753f07915a7475e2c7fd",
     "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
-    "zh:f76136bfb3c9c0848ff84a3bf98fba1b61c13124ade4194020d18583951b9df2",
   ]
 }
diff --git a/infra/tf/functions.tf b/infra/tf/functions.tf
index fde30c05..ee769352 100644
--- a/infra/tf/functions.tf
+++ b/infra/tf/functions.tf
@@ -25,7 +25,7 @@ resource "google_bigquery_connection" "procedures" {
    }
 }
 
-resource "google_project_iam_member" "bigquery-functions-identity" {
+resource "google_project_iam_member" "bigquery-functions-connector" {
   project = local.project
   role    = "roles/run.invoker"
   member  = "serviceAccount:bqcx-226352634162-1s4t@gcp-sa-bigquery-condel.iam.gserviceaccount.com"
diff --git a/infra/tf/masthead/main.tf b/infra/tf/masthead/main.tf
index 01c97d7b..246d933d 100644
--- a/infra/tf/masthead/main.tf
+++ b/infra/tf/masthead/main.tf
@@ -52,7 +52,7 @@ resource "google_project_iam_member" "masthead_pubsub_publisherer_member" {
 resource "google_project_iam_custom_role" "masthead_bq_meta_reader" {
   project     = var.project
   description = "Masthead BigQuery assets metadata reader"
-  permissions = ["bigquery.datasets.get", "bigquery.tables.get", "bigquery.tables.list"]
+  permissions = ["bigquery.datasets.get", "bigquery.tables.get", "bigquery.tables.list", "bigquery.routines.get", "bigquery.routines.list"]
   role_id     = "masthead_bq_meta_reader"
   stage       = "GA"
   title       = "masthead_bq_meta_reader"

From 7ce60cd9857fe12c3c0f84816c84f289670cfe35 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Mon, 24 Feb 2025 19:43:42 +0100
Subject: [PATCH 04/52] formatting

---
 infra/tf/functions.tf | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/infra/tf/functions.tf b/infra/tf/functions.tf
index ee769352..ed444746 100644
--- a/infra/tf/functions.tf
+++ b/infra/tf/functions.tf
@@ -19,10 +19,10 @@ resource "google_bigquery_dataset_iam_member" "cloud_function_dataset_reader_rol
 }
 
 resource "google_bigquery_connection" "procedures" {
-   connection_id = "procedures"
-   location      = "US"
-   spark {
-   }
+  connection_id = "procedures"
+  location      = "US"
+  spark {
+  }
 }
 
 resource "google_project_iam_member" "bigquery-functions-connector" {

From 5bc3de9e99952f6146b2031244d334b940e0359d Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Sun, 2 Mar 2025 00:26:45 +0100
Subject: [PATCH 05/52] extend description

---
 README.md                              | 36 +++++++++++++++++++++++---
 infra/bigquery_export_spark/Dockerfile | 25 ++++++++++++++++++
 infra/dataform-export/index.js         | 16 +++++-------
 3 files changed, 65 insertions(+), 12 deletions(-)
 create mode 100644 infra/bigquery_export_spark/Dockerfile

diff --git a/README.md b/README.md
index 6cd53b6f..eba8b55d 100644
--- a/README.md
+++ b/README.md
@@ -47,7 +47,7 @@ Consumers:
 
 ### Triggering workflows
 
-In order to unify the workflow triggering mechanism, we use [a Cloud Run function](./src/README.md) that can be invoked in a number of ways (e.g. listen to PubSub messages), do intermediate checks and trigger the particular Dataform workflow execution configuration.
+In order to unify the workflow triggering mechanism, we use [a Cloud Run function](./infra/README.md) that can be invoked in a number of ways (e.g. listen to PubSub messages), do intermediate checks and trigger the particular Dataform workflow execution configuration.
 
 ## Contributing
 
@@ -59,5 +59,35 @@ In order to unify the workflow triggering mechanism, we use [a Cloud Run functio
 
 #### Workspace hints
 
-1. In `workflow_settings.yaml` set `env_name: dev` to process sampled data.
-2. In `includes/constants.js` set `today` or other variables to a custome value.
+1. In `workflow_settings.yaml` set `environment: dev` to process sampled data.
+2. For development and testing, you can modify variables in `includes/constants.js`, but note that these are programmatically generated.
+
+## Repository Structure
+
+- `definitions/` - Contains the core Dataform SQL definitions and declarations
+  - `output/` - Contains the main pipeline transformation logic
+  - `declarations/` - Contains referenced tables/views declarations and other resources definitions
+- `includes/` - Contains shared JavaScript utilities and constants
+- `infra/` - Infrastructure code and deployment configurations
+  - `dataform-trigger/` - Cloud Run function for workflow automation
+  - `tf/` - Terraform configurations
+  - `bigquery-export/` - BigQuery export configurations
+- `docs/` - Additional documentation
+
+## Development Setup
+
+1. Install dependencies:
+   ```bash
+   npm install
+   ```
+
+2. Available Scripts:
+   - `npm run format` - Format code using Standard.js, fix markdown issues, and format Terraform files
+   - `npm run lint` - Run linting checks on JavaScript, markdown files, and compile Dataform configs
+
+## Code Quality
+
+This repository uses:
+- Standard.js for JavaScript code style
+- Markdownlint for markdown file formatting
+- Dataform's built-in compiler for SQL validation
diff --git a/infra/bigquery_export_spark/Dockerfile b/infra/bigquery_export_spark/Dockerfile
new file mode 100644
index 00000000..d66edf70
--- /dev/null
+++ b/infra/bigquery_export_spark/Dockerfile
@@ -0,0 +1,25 @@
+# Recommendation: Use Debian 12.
+FROM python:3.12-slim
+
+# Suppress interactive prompts
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install utilities required by Spark scripts.
+RUN apt update && apt install -y procps tini libjemalloc2
+
+# Enable jemalloc2 as default memory allocator
+ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2
+
+# Install packages.
+RUN pip install --no-cache-dir \
+    google-cloud-bigquery \
+    google-cloud-storage \
+    google-cloud-firestore \
+    grpcio \
+    google-cloud-secret-manager
+
+# Create the 'spark' group/user.
+# The GID and UID must be 1099. Home directory is required.
+RUN groupadd -g 1099 spark
+RUN useradd -u 1099 -g 1099 -d /home/spark -m spark
+USER spark
diff --git a/infra/dataform-export/index.js b/infra/dataform-export/index.js
index f215203c..7d4fc47f 100644
--- a/infra/dataform-export/index.js
+++ b/infra/dataform-export/index.js
@@ -29,12 +29,9 @@ async function callRunJob (payload = {}) {
 }
 
 function hasRequiredKeys (obj) {
-  const requiredKeys = ['dataform_trigger', 'name', 'type', 'environment']
-  if (obj.type === 'report') {
-    requiredKeys.push('date')
-  }
-
-  return requiredKeys.every(key => Object.hasOwn(obj, key))
+  const baseRequiredKeys = ['dataform_trigger', 'name', 'type', 'environment']
+  const requiredKeys = obj.type === 'report' ? [...baseRequiredKeys, 'date'] : baseRequiredKeys
+  return requiredKeys.every(key => key in obj)
 }
 
 /**
@@ -50,21 +47,22 @@ functions.http('dataform-export', async (req, res) => {
     if (!payload) {
       res.status(400).json({
         replies: [400],
-        errorMessage: 'Bad Request: no payload received'
+        errorMessage: 'Bad Request: no payload received, expected JSON object'
       })
     }
 
     if (!hasRequiredKeys(payload)) {
       res.status(400).json({
         replies: [400],
-        errorMessage: 'Bad Request: unexpected payload structure'
+        errorMessage: 'Bad Request: unexpected payload structure, required keys: dataform_trigger, name, type, environment, (optional)date'
       })
     }
 
     await callRunJob(payload)
 
     res.status(200).json({
-      replies: [200]
+      replies: [200],
+      message: 'Export job initialized'
     })
   } catch (error) {
     res.status(400).json({

From 2470a8bedc34818f316c7406cdabff8f47dbf5c3 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Sun, 2 Mar 2025 17:29:37 +0100
Subject: [PATCH 06/52] reservation off

---
 definitions/declarations/httparchive.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/definitions/declarations/httparchive.js b/definitions/declarations/httparchive.js
index db4958b1..2ba01d99 100644
--- a/definitions/declarations/httparchive.js
+++ b/definitions/declarations/httparchive.js
@@ -42,7 +42,7 @@ ORDER BY cnt_pages DESC
 )
 
 operate('create_reservation_assignment')
-  .tags(['crawl_complete'])
+  //.tags(['crawl_complete'])
   .queries(ctx => `
 CREATE ASSIGNMENT
 \`httparchive.region-us.retrospective-reprocessing.pipeline\`
@@ -52,8 +52,8 @@ OPTIONS (
 `)
 
 operate('drop_reservation_assignment')
-  .dependencies(['requests_10k'])
-  .tags(['crawl_complete'])
+  //.dependencies(['requests_10k'])
+  //.tags(['crawl_complete'])
   .queries(ctx => `
 DROP ASSIGNMENT IF EXISTS
 \`httparchive.region-us.retrospective-reprocessing.pipeline\`

From c2b84e0b6682f555462c8fc40a10cb6212b424ca Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Sun, 2 Mar 2025 19:38:24 +0100
Subject: [PATCH 07/52] tf update

---
 README.md                               |  2 +
 definitions/declarations/httparchive.js |  6 +--
 infra/tf/.terraform.lock.hcl            | 52 ++++++++++++-------------
 infra/tf/bigquery_export/main.tf        | 14 +------
 4 files changed, 32 insertions(+), 42 deletions(-)

diff --git a/README.md b/README.md
index eba8b55d..9f8683cc 100644
--- a/README.md
+++ b/README.md
@@ -77,6 +77,7 @@ In order to unify the workflow triggering mechanism, we use [a Cloud Run functio
 ## Development Setup
 
 1. Install dependencies:
+
    ```bash
    npm install
    ```
@@ -88,6 +89,7 @@ In order to unify the workflow triggering mechanism, we use [a Cloud Run functio
 ## Code Quality
 
 This repository uses:
+
 - Standard.js for JavaScript code style
 - Markdownlint for markdown file formatting
 - Dataform's built-in compiler for SQL validation
diff --git a/definitions/declarations/httparchive.js b/definitions/declarations/httparchive.js
index 2ba01d99..f7b2e86d 100644
--- a/definitions/declarations/httparchive.js
+++ b/definitions/declarations/httparchive.js
@@ -42,7 +42,7 @@ ORDER BY cnt_pages DESC
 )
 
 operate('create_reservation_assignment')
-  //.tags(['crawl_complete'])
+  // .tags(['crawl_complete'])
   .queries(ctx => `
 CREATE ASSIGNMENT
 \`httparchive.region-us.retrospective-reprocessing.pipeline\`
@@ -52,8 +52,8 @@ OPTIONS (
 `)
 
 operate('drop_reservation_assignment')
-  //.dependencies(['requests_10k'])
-  //.tags(['crawl_complete'])
+  // .dependencies(['requests_10k'])
+  // .tags(['crawl_complete'])
   .queries(ctx => `
 DROP ASSIGNMENT IF EXISTS
 \`httparchive.region-us.retrospective-reprocessing.pipeline\`
diff --git a/infra/tf/.terraform.lock.hcl b/infra/tf/.terraform.lock.hcl
index 7f2cd4e8..1aa60309 100644
--- a/infra/tf/.terraform.lock.hcl
+++ b/infra/tf/.terraform.lock.hcl
@@ -21,41 +21,41 @@ provider "registry.terraform.io/hashicorp/archive" {
 }
 
 provider "registry.terraform.io/hashicorp/google" {
-  version     = "6.21.0"
+  version     = "6.23.0"
   constraints = ">= 6.13.0"
   hashes = [
-    "h1:M4X6/r6w8QrkzoOu/8+61+fkYy4+kqNt1fbHE7Igvgg=",
-    "zh:1c2462367d92f6f8f6c527115905f7cca78e48cf5d5bc7448d3beeb7c9e895eb",
-    "zh:3644dbd09c3740e6d843e035de34a74ed41ffc32e7ed04a19aecddc4c57334cc",
-    "zh:3a586bbb9a9c6463c975a94ddd4671f2a84992a2c169bfb2f5053c2cea55849c",
-    "zh:4ae96672e6a52a077760a11c95946ec9d3f84f7ab84c0ba3c0cb66c3d3580d59",
-    "zh:9c26b3dbc1f9a594d1d07b6a25ce089f8463e8331324f4ecde73829e9d1d5ee5",
-    "zh:b99a602111d6ca5842c852ac1eff5c009f1d75492e355ea25f3dbd6e008e4d9a",
-    "zh:d45100c41c940c35c07fae2876f6cc654328d405077f01d268e8bd5a25b56c30",
-    "zh:de6e14e85a9ea2322a4fd971fde3b71071e7b6435a12dbcd3b8c5f42765e8b3c",
-    "zh:e22f6b54cfebb0c1a0991d83adc83b3d454ba6d9b5c21574af135799b488ed66",
+    "h1:Gr39ABNw+A6lwP2gPG+yCzGmU5T97iI5qT0XLCd3Dh4=",
+    "zh:032dd78eff887a673a1067008a8e47a69983bbcea9f41832320470247a76863a",
+    "zh:1af89f75142cf9c54499a466c8dc7055e2bbf02771a6b8c8cd57eb13dce9a800",
+    "zh:3696a8e72c6cef80fec3c3574fc8519f0410f23f6dc3e3540d2f03345c140d38",
+    "zh:58a15c71ae128ff64117c1c6b9ccf8ab2ac3e8f9c2c52957d8327f93495f62b1",
+    "zh:70ba2909611e8d1cc8009567e50e195c4269e6582d6a6fa0bce0d4e6313ab8d5",
+    "zh:8f8489d1eb8c189d59dc85e519e51ab4c4b1940e4d72450ae130ba752028fa01",
+    "zh:99c8c4e8dc67a7ab597d46ed566f64c4409761276f34bd863457a37254620fa6",
+    "zh:9b24d53440e8d7e06020e7b3aeca0dde4f1a3ef997d7da05ff3acb918896fcc2",
+    "zh:b3667fd6057997dbf0bd0179ddf686c272d4ab4fc7da6a03fdf2fad31ad4ecb5",
+    "zh:cc6df6d2291a337a5f434b7335c693164c6987604b9690d6b953b796d8eaa08a",
+    "zh:d871f39c3c5b63995793c9a70f107e52ca699d211c770a6e7ebc398ba59bcdc7",
     "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
-    "zh:f6964268874a15788dfed47a26cb880718f47c94cba7c8a0284b70921fec807b",
-    "zh:ff51b3e83149798ce6c7545688fe3b9703b04d5c0376cd55215a93767144f40e",
   ]
 }
 
 provider "registry.terraform.io/hashicorp/google-beta" {
-  version     = "6.21.0"
+  version     = "6.23.0"
   constraints = ">= 6.13.0"
   hashes = [
-    "h1:emD7GhPVQW+twGaoChbc+fVEgUdRlU3Qvi3R7aj8q2I=",
-    "zh:13945569f2f0859199f21d74395f4263ec576572db29fc7ab0c6af7b2c7611e7",
-    "zh:459e5114343509144397f7114a15e5eb4435e786fe4ab7a1d8809a3def0364f6",
-    "zh:45a363a8f31bfe3b238230949f568b0f591e6f5bebad839bcd13cd5b937ff6df",
-    "zh:86a8b26a4fd45da6561f87b6b01bc5d41ffe0dd05d285f144accc7c97a16a1f3",
-    "zh:aadd5a8828c87f482cf551224cc3ecfaa38d9ba8d6da54850a9dcdb24ffbab3a",
-    "zh:ae16ed6f8b971de85b28bf040c60e72dcd0d310f86288ad8cc52161c2208b461",
-    "zh:bc6c0f0147b78e103cd086acc29b18110ef3f84f970ea0291064c6b3552c133a",
-    "zh:bc796494f601caf538a83662c13fa7f43d118572ef6666bd1e163f8f17ce6b0e",
-    "zh:bce97850c2855eee3b8f94fa540bbe2ad4fe75ada841aa8e672140bb7d179dda",
-    "zh:bd7420d1c03cc72730e4b718d184ee769dc3dd4247606751b567a5ac416705a0",
-    "zh:f157138eecd0fdb1080994641521c50c7ab8fff0a5f3753f07915a7475e2c7fd",
+    "h1:i/TKWST1xe697zDOVDeHcXpjZ4TBxxJK5X7+DS4ozk4=",
+    "zh:0801ceda0de3f61745d379aef0bfe6c3a918c597a097f91e64791fb77d44dd68",
+    "zh:3b8b48de49892f25af14ce0e9abde055a3b008011dd9f7907706067348c32126",
+    "zh:74a20663b00e0eb8bcb0305c3180e7061dd5f0b1d77f36768c3f58f7a773672f",
+    "zh:763e2f0e69a03a4c85e9472b2c5af13e683519b92e3c4bad8fa4ecd650c2b256",
+    "zh:99e9379b1511fd75917a5bab01be0fbec8ac45f4a2b811571ca8cec7326f5f6d",
+    "zh:b28199d64772d7536704554b0fe66954e6c7cef1e15d4a488252c8da8a6b3648",
+    "zh:ce467d73661458e88163a9e33e2142480bf01aaf3477a87a45bbb9227b65c2ea",
+    "zh:d8fb5ee8d07bcaf717f09cebe27fe247a1c9a6a253476bba64a3048f094c377b",
+    "zh:dbdb5dc2f18b1b48b501df2f3a4f201c8646b0c765d01b9fe69116e81d71ce53",
+    "zh:e3e5f2a44734bc23c4fac7b8e4f2b368f9bbde68295129ec3290fa135d1de224",
+    "zh:eba860faa6b49d917460eb8f3594bda0be8b8ca3f5f17465096148717d5f600e",
     "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
   ]
 }
diff --git a/infra/tf/bigquery_export/main.tf b/infra/tf/bigquery_export/main.tf
index 39814da2..f47e132f 100644
--- a/infra/tf/bigquery_export/main.tf
+++ b/infra/tf/bigquery_export/main.tf
@@ -4,7 +4,7 @@ terraform {
   required_providers {
     archive = {
       source  = "hashicorp/archive"
-      version = "2.6.0"
+      version = ">= 2.6.0"
     }
     google = {
       source  = "hashicorp/google"
@@ -13,18 +13,6 @@ terraform {
   }
 }
 
-data "archive_file" "zip" {
-  type        = "zip"
-  source_dir  = "../${var.function_name}/"
-  output_path = "./tmp/${var.function_name}.zip"
-}
-
-resource "google_storage_bucket_object" "zource" {
-  bucket = "gcf-v2-uploads-${var.project_number}-${var.region}"
-  name   = "${var.function_name}_${data.archive_file.zip.id}.zip"
-  source = data.archive_file.zip.output_path
-}
-
 resource "google_cloud_run_v2_job" "bigquery_export" {
   name     = var.function_name
   location = var.region

From 54a3e2f4b2cdd7c07cce6332cda92be2aa5e4ecf Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Sun, 2 Mar 2025 20:03:18 +0100
Subject: [PATCH 08/52] dataform export routine

---
 infra/tf/bigquery_export/main.tf  |  1 -
 infra/tf/dataform_export/main.tf  | 68 ++++++++-----------------------
 infra/tf/dataform_trigger/main.tf |  2 +-
 3 files changed, 17 insertions(+), 54 deletions(-)

diff --git a/infra/tf/bigquery_export/main.tf b/infra/tf/bigquery_export/main.tf
index f47e132f..4347e806 100644
--- a/infra/tf/bigquery_export/main.tf
+++ b/infra/tf/bigquery_export/main.tf
@@ -41,4 +41,3 @@ resource "google_cloud_run_v2_job" "bigquery_export" {
     }
   }
 }
-
diff --git a/infra/tf/dataform_export/main.tf b/infra/tf/dataform_export/main.tf
index 3810ab1e..00960db6 100644
--- a/infra/tf/dataform_export/main.tf
+++ b/infra/tf/dataform_export/main.tf
@@ -4,7 +4,7 @@ terraform {
   required_providers {
     archive = {
       source  = "hashicorp/archive"
-      version = "2.6.0"
+      version = ">= 2.6.0"
     }
     google = {
       source  = "hashicorp/google"
@@ -48,58 +48,22 @@ resource "google_cloudfunctions2_function" "dataform_export" {
   }
 }
 
-# Pub/Sub Topic to trigger Crawl Data Dataform workflow
-resource "google_pubsub_topic" "bigquery_data_updated" {
-  #checkov:skip=CKV_GCP_83:Ensure PubSub Topics are encrypted with Customer Supplied Encryption Keys (CSEK)
-  name    = "bigquery-data-updated"
-  project = var.project
-}
-
-# Logs sink for Dataform triggers
-resource "google_logging_project_sink" "dataform_export_triggers" {
-  name        = "dataform-export-triggers"
-  destination = "pubsub.googleapis.com/projects/${var.project}/topics/bigquery-data-updated"
-  filter      = <<EOT
--- dataform
-protoPayload.authenticationInfo.principalEmail="service-226352634162@gcp-sa-dataform.iam.gserviceaccount.com"
-protoPayload.serviceData.jobCompletedEvent.job.jobConfiguration.labels.dataform_repository_id=~"crawl-data"
-protoPayload.resourceName=~"projects/httparchive/jobs/dataform-gcp-"
-
---successful query
-protoPayload.serviceData.jobCompletedEvent.job.jobStatus.state="DONE"
--protoPayload.serviceData.jobCompletedEvent.job.jobStatus.error.message:*
-
---check for trigger config
-protoPayload.serviceData.jobCompletedEvent.job.jobConfiguration.query.query=~"/* {\"dataform_trigger\": "
-EOT
-  project     = var.project
-}
+resource "google_bigquery_routine" "run_export_job" {
+  dataset_id = "reports"
+  routine_id = "run_export_job"
+  routine_type = "SCALAR_FUNCTION"
+  definition_body = ""
+  description = "Export data from Google BigQuery.\nExample payload JSON: {\"dataform_trigger\": \"tech_report_complete\", \"date\": \"${pastMonth}\", \"name\": \"adoption\", \"type\": \"report\"}"
 
-# Topic Subscription for dataform_export function
-resource "google_pubsub_subscription" "dataform_export" {
-  ack_deadline_seconds         = 60
-  enable_exactly_once_delivery = false
-  enable_message_ordering      = false
-  filter                       = null
-  labels                       = {}
-  message_retention_duration   = "3600s"
-  name                         = google_cloudfunctions2_function.dataform_export.name
-  project                      = var.project
-  retain_acked_messages        = false
-  topic                        = google_pubsub_topic.bigquery_data_updated.name
-  expiration_policy {
-    ttl = ""
-  }
-  push_config {
-    attributes    = {}
-    push_endpoint = google_cloudfunctions2_function.dataform_export.service_config[0].uri
-    oidc_token {
-      audience              = google_cloudfunctions2_function.dataform_export.service_config[0].uri
-      service_account_email = var.function_identity
-    }
+  arguments {
+    name      = "payload"
+    data_type = "{\"typeKind\" :  \"JSON\"}"
   }
-  retry_policy {
-    maximum_backoff = "60s"
-    minimum_backoff = "10s"
+  return_type = "{\"typeKind\" :  \"INT64\"}"
+
+  remote_function_options {
+    endpoint          = google_cloudfunctions2_function.dataform_export.https_trigger_url
+    connection        = "${var.region}.remote-functions"
+    max_batching_rows = "1"
   }
 }
diff --git a/infra/tf/dataform_trigger/main.tf b/infra/tf/dataform_trigger/main.tf
index 001c839b..d5f73f62 100644
--- a/infra/tf/dataform_trigger/main.tf
+++ b/infra/tf/dataform_trigger/main.tf
@@ -4,7 +4,7 @@ terraform {
   required_providers {
     archive = {
       source  = "hashicorp/archive"
-      version = "2.6.0"
+      version = ">= 2.6.0"
     }
     google = {
       source  = "hashicorp/google"

From e0ef441c71bc9e4e9c0b496233aeec0d0aa69f90 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Sun, 2 Mar 2025 20:49:43 +0100
Subject: [PATCH 09/52] bq connections

---
 infra/tf/.terraform.lock.hcl          | 27 ++++++++++++++-------------
 infra/tf/dataform_export/main.tf      |  9 ++++++---
 infra/tf/dataform_export/variables.tf |  8 ++++++++
 infra/tf/functions.tf                 | 19 ++++++++++++-------
 infra/tf/main.tf                      |  2 ++
 5 files changed, 42 insertions(+), 23 deletions(-)

diff --git a/infra/tf/.terraform.lock.hcl b/infra/tf/.terraform.lock.hcl
index 1aa60309..fda13e83 100644
--- a/infra/tf/.terraform.lock.hcl
+++ b/infra/tf/.terraform.lock.hcl
@@ -2,21 +2,22 @@
 # Manual edits may be lost in future updates.
 
 provider "registry.terraform.io/hashicorp/archive" {
-  version = "2.6.0"
+  version     = "2.7.0"
+  constraints = ">= 2.6.0"
   hashes = [
-    "h1:upAbF0KeKLAs3UImwwp5veC7jRcLnpKWVjkbd4ziWhM=",
-    "zh:29273484f7423b7c5b3f5df34ccfc53e52bb5e3d7f46a81b65908e7a8fd69072",
-    "zh:3cba58ec3aea5f301caf2acc31e184c55d994cc648126cac39c63ae509a14179",
-    "zh:55170cd17dbfdea842852c6ae2416d057fec631ba49f3bb6466a7268cd39130e",
-    "zh:7197db402ba35631930c3a4814520f0ebe980ae3acb7f8b5a6f70ec90dc4a388",
+    "h1:1niS9AcwxN8CrWemnJS2Xf6vM72+48Xh3xFSS3DFWQo=",
+    "zh:04e23bebca7f665a19a032343aeecd230028a3822e546e6f618f24c47ff87f67",
+    "zh:5bb38114238e25c45bf85f5c9f627a2d0c4b98fe44a0837e37d48574385f8dad",
+    "zh:64584bc1db4c390abd81c76de438d93acf967c8a33e9b923d68da6ed749d55bd",
+    "zh:697695ab9cce351adf91a1823bdd72ce6f0d219138f5124ef7645cedf8f59a1f",
     "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
-    "zh:8bf7fe0915d7fb152a3a6b9162614d2ec82749a06dba13fab3f98d33c020ec4f",
-    "zh:8ce811844fd53adb0dabc9a541f8cb43aacfa7d8e39324e4bd3592b3428f5bfb",
-    "zh:bca795bca815b8ac90e3054c0a9ab1ccfb16eedbb3418f8ad473fc5ad6bf0ef7",
-    "zh:d9355a18df5a36cf19580748b23249de2eb445c231c36a353709f8f40a6c8432",
-    "zh:dc32cc32cfd8abf8752d34f2a783de0d3f7200c573b885ecb64ece5acea173b4",
-    "zh:ef498e20391bf7a280d0fd6fd6675621c85fbe4e92f0f517ae4394747db89bde",
-    "zh:f2bc5226c765b0c8055a7b6207d0fe1eb9484e3ec8880649d158827ac6ed3b22",
+    "zh:7edefb1d1e2fead8fd155f7b50a2cb49f2f3fed154ac3ef5f991ccaff93d6120",
+    "zh:807fb15b75910bf14795f2ad1a2d41b069f9ef52c242131b2964c8527312e235",
+    "zh:821d9148d261df1d1a8e5a4812df2a6a3ffaf0d2070dad3c785382e489069239",
+    "zh:a7d92251118fb723048c482154a6ac6368aad583d28d15fffc6f5dafd9507463",
+    "zh:b627d4cef192b3c12ddaf9cb2c4f98c10d0129883c8c2a9c0049983f9de7030d",
+    "zh:dfb70306fcc0ad1d512ab7c24765703783cc286062d4849de4fbe23526f5dc8e",
+    "zh:f21de276f857b7e51fa2593d8fef05a7faafb0a7b62db14ac58a03ce1be7d881",
   ]
 }
 
diff --git a/infra/tf/dataform_export/main.tf b/infra/tf/dataform_export/main.tf
index 00960db6..cef1240f 100644
--- a/infra/tf/dataform_export/main.tf
+++ b/infra/tf/dataform_export/main.tf
@@ -53,7 +53,10 @@ resource "google_bigquery_routine" "run_export_job" {
   routine_id = "run_export_job"
   routine_type = "SCALAR_FUNCTION"
   definition_body = ""
-  description = "Export data from Google BigQuery.\nExample payload JSON: {\"dataform_trigger\": \"tech_report_complete\", \"date\": \"${pastMonth}\", \"name\": \"adoption\", \"type\": \"report\"}"
+  description = <<EOT
+  Export data from Google BigQuery.
+  Example payload JSON: {"dataform_trigger": "tech_report_complete", "date": "2025-01-01", "name": "adoption", "type": "report"}
+  EOT
 
   arguments {
     name      = "payload"
@@ -62,8 +65,8 @@ resource "google_bigquery_routine" "run_export_job" {
   return_type = "{\"typeKind\" :  \"INT64\"}"
 
   remote_function_options {
-    endpoint          = google_cloudfunctions2_function.dataform_export.https_trigger_url
-    connection        = "${var.region}.remote-functions"
+    endpoint          = google_cloudfunctions2_function.dataform_export.service_config[0].uri
+    connection        = var.remote_functions_connection
     max_batching_rows = "1"
   }
 }
diff --git a/infra/tf/dataform_export/variables.tf b/infra/tf/dataform_export/variables.tf
index 5b2d7b0d..fcbf819c 100644
--- a/infra/tf/dataform_export/variables.tf
+++ b/infra/tf/dataform_export/variables.tf
@@ -17,3 +17,11 @@ variable "function_identity" {
 variable "function_name" {
   type = string
 }
+
+variable "location" {
+  type = string
+}
+
+variable "remote_functions_connection" {
+  type = string
+}
diff --git a/infra/tf/functions.tf b/infra/tf/functions.tf
index ed444746..60251f62 100644
--- a/infra/tf/functions.tf
+++ b/infra/tf/functions.tf
@@ -18,15 +18,20 @@ resource "google_bigquery_dataset_iam_member" "cloud_function_dataset_reader_rol
   member     = "serviceAccount:${local.function_identity}"
 }
 
-resource "google_bigquery_connection" "procedures" {
-  connection_id = "procedures"
-  location      = "US"
-  spark {
-  }
+resource "google_bigquery_connection" "spark-procedures" {
+  connection_id = "spark-procedures"
+  location      = local.location
+  spark {}
 }
 
-resource "google_project_iam_member" "bigquery-functions-connector" {
+resource "google_bigquery_connection" "remote-functions" {
+  connection_id = "remote-functions"
+  location      = local.location
+  cloud_resource {}
+}
+
+resource "google_project_iam_member" "bigquery-remote-functions-connector" {
   project = local.project
   role    = "roles/run.invoker"
-  member  = "serviceAccount:bqcx-226352634162-1s4t@gcp-sa-bigquery-condel.iam.gserviceaccount.com"
+  member  = "serviceAccount:${google_bigquery_connection.remote-functions.cloud_resource[0].service_account_id}"
 }
diff --git a/infra/tf/main.tf b/infra/tf/main.tf
index 7c010ec9..954f6da5 100644
--- a/infra/tf/main.tf
+++ b/infra/tf/main.tf
@@ -30,8 +30,10 @@ module "dataform_export" {
   project           = local.project
   project_number    = local.project_number
   region            = local.region
+  location          = local.location
   function_identity = "cloud-function@httparchive.iam.gserviceaccount.com"
   function_name     = "dataform-export"
+  remote_functions_connection  = google_bigquery_connection.remote-functions.id
 }
 
 module "dataform_trigger" {

From 363a482eafb8d77ab9c149b2dae030917f20d5d9 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Sun, 2 Mar 2025 21:16:07 +0100
Subject: [PATCH 10/52] spark procedure role

---
 Makefile              | 3 +++
 infra/tf/functions.tf | 6 ++++++
 2 files changed, 9 insertions(+)

diff --git a/Makefile b/Makefile
index b9f45b89..3d65a6d5 100644
--- a/Makefile
+++ b/Makefile
@@ -11,3 +11,6 @@ tf_plan:
 
 tf_apply:
 	terraform -chdir=infra/tf init && terraform -chdir=infra/tf apply -auto-approve
+
+bigquery_export_deploy:
+	cd infra/bigquery-export && npm run buildpack
diff --git a/infra/tf/functions.tf b/infra/tf/functions.tf
index 60251f62..a7e17d20 100644
--- a/infra/tf/functions.tf
+++ b/infra/tf/functions.tf
@@ -35,3 +35,9 @@ resource "google_project_iam_member" "bigquery-remote-functions-connector" {
   role    = "roles/run.invoker"
   member  = "serviceAccount:${google_bigquery_connection.remote-functions.cloud_resource[0].service_account_id}"
 }
+
+resource "google_project_iam_member" "spark-procedures-connector" {
+  project = local.project
+  role    = "roles/artifactregistry.reader"
+  member  = "serviceAccount:${google_bigquery_connection.spark-procedures.spark[0].service_account_id}"
+}

From 14ab9a93abf2486df801dae41a004c659d4f7336 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Sun, 2 Mar 2025 23:01:10 +0100
Subject: [PATCH 11/52] docker update

---
 Makefile                                     |  3 +++
 infra/bigquery_export_spark/Dockerfile       | 25 +++++++++++++-------
 infra/bigquery_export_spark/requirements.txt |  3 +++
 3 files changed, 23 insertions(+), 8 deletions(-)
 create mode 100644 infra/bigquery_export_spark/requirements.txt

diff --git a/Makefile b/Makefile
index 3d65a6d5..55a19a5b 100644
--- a/Makefile
+++ b/Makefile
@@ -14,3 +14,6 @@ tf_apply:
 
 bigquery_export_deploy:
 	cd infra/bigquery-export && npm run buildpack
+
+bigquery_export_spark_deploy:
+	cd infra/bigquery_export_spark && gcloud builds submit --region=global --tag us-docker.pkg.dev/httparchive/bigquery-spark-procedures/firestore_export:latest
diff --git a/infra/bigquery_export_spark/Dockerfile b/infra/bigquery_export_spark/Dockerfile
index d66edf70..2692f275 100644
--- a/infra/bigquery_export_spark/Dockerfile
+++ b/infra/bigquery_export_spark/Dockerfile
@@ -1,5 +1,7 @@
+# Dataproc image example: https://cloud.google.com/dataproc-serverless/docs/guides/custom-containers
 # Recommendation: Use Debian 12.
-FROM python:3.12-slim
+FROM debian:12-slim
+# python:3.12-slim
 
 # Suppress interactive prompts
 ENV DEBIAN_FRONTEND=noninteractive
@@ -10,13 +12,20 @@ RUN apt update && apt install -y procps tini libjemalloc2
 # Enable jemalloc2 as default memory allocator
 ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2
 
-# Install packages.
-RUN pip install --no-cache-dir \
-    google-cloud-bigquery \
-    google-cloud-storage \
-    google-cloud-firestore \
-    grpcio \
-    google-cloud-secret-manager
+# Install and configure Miniconda3.
+ENV CONDA_HOME=/opt/miniforge3
+ENV PYSPARK_PYTHON=${CONDA_HOME}/bin/python
+ENV PATH=${CONDA_HOME}/bin:${PATH}
+ADD https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh .
+RUN bash Miniforge3-Linux-x86_64.sh -b -p /opt/miniforge3 \
+  && ${CONDA_HOME}/bin/conda config --system --set always_yes True \
+  && ${CONDA_HOME}/bin/conda config --system --set auto_update_conda False \
+  && ${CONDA_HOME}/bin/conda config --system --set channel_priority strict
+
+COPY . .
+
+# Install pip packages.
+RUN ${PYSPARK_PYTHON} -m pip install -r requirements.txt
 
 # Create the 'spark' group/user.
 # The GID and UID must be 1099. Home directory is required.
diff --git a/infra/bigquery_export_spark/requirements.txt b/infra/bigquery_export_spark/requirements.txt
new file mode 100644
index 00000000..bcdb2fe6
--- /dev/null
+++ b/infra/bigquery_export_spark/requirements.txt
@@ -0,0 +1,3 @@
+google-cloud-bigquery==3.23
+google-cloud-storage==2.16
+google-cloud-firestore==2.20.1

From a48ca800623d30418e540e50a89d3309a033d81a Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Sun, 2 Mar 2025 23:35:37 +0100
Subject: [PATCH 12/52] lint

---
 .gitignore                             |  1 +
 README.md                              | 13 +++++++------
 infra/bigquery_export_spark/Dockerfile |  5 +++--
 infra/tf/bigquery_export/variables.tf  |  4 ----
 infra/tf/dataform_export/variables.tf  |  8 --------
 infra/tf/main.tf                       |  3 ---
 6 files changed, 11 insertions(+), 23 deletions(-)

diff --git a/.gitignore b/.gitignore
index 64b48092..978f945f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 node_modules/
 .DS_Store
+.venv/
 
 # Terraform
 infra/tf/.terraform/
diff --git a/README.md b/README.md
index 9f8683cc..6e579149 100644
--- a/README.md
+++ b/README.md
@@ -78,18 +78,19 @@ In order to unify the workflow triggering mechanism, we use [a Cloud Run functio
 
 1. Install dependencies:
 
-   ```bash
-   npm install
-   ```
+    ```bash
+    npm install
+    ```
 
 2. Available Scripts:
-   - `npm run format` - Format code using Standard.js, fix markdown issues, and format Terraform files
-   - `npm run lint` - Run linting checks on JavaScript, markdown files, and compile Dataform configs
+
+    - `npm run format` - Format code using Standard.js, fix Markdown issues, and format Terraform files
+    - `npm run lint` - Run linting checks on JavaScript, Markdown files, and compile Dataform configs
 
 ## Code Quality
 
 This repository uses:
 
 - Standard.js for JavaScript code style
-- Markdownlint for markdown file formatting
+- Markdownlint for Markdown file formatting
 - Dataform's built-in compiler for SQL validation
diff --git a/infra/bigquery_export_spark/Dockerfile b/infra/bigquery_export_spark/Dockerfile
index 2692f275..c2cfca75 100644
--- a/infra/bigquery_export_spark/Dockerfile
+++ b/infra/bigquery_export_spark/Dockerfile
@@ -7,7 +7,7 @@ FROM debian:12-slim
 ENV DEBIAN_FRONTEND=noninteractive
 
 # Install utilities required by Spark scripts.
-RUN apt update && apt install -y procps tini libjemalloc2
+RUN apt-get update && apt-get install -y procps tini libjemalloc2
 
 # Enable jemalloc2 as default memory allocator
 ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2
@@ -22,10 +22,11 @@ RUN bash Miniforge3-Linux-x86_64.sh -b -p /opt/miniforge3 \
   && ${CONDA_HOME}/bin/conda config --system --set auto_update_conda False \
   && ${CONDA_HOME}/bin/conda config --system --set channel_priority strict
 
+WORKDIR /app
 COPY . .
 
 # Install pip packages.
-RUN ${PYSPARK_PYTHON} -m pip install -r requirements.txt
+RUN ${PYSPARK_PYTHON} -m pip install --no-cache-dir -r app/requirements.txt
 
 # Create the 'spark' group/user.
 # The GID and UID must be 1099. Home directory is required.
diff --git a/infra/tf/bigquery_export/variables.tf b/infra/tf/bigquery_export/variables.tf
index 6bb26fac..da7dd2a5 100644
--- a/infra/tf/bigquery_export/variables.tf
+++ b/infra/tf/bigquery_export/variables.tf
@@ -2,10 +2,6 @@ variable "project" {
   type = string
 }
 
-variable "project_number" {
-  type = string
-}
-
 variable "region" {
   type = string
 }
diff --git a/infra/tf/dataform_export/variables.tf b/infra/tf/dataform_export/variables.tf
index fcbf819c..72f33c1c 100644
--- a/infra/tf/dataform_export/variables.tf
+++ b/infra/tf/dataform_export/variables.tf
@@ -1,7 +1,3 @@
-variable "project" {
-  type = string
-}
-
 variable "project_number" {
   type = string
 }
@@ -18,10 +14,6 @@ variable "function_name" {
   type = string
 }
 
-variable "location" {
-  type = string
-}
-
 variable "remote_functions_connection" {
   type = string
 }
diff --git a/infra/tf/main.tf b/infra/tf/main.tf
index 954f6da5..f999f91c 100644
--- a/infra/tf/main.tf
+++ b/infra/tf/main.tf
@@ -27,10 +27,8 @@ provider "google" {
 module "dataform_export" {
   source = "./dataform_export"
 
-  project           = local.project
   project_number    = local.project_number
   region            = local.region
-  location          = local.location
   function_identity = "cloud-function@httparchive.iam.gserviceaccount.com"
   function_name     = "dataform-export"
   remote_functions_connection  = google_bigquery_connection.remote-functions.id
@@ -50,7 +48,6 @@ module "bigquery_export" {
   source = "./bigquery_export"
 
   project           = local.project
-  project_number    = local.project_number
   region            = local.region
   location          = local.location
   function_identity = "cloud-function@httparchive.iam.gserviceaccount.com"

From bd1cf17c2eed0aeaf86ca24616e95adab7efd7a6 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Mon, 3 Mar 2025 00:04:29 +0100
Subject: [PATCH 13/52] lint

---
 infra/bigquery_export_spark/Dockerfile       |   8 +-
 infra/bigquery_export_spark/requirements.txt |   1 +
 infra/bigquery_export_spark/src/firestore.py | 114 +++++++++++++++++++
 infra/tf/dataform_export/main.tf             |   8 +-
 infra/tf/main.tf                             |  10 +-
 package.json                                 |   2 +-
 6 files changed, 130 insertions(+), 13 deletions(-)
 create mode 100644 infra/bigquery_export_spark/src/firestore.py

diff --git a/infra/bigquery_export_spark/Dockerfile b/infra/bigquery_export_spark/Dockerfile
index c2cfca75..49b16fd0 100644
--- a/infra/bigquery_export_spark/Dockerfile
+++ b/infra/bigquery_export_spark/Dockerfile
@@ -7,7 +7,9 @@ FROM debian:12-slim
 ENV DEBIAN_FRONTEND=noninteractive
 
 # Install utilities required by Spark scripts.
-RUN apt-get update && apt-get install -y procps tini libjemalloc2
+RUN apt-get update && apt-get install -y procps=\* tini=\* libjemalloc2=\* \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
 
 # Enable jemalloc2 as default memory allocator
 ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2
@@ -23,10 +25,10 @@ RUN bash Miniforge3-Linux-x86_64.sh -b -p /opt/miniforge3 \
   && ${CONDA_HOME}/bin/conda config --system --set channel_priority strict
 
 WORKDIR /app
-COPY . .
+COPY requirements.txt .
 
 # Install pip packages.
-RUN ${PYSPARK_PYTHON} -m pip install --no-cache-dir -r app/requirements.txt
+RUN ${PYSPARK_PYTHON} -m pip install --no-cache-dir -r requirements.txt
 
 # Create the 'spark' group/user.
 # The GID and UID must be 1099. Home directory is required.
diff --git a/infra/bigquery_export_spark/requirements.txt b/infra/bigquery_export_spark/requirements.txt
index bcdb2fe6..2493daed 100644
--- a/infra/bigquery_export_spark/requirements.txt
+++ b/infra/bigquery_export_spark/requirements.txt
@@ -1,3 +1,4 @@
 google-cloud-bigquery==3.23
 google-cloud-storage==2.16
 google-cloud-firestore==2.20.1
+# pyspark==3.5.5
diff --git a/infra/bigquery_export_spark/src/firestore.py b/infra/bigquery_export_spark/src/firestore.py
new file mode 100644
index 00000000..d5bebf7f
--- /dev/null
+++ b/infra/bigquery_export_spark/src/firestore.py
@@ -0,0 +1,114 @@
+from pyspark.sql import SparkSession
+from google.cloud import firestore
+from google.cloud import bigquery
+import json
+import os
+
+class FirestoreBatch:
+    def __init__(self):
+        self.firestore = firestore.Client()
+        self.bigquery = bigquery.Client()
+        self.batch_size = 500
+        self.max_concurrent_batches = 200
+        self.current_batch = []
+        self.batch_promises = []
+        self.spark = SparkSession.builder.appName("FirestoreBatchProcessor").getOrCreate()
+
+    def queue_batch(self, operation):
+        batch = self.firestore.batch()
+
+        for doc in self.current_batch:
+            if operation == "delete":
+                batch.delete(doc.reference)
+            elif operation == "set":
+                doc_ref = self.firestore.collection(self.collection_name).document()
+                batch.set(doc_ref, doc)
+            else:
+                raise ValueError("Invalid operation")
+
+        self.batch_promises.append(batch.commit())
+        self.current_batch = []
+
+    def commit_batches(self):
+        print(f"Committing {len(self.batch_promises)} batches to {self.collection_name}")
+        for batch_promise in self.batch_promises:
+            try:
+                batch_promise
+            except Exception as e:
+                print(f"Error committing batch: {e}")
+                raise
+        self.batch_promises = []
+
+    def final_flush(self, operation):
+        if self.current_batch:
+            self.queue_batch(operation)
+        if self.batch_promises:
+            self.commit_batches()
+
+    def batch_delete(self):
+        print("Starting batch deletion...")
+        start_time = self.spark.sparkContext.startTime
+        self.current_batch = []
+        self.batch_promises = []
+        total_docs_deleted = 0
+
+        collection_ref = self.firestore.collection(self.collection_name)
+        if self.collection_type == "report":
+            print(f"Deleting documents from {self.collection_name} for date {self.date}")
+            query = collection_ref.where("date", "==", self.date)
+        elif self.collection_type == "dict":
+            print(f"Deleting documents from {self.collection_name}")
+            query = collection_ref
+        else:
+            raise ValueError("Invalid collection type")
+
+        while True:
+            docs = list(query.limit(self.batch_size * self.max_concurrent_batches).stream())
+            if not docs:
+                break
+
+            for doc in docs:
+                self.current_batch.append(doc)
+                if len(self.current_batch) >= self.batch_size:
+                    self.queue_batch("delete")
+                if len(self.batch_promises) >= self.max_concurrent_batches:
+                    self.commit_batches()
+                total_docs_deleted += 1
+
+        self.final_flush("delete")
+        duration = (self.spark.sparkContext.startTime - start_time) / 1000
+        print(f"Deletion complete. Total docs deleted: {total_docs_deleted}. Time: {duration} seconds")
+
+    def stream_from_bigquery(self, query):
+        print("Starting BigQuery to Firestore transfer...")
+        start_time = self.spark.sparkContext.startTime
+        total_rows_processed = 0
+
+        df = self.spark.read.format("bigquery").option("query", query).load()
+
+        for row in df.collect():
+            self.current_batch.append(row.asDict())
+            if len(self.current_batch) >= self.batch_size:
+                self.queue_batch("set")
+            if len(self.batch_promises) >= self.max_concurrent_batches:
+                self.commit_batches()
+            total_rows_processed += 1
+
+        self.final_flush("set")
+        duration = (self.spark.sparkContext.startTime - start_time) / 1000
+        print(f"Transfer to {self.collection_name} complete. Total rows processed: {total_rows_processed}. Time: {duration} seconds")
+
+    def export(self):
+        export_config = json.loads('{"name": "technologies", "type": "dict", "environment": "dev"}')
+        query = str(json.loads("SELECT * FROM report.tech_report_technologies"))
+
+        self.date = getattr(export_config, "date", "")
+        self.collection_name = export_config["name"]
+        self.collection_type = export_config["type"]
+
+        self.batch_delete()
+        self.stream_from_bigquery(query)
+
+if __name__ == "__main__":
+    processor = FirestoreBatch()
+    processor.export()
diff --git a/infra/tf/dataform_export/main.tf b/infra/tf/dataform_export/main.tf
index cef1240f..7029c817 100644
--- a/infra/tf/dataform_export/main.tf
+++ b/infra/tf/dataform_export/main.tf
@@ -49,11 +49,11 @@ resource "google_cloudfunctions2_function" "dataform_export" {
 }
 
 resource "google_bigquery_routine" "run_export_job" {
-  dataset_id = "reports"
-  routine_id = "run_export_job"
-  routine_type = "SCALAR_FUNCTION"
+  dataset_id      = "reports"
+  routine_id      = "run_export_job"
+  routine_type    = "SCALAR_FUNCTION"
   definition_body = ""
-  description = <<EOT
+  description     = <<EOT
   Export data from Google BigQuery.
   Example payload JSON: {"dataform_trigger": "tech_report_complete", "date": "2025-01-01", "name": "adoption", "type": "report"}
   EOT
diff --git a/infra/tf/main.tf b/infra/tf/main.tf
index f999f91c..0bf4c1a9 100644
--- a/infra/tf/main.tf
+++ b/infra/tf/main.tf
@@ -27,11 +27,11 @@ provider "google" {
 module "dataform_export" {
   source = "./dataform_export"
 
-  project_number    = local.project_number
-  region            = local.region
-  function_identity = "cloud-function@httparchive.iam.gserviceaccount.com"
-  function_name     = "dataform-export"
-  remote_functions_connection  = google_bigquery_connection.remote-functions.id
+  project_number              = local.project_number
+  region                      = local.region
+  function_identity           = "cloud-function@httparchive.iam.gserviceaccount.com"
+  function_name               = "dataform-export"
+  remote_functions_connection = google_bigquery_connection.remote-functions.id
 }
 
 module "dataform_trigger" {
diff --git a/package.json b/package.json
index b4528bec..b7b35403 100644
--- a/package.json
+++ b/package.json
@@ -5,7 +5,7 @@
     "@dataform/core": "3.0.14"
   },
   "scripts": {
-    "format": "npx standard --fix; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint . --fix; terraform -chdir=infra/tf fmt",
+    "format": "npx standard --fix; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint . --fix; terraform -chdir=infra/tf fmt -recursive",
     "lint": "npx standard; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint .; dataform compile"
   },
   "standard": {

From d58054d5a82860dbea48205aabf2a35bc9831dcb Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Mon, 3 Mar 2025 00:15:40 +0100
Subject: [PATCH 14/52] lint

---
 infra/bigquery_export_spark/src/firestore.py | 81 +++++++++++++++-----
 1 file changed, 61 insertions(+), 20 deletions(-)

diff --git a/infra/bigquery_export_spark/src/firestore.py b/infra/bigquery_export_spark/src/firestore.py
index d5bebf7f..0fc37cce 100644
--- a/infra/bigquery_export_spark/src/firestore.py
+++ b/infra/bigquery_export_spark/src/firestore.py
@@ -1,27 +1,39 @@
-from pyspark.sql import SparkSession
-from google.cloud import firestore
-from google.cloud import bigquery
+"""This module processes Firestore documents from BigQuery using Spark."""
+
 import json
-import os
+from google.cloud import bigquery, firestore  # type: ignore
+from pyspark.sql import SparkSession  # type: ignore
+
 
 class FirestoreBatch:
+    """Handles Firestore data batching from BigQuery using Spark."""
+
     def __init__(self):
+        """Initialize FirestoreBatch with default settings."""
         self.firestore = firestore.Client()
         self.bigquery = bigquery.Client()
         self.batch_size = 500
         self.max_concurrent_batches = 200
         self.current_batch = []
         self.batch_promises = []
-        self.spark = SparkSession.builder.appName("FirestoreBatchProcessor").getOrCreate()
+        self.spark = SparkSession.builder.appName(
+            "FirestoreBatchProcessor"
+        ).getOrCreate()
+        self.config = {
+            "date": "",
+            "collection_name": "",
+            "collection_type": ""
+        }
 
     def queue_batch(self, operation):
+        """Queue a batch commit operation for Firestore."""
         batch = self.firestore.batch()
 
         for doc in self.current_batch:
             if operation == "delete":
                 batch.delete(doc.reference)
             elif operation == "set":
-                doc_ref = self.firestore.collection(self.collection_name).document()
+                doc_ref = self.firestore.collection(self.config["collection_name"]).document()
                 batch.set(doc_ref, doc)
             else:
                 raise ValueError("Invalid operation")
@@ -30,7 +42,11 @@ def queue_batch(self, operation):
         self.current_batch = []
 
     def commit_batches(self):
-        print(f"Committing {len(self.batch_promises)} batches to {self.collection_name}")
+        """Commit all queued batch promises."""
+        print(
+            f"Committing {len(self.batch_promises)} "
+            f"batches to {self.config['collection_name']}"
+        )
         for batch_promise in self.batch_promises:
             try:
                 batch_promise
@@ -40,30 +56,39 @@ def commit_batches(self):
         self.batch_promises = []
 
     def final_flush(self, operation):
+        """Flush any pending batch operations."""
         if self.current_batch:
             self.queue_batch(operation)
         if self.batch_promises:
             self.commit_batches()
 
     def batch_delete(self):
+        """Delete Firestore documents in batches."""
         print("Starting batch deletion...")
         start_time = self.spark.sparkContext.startTime
         self.current_batch = []
         self.batch_promises = []
         total_docs_deleted = 0
 
-        collection_ref = self.firestore.collection(self.collection_name)
-        if self.collection_type == "report":
-            print(f"Deleting documents from {self.collection_name} for date {self.date}")
-            query = collection_ref.where("date", "==", self.date)
-        elif self.collection_type == "dict":
-            print(f"Deleting documents from {self.collection_name}")
+        collection_ref = self.firestore.collection(self.config["collection_name"])
+        if self.config["collection_type"] == "report":
+            print(
+                f"Deleting documents from {self.config['collection_name']} "
+                f"for date {self.config['date']}"
+            )
+            query = collection_ref.where("date", "==", self.config["date"])
+        elif self.config["collection_type"] == "dict":
+            print(
+                f"Deleting documents from {self.config['collection_name']}"
+            )
             query = collection_ref
         else:
             raise ValueError("Invalid collection type")
 
         while True:
-            docs = list(query.limit(self.batch_size * self.max_concurrent_batches).stream())
+            docs = list(
+                query.limit(self.batch_size * self.max_concurrent_batches).stream()
+            )
             if not docs:
                 break
 
@@ -77,9 +102,14 @@ def batch_delete(self):
 
         self.final_flush("delete")
         duration = (self.spark.sparkContext.startTime - start_time) / 1000
-        print(f"Deletion complete. Total docs deleted: {total_docs_deleted}. Time: {duration} seconds")
+        print(
+            f"Deletion complete. "
+            f"Total docs deleted: {total_docs_deleted}. "
+            f"Time: {duration} seconds"
+        )
 
     def stream_from_bigquery(self, query):
+        """Stream data from BigQuery to Firestore."""
         print("Starting BigQuery to Firestore transfer...")
         start_time = self.spark.sparkContext.startTime
         total_rows_processed = 0
@@ -96,19 +126,30 @@ def stream_from_bigquery(self, query):
 
         self.final_flush("set")
         duration = (self.spark.sparkContext.startTime - start_time) / 1000
-        print(f"Transfer to {self.collection_name} complete. Total rows processed: {total_rows_processed}. Time: {duration} seconds")
+        print(
+            f"Transfer to {self.config['collection_name']} "
+            f"complete. "
+            f"Total rows processed: "
+            f"{total_rows_processed}. "
+            f"Time: {duration} "
+            f"seconds"
+        )
 
     def export(self):
-        export_config = json.loads('{"name": "technologies", "type": "dict", "environment": "dev"}')
+        """Export data from BigQuery to Firestore."""
+        export_config = json.loads(
+            '{"name": "technologies", "type": "dict", "environment": "dev"}'
+        )
         query = str(json.loads("SELECT * FROM report.tech_report_technologies"))
 
-        self.date = getattr(export_config, "date", "")
-        self.collection_name = export_config["name"]
-        self.collection_type = export_config["type"]
+        self.config["date"] = getattr(export_config, "date", "")
+        self.config["collection_name"] = export_config["name"]
+        self.config["collection_type"] = export_config["type"]
 
         self.batch_delete()
         self.stream_from_bigquery(query)
 
+
 if __name__ == "__main__":
     processor = FirestoreBatch()
     processor.export()

From 55fe58b3b0359adebf98bc115086f3a831a4dd54 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Mon, 3 Mar 2025 00:33:20 +0100
Subject: [PATCH 15/52] lint

---
 infra/bigquery_export_spark/src/firestore.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/infra/bigquery_export_spark/src/firestore.py b/infra/bigquery_export_spark/src/firestore.py
index 0fc37cce..a8d6e0a3 100644
--- a/infra/bigquery_export_spark/src/firestore.py
+++ b/infra/bigquery_export_spark/src/firestore.py
@@ -1,6 +1,7 @@
 """This module processes Firestore documents from BigQuery using Spark."""
 
 import json
+
 from google.cloud import bigquery, firestore  # type: ignore
 from pyspark.sql import SparkSession  # type: ignore
 
@@ -19,11 +20,7 @@ def __init__(self):
         self.spark = SparkSession.builder.appName(
             "FirestoreBatchProcessor"
         ).getOrCreate()
-        self.config = {
-            "date": "",
-            "collection_name": "",
-            "collection_type": ""
-        }
+        self.config = {"date": "", "collection_name": "", "collection_type": ""}
 
     def queue_batch(self, operation):
         """Queue a batch commit operation for Firestore."""
@@ -33,7 +30,9 @@ def queue_batch(self, operation):
             if operation == "delete":
                 batch.delete(doc.reference)
             elif operation == "set":
-                doc_ref = self.firestore.collection(self.config["collection_name"]).document()
+                doc_ref = self.firestore.collection(
+                    self.config["collection_name"]
+                ).document()
                 batch.set(doc_ref, doc)
             else:
                 raise ValueError("Invalid operation")
@@ -78,9 +77,7 @@ def batch_delete(self):
             )
             query = collection_ref.where("date", "==", self.config["date"])
         elif self.config["collection_type"] == "dict":
-            print(
-                f"Deleting documents from {self.config['collection_name']}"
-            )
+            print(f"Deleting documents from {self.config['collection_name']}")
             query = collection_ref
         else:
             raise ValueError("Invalid collection type")

From fc0d49671d9516310b29584543ddfa6945b365f7 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Mon, 3 Mar 2025 13:56:33 +0100
Subject: [PATCH 16/52] more spark roles

---
 infra/bigquery_export_spark/requirements.txt |  2 -
 infra/bigquery_export_spark/src/firestore.py | 56 +++++++++++---------
 infra/tf/functions.tf                        |  6 ++-
 3 files changed, 36 insertions(+), 28 deletions(-)

diff --git a/infra/bigquery_export_spark/requirements.txt b/infra/bigquery_export_spark/requirements.txt
index 2493daed..be4c0b9b 100644
--- a/infra/bigquery_export_spark/requirements.txt
+++ b/infra/bigquery_export_spark/requirements.txt
@@ -1,4 +1,2 @@
-google-cloud-bigquery==3.23
-google-cloud-storage==2.16
 google-cloud-firestore==2.20.1
 # pyspark==3.5.5
diff --git a/infra/bigquery_export_spark/src/firestore.py b/infra/bigquery_export_spark/src/firestore.py
index a8d6e0a3..366c4106 100644
--- a/infra/bigquery_export_spark/src/firestore.py
+++ b/infra/bigquery_export_spark/src/firestore.py
@@ -1,18 +1,29 @@
 """This module processes Firestore documents from BigQuery using Spark."""
 
 import json
+import os
 
-from google.cloud import bigquery, firestore  # type: ignore
+from google.cloud import firestore  # type: ignore
 from pyspark.sql import SparkSession  # type: ignore
 
 
+PROJECT = "httparchive"
+
+
+# pylint: disable=too-many-instance-attributes
 class FirestoreBatch:
     """Handles Firestore data batching from BigQuery using Spark."""
 
-    def __init__(self):
+    def __init__(self, export_config):
         """Initialize FirestoreBatch with default settings."""
-        self.firestore = firestore.Client()
-        self.bigquery = bigquery.Client()
+        self.config = {
+            "collection_name": export_config["name"],
+            "date": getattr(export_config, "date", ""),
+            "collection_type": export_config["type"],
+        }
+        self.firestore = firestore.Client(
+            project=PROJECT, database=export_config["database"]
+        )
         self.batch_size = 500
         self.max_concurrent_batches = 200
         self.current_batch = []
@@ -20,7 +31,6 @@ def __init__(self):
         self.spark = SparkSession.builder.appName(
             "FirestoreBatchProcessor"
         ).getOrCreate()
-        self.config = {"date": "", "collection_name": "", "collection_type": ""}
 
     def queue_batch(self, operation):
         """Queue a batch commit operation for Firestore."""
@@ -36,7 +46,6 @@ def queue_batch(self, operation):
                 batch.set(doc_ref, doc)
             else:
                 raise ValueError("Invalid operation")
-
         self.batch_promises.append(batch.commit())
         self.current_batch = []
 
@@ -75,16 +84,17 @@ def batch_delete(self):
                 f"Deleting documents from {self.config['collection_name']} "
                 f"for date {self.config['date']}"
             )
-            query = collection_ref.where("date", "==", self.config["date"])
+            collection_query = collection_ref.where("date", "==", self.config["date"])
         elif self.config["collection_type"] == "dict":
             print(f"Deleting documents from {self.config['collection_name']}")
-            query = collection_ref
+            collection_query = collection_ref
         else:
             raise ValueError("Invalid collection type")
-
         while True:
             docs = list(
-                query.limit(self.batch_size * self.max_concurrent_batches).stream()
+                collection_query.limit(
+                    self.batch_size * self.max_concurrent_batches
+                ).stream()
             )
             if not docs:
                 break
@@ -105,13 +115,13 @@ def batch_delete(self):
             f"Time: {duration} seconds"
         )
 
-    def stream_from_bigquery(self, query):
+    def stream_from_bigquery(self, query_str):
         """Stream data from BigQuery to Firestore."""
         print("Starting BigQuery to Firestore transfer...")
         start_time = self.spark.sparkContext.startTime
         total_rows_processed = 0
 
-        df = self.spark.read.format("bigquery").option("query", query).load()
+        df = self.spark.read.format("bigquery").option("query", query_str).load()
 
         for row in df.collect():
             self.current_batch.append(row.asDict())
@@ -132,21 +142,19 @@ def stream_from_bigquery(self, query):
             f"seconds"
         )
 
-    def export(self):
+    def export(self, query_str):
         """Export data from BigQuery to Firestore."""
-        export_config = json.loads(
-            '{"name": "technologies", "type": "dict", "environment": "dev"}'
-        )
-        query = str(json.loads("SELECT * FROM report.tech_report_technologies"))
-
-        self.config["date"] = getattr(export_config, "date", "")
-        self.config["collection_name"] = export_config["name"]
-        self.config["collection_type"] = export_config["type"]
 
         self.batch_delete()
-        self.stream_from_bigquery(query)
+        self.stream_from_bigquery(query_str)
 
 
 if __name__ == "__main__":
-    processor = FirestoreBatch()
-    processor.export()
+    # config_data = json.loads('{"name": "technologies", "type": "dict", "environment": "dev"}')
+    # QUERY_STR = str(json.loads("SELECT * FROM report.tech_report_technologies"))
+
+    config_data = json.loads(os.environ["BIGQUERY_PROC_PARAM.export_config"])
+    QUERY_STR = str(json.loads(os.environ["BIGQUERY_PROC_PARAM.query"]))
+
+    processor = FirestoreBatch(config_data)
+    processor.export(QUERY_STR)
diff --git a/infra/tf/functions.tf b/infra/tf/functions.tf
index a7e17d20..41226f8d 100644
--- a/infra/tf/functions.tf
+++ b/infra/tf/functions.tf
@@ -3,7 +3,7 @@ locals {
 }
 
 resource "google_project_iam_member" "project" {
-  for_each = toset(["roles/bigquery.jobUser", "roles/dataform.serviceAgent", "roles/run.invoker", "roles/run.jobsExecutorWithOverrides"])
+  for_each = toset(["roles/bigquery.user", "roles/dataform.serviceAgent", "roles/run.invoker", "roles/run.jobsExecutorWithOverrides", "roles/datastore.user", "roles/storage.objectAdmin"])
 
   project = local.project
   role    = each.value
@@ -37,7 +37,9 @@ resource "google_project_iam_member" "bigquery-remote-functions-connector" {
 }
 
 resource "google_project_iam_member" "spark-procedures-connector" {
+  for_each = toset(["roles/datastore.user", "roles/artifactregistry.reader", "roles/bigquery.user"])
+
   project = local.project
-  role    = "roles/artifactregistry.reader"
+  role    = each.value
   member  = "serviceAccount:${google_bigquery_connection.spark-procedures.spark[0].service_account_id}"
 }

From 6ee4cfd3a97baa84e6737d9b6f9e250073c3f197 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Wed, 12 Mar 2025 22:10:26 +0100
Subject: [PATCH 17/52] mh submodule

---
 .gitmodules                  |  3 +++
 infra/tf/.terraform.lock.hcl | 52 ++++++++++++++++++------------------
 infra/tf/main.tf             |  3 +++
 3 files changed, 32 insertions(+), 26 deletions(-)
 create mode 100644 .gitmodules

diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 00000000..d836ba8d
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "infra/tf/masthead-deployment"]
+	path = infra/tf/masthead-deployment
+	url = https://github.com/Masthead-Data/masthead-deployment.git
diff --git a/infra/tf/.terraform.lock.hcl b/infra/tf/.terraform.lock.hcl
index fda13e83..653ae725 100644
--- a/infra/tf/.terraform.lock.hcl
+++ b/infra/tf/.terraform.lock.hcl
@@ -22,41 +22,41 @@ provider "registry.terraform.io/hashicorp/archive" {
 }
 
 provider "registry.terraform.io/hashicorp/google" {
-  version     = "6.23.0"
+  version     = "6.25.0"
   constraints = ">= 6.13.0"
   hashes = [
-    "h1:Gr39ABNw+A6lwP2gPG+yCzGmU5T97iI5qT0XLCd3Dh4=",
-    "zh:032dd78eff887a673a1067008a8e47a69983bbcea9f41832320470247a76863a",
-    "zh:1af89f75142cf9c54499a466c8dc7055e2bbf02771a6b8c8cd57eb13dce9a800",
-    "zh:3696a8e72c6cef80fec3c3574fc8519f0410f23f6dc3e3540d2f03345c140d38",
-    "zh:58a15c71ae128ff64117c1c6b9ccf8ab2ac3e8f9c2c52957d8327f93495f62b1",
-    "zh:70ba2909611e8d1cc8009567e50e195c4269e6582d6a6fa0bce0d4e6313ab8d5",
-    "zh:8f8489d1eb8c189d59dc85e519e51ab4c4b1940e4d72450ae130ba752028fa01",
-    "zh:99c8c4e8dc67a7ab597d46ed566f64c4409761276f34bd863457a37254620fa6",
-    "zh:9b24d53440e8d7e06020e7b3aeca0dde4f1a3ef997d7da05ff3acb918896fcc2",
-    "zh:b3667fd6057997dbf0bd0179ddf686c272d4ab4fc7da6a03fdf2fad31ad4ecb5",
-    "zh:cc6df6d2291a337a5f434b7335c693164c6987604b9690d6b953b796d8eaa08a",
-    "zh:d871f39c3c5b63995793c9a70f107e52ca699d211c770a6e7ebc398ba59bcdc7",
+    "h1:rvsmJc5J5OMMFXj9th8Yqfwm7cLZdRM/stPo0B8Datg=",
+    "zh:115ce3e84a02412a9a42c7c8f1c4568ab470bdd93c9a3a1f9202d4b77d7bc236",
+    "zh:623a32acea92d98a8bb66481fa9489e0a1e4dd6fc57ab70dbb53fae5732c1c63",
+    "zh:6440fc959b5e316152e26c916d55566311dafbe5b64e45d4b9c9931a5b29fa13",
+    "zh:91edb056638e723b1c7802d0e6e293611208e2825f645572ddba98122723f11c",
+    "zh:9b57ee44172677d2c2df03b22cf4b40e184ac8bd8facdd456ccbfdb7fee4684a",
+    "zh:9e2a97dd09b78b36caecdd990fff80bdb47a6a6d45c8b2dd4d23885d6bc89e65",
+    "zh:b33e40e9ba745f15b1c2e9ebcdccbac185788a4eb57450820cec7b9585858522",
+    "zh:d9e051bb703597384d83d924c49770e3bcdc1b68583d3def33a607ab168c634b",
+    "zh:e84253140fc3b0bd5cf7a1ebb54f993bacc6d5ee33c7b5e714ad834d5b2d35d0",
+    "zh:ebb624504c6f4297e691b6e3c00f789a6729461e5aa80fc165ef2ecd878e2d87",
     "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+    "zh:f8c3e8192e98eff09a9453b52424c0d9727ae284de4fafbc498f2d527e026850",
   ]
 }
 
 provider "registry.terraform.io/hashicorp/google-beta" {
-  version     = "6.23.0"
+  version     = "6.24.0"
   constraints = ">= 6.13.0"
   hashes = [
-    "h1:i/TKWST1xe697zDOVDeHcXpjZ4TBxxJK5X7+DS4ozk4=",
-    "zh:0801ceda0de3f61745d379aef0bfe6c3a918c597a097f91e64791fb77d44dd68",
-    "zh:3b8b48de49892f25af14ce0e9abde055a3b008011dd9f7907706067348c32126",
-    "zh:74a20663b00e0eb8bcb0305c3180e7061dd5f0b1d77f36768c3f58f7a773672f",
-    "zh:763e2f0e69a03a4c85e9472b2c5af13e683519b92e3c4bad8fa4ecd650c2b256",
-    "zh:99e9379b1511fd75917a5bab01be0fbec8ac45f4a2b811571ca8cec7326f5f6d",
-    "zh:b28199d64772d7536704554b0fe66954e6c7cef1e15d4a488252c8da8a6b3648",
-    "zh:ce467d73661458e88163a9e33e2142480bf01aaf3477a87a45bbb9227b65c2ea",
-    "zh:d8fb5ee8d07bcaf717f09cebe27fe247a1c9a6a253476bba64a3048f094c377b",
-    "zh:dbdb5dc2f18b1b48b501df2f3a4f201c8646b0c765d01b9fe69116e81d71ce53",
-    "zh:e3e5f2a44734bc23c4fac7b8e4f2b368f9bbde68295129ec3290fa135d1de224",
-    "zh:eba860faa6b49d917460eb8f3594bda0be8b8ca3f5f17465096148717d5f600e",
+    "h1:rSlf/p6mrMGjIy7yt7H0rFS+W1kLe/JqIPH8nJdLvl0=",
+    "zh:0daad0f1b3e030d6931a1d252d6625851a96927ce3cae86779c213f686f4f848",
+    "zh:24ae128cfbd8b2b6c3a10a3c8719719cafee9f449e4a737404dabc0a2b63845a",
+    "zh:43343015e1fe0653a855a8980b85b4c03e815903a4cde6c615c105c66ef7331d",
+    "zh:5eb97b570b4f9ab59ad3b7700e2a5019805ceb09e7ec557701c849c5c4a2a903",
+    "zh:63cbf916a40c8ac20c2efe80496a97c45d18f748c676faedd235d9d09b8c3616",
+    "zh:74063fba0ae1a951ddd9fef53158e08144901fbe7798d385beb08b5aae847ac4",
+    "zh:7716d4a3c1f7b338f231b9dc856dc83c000f0b44f860d78bda215fc48783d72a",
+    "zh:77709c91c0cc8bee738c59696063e1b1aeb9f672db9ca9b456e3a0bdbc4695c3",
+    "zh:96c337ecc8869c5a67d5a54bea9563e283dab0b4050eec829c3e3b5075399153",
+    "zh:98c5324f0a5b0da29b106d044373b1bc6185de9e8c7481e25c0d2c2a36eb1217",
+    "zh:cb21d979314ed318b20952f3297f53ba15afbe3cdaf430c4f9bbdaced81c827c",
     "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
   ]
 }
diff --git a/infra/tf/main.tf b/infra/tf/main.tf
index 0bf4c1a9..77ee684d 100644
--- a/infra/tf/main.tf
+++ b/infra/tf/main.tf
@@ -58,4 +58,7 @@ module "masthead" {
   source = "./masthead"
 
   project = local.project
+  # source = "./masthead-deployment/app"
+  # project_id = local.project
+  # project_number = local.project_number
 }

From 5baab97e0a7f42df3c09526bd9774461ae164db2 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Thu, 13 Mar 2025 02:52:38 +0100
Subject: [PATCH 18/52] submodule

---
 infra/tf/masthead-deployment | 1 +
 1 file changed, 1 insertion(+)
 create mode 160000 infra/tf/masthead-deployment

diff --git a/infra/tf/masthead-deployment b/infra/tf/masthead-deployment
new file mode 160000
index 00000000..92b5b6db
--- /dev/null
+++ b/infra/tf/masthead-deployment
@@ -0,0 +1 @@
+Subproject commit 92b5b6db174e0b7939efcc14f17a7f421a4e81d5

From f2047d3c27c9c16ee1400d3325d2d8fc471f8472 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Thu, 13 Mar 2025 03:34:01 +0100
Subject: [PATCH 19/52] lint

---
 .gitmodules                                  | 4 ++--
 infra/bigquery_export_spark/src/firestore.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.gitmodules b/.gitmodules
index d836ba8d..cdd9adbf 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,3 +1,3 @@
 [submodule "infra/tf/masthead-deployment"]
-	path = infra/tf/masthead-deployment
-	url = https://github.com/Masthead-Data/masthead-deployment.git
+    path = infra/tf/masthead-deployment
+    url = https://github.com/Masthead-Data/masthead-deployment.git
diff --git a/infra/bigquery_export_spark/src/firestore.py b/infra/bigquery_export_spark/src/firestore.py
index 366c4106..430b3af0 100644
--- a/infra/bigquery_export_spark/src/firestore.py
+++ b/infra/bigquery_export_spark/src/firestore.py
@@ -6,8 +6,8 @@
 from google.cloud import firestore  # type: ignore
 from pyspark.sql import SparkSession  # type: ignore
 
-
 PROJECT = "httparchive"
+DATABASE = "tech-report-api"
 
 
 # pylint: disable=too-many-instance-attributes
@@ -22,7 +22,7 @@ def __init__(self, export_config):
             "collection_type": export_config["type"],
         }
         self.firestore = firestore.Client(
-            project=PROJECT, database=export_config["database"]
+            project=PROJECT, database=DATABASE
         )
         self.batch_size = 500
         self.max_concurrent_batches = 200

From a1fc9bf3019349085b2be74d02865ed4855cc9b0 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Thu, 13 Mar 2025 03:37:44 +0100
Subject: [PATCH 20/52] lint

---
 infra/bigquery_export_spark/src/firestore.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/infra/bigquery_export_spark/src/firestore.py b/infra/bigquery_export_spark/src/firestore.py
index 430b3af0..0e6d1e0c 100644
--- a/infra/bigquery_export_spark/src/firestore.py
+++ b/infra/bigquery_export_spark/src/firestore.py
@@ -21,9 +21,7 @@ def __init__(self, export_config):
             "date": getattr(export_config, "date", ""),
             "collection_type": export_config["type"],
         }
-        self.firestore = firestore.Client(
-            project=PROJECT, database=DATABASE
-        )
+        self.firestore = firestore.Client(project=PROJECT, database=DATABASE)
         self.batch_size = 500
         self.max_concurrent_batches = 200
         self.current_batch = []

From f2e26000b953f4f59fb91f30fac7b8a0dd7833c8 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Thu, 13 Mar 2025 19:49:08 +0100
Subject: [PATCH 21/52] use connections from dataform

---
 infra/tf/dataform.tf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/infra/tf/dataform.tf b/infra/tf/dataform.tf
index d514a6ff..9cdaca01 100644
--- a/infra/tf/dataform.tf
+++ b/infra/tf/dataform.tf
@@ -26,6 +26,7 @@ locals {
 
   dataform_service_account_roles = [
     "roles/bigquery.jobUser",
+    "roles/bigquery.connectionUser",
     "roles/dataform.serviceAgent",
   ]
 }

From a95f907bc61bbfb14fa2b6b7c2d4bc30216cefaa Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Thu, 13 Mar 2025 19:49:38 +0100
Subject: [PATCH 22/52] sync with latest version

---
 .../output/reports/cwv_tech_technologies.js   | 47 ++++++++++++-------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/definitions/output/reports/cwv_tech_technologies.js b/definitions/output/reports/cwv_tech_technologies.js
index fc586cde..07f184c8 100644
--- a/definitions/output/reports/cwv_tech_technologies.js
+++ b/definitions/output/reports/cwv_tech_technologies.js
@@ -11,8 +11,8 @@ WITH pages AS (
     client,
     root_page,
     tech.technology
-  FROM ${ctx.ref('crawl', 'pages')},
-    UNNEST(technologies) AS tech
+  FROM ${ctx.ref('crawl', 'pages')} AS pages
+  INNER JOIN pages.technologies AS tech
   WHERE
     date = '${pastMonth}'
     ${constants.devRankFilter}
@@ -20,28 +20,38 @@ WITH pages AS (
 
 tech_origins AS (
   SELECT
-    client,
     technology,
-    COUNT(DISTINCT root_page) AS origins
-  FROM pages
-  GROUP BY
-    client,
-    technology
+    STRUCT(
+      MAX(IF(client = 'desktop', origins, 0)) AS desktop,
+      MAX(IF(client = 'mobile', origins, 0)) AS mobile
+    ) AS origins
+  FROM (
+    SELECT
+      client,
+      technology,
+      COUNT(DISTINCT root_page) AS origins
+    FROM pages
+    GROUP BY
+      client,
+      technology
+  )
+  GROUP BY technology
 ),
 
 technologies AS (
   SELECT
     name AS technology,
     description,
+    icon,
     STRING_AGG(DISTINCT category, ', ' ORDER BY category ASC) AS category,
-    categories AS category_obj,
-    NULL AS similar_technologies
-  FROM ${ctx.ref('wappalyzer', 'technologies')},
-    UNNEST(categories) AS category
+    categories AS category_obj
+  FROM ${ctx.ref('wappalyzer', 'technologies')} AS technologies
+  INNER JOIN technologies.categories AS category
   GROUP BY
     technology,
     description,
-    categories
+    categories,
+    icon
 ),
 
 total_pages AS (
@@ -53,12 +63,11 @@ total_pages AS (
 )
 
 SELECT
-  client,
   technology,
   description,
+  icon,
   category,
   category_obj,
-  similar_technologies,
   origins
 FROM tech_origins
 INNER JOIN technologies
@@ -67,12 +76,14 @@ USING(technology)
 UNION ALL
 
 SELECT
-  client,
   'ALL' AS technology,
   NULL AS description,
+  NULL AS icon,
   NULL AS category,
   NULL AS category_obj,
-  NULL AS similar_technologies,
-  origins
+  STRUCT(
+    MAX(IF(client = 'desktop', origins, 0)) AS desktop,
+    MAX(IF(client = 'mobile', origins, 0)) AS mobile
+  ) AS origins
 FROM total_pages
 `)

From 6e246fbe0a0049604220c8af904d36d3fcff5d7d Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Thu, 13 Mar 2025 19:56:27 +0100
Subject: [PATCH 23/52] fix package versions

---
 package.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/package.json b/package.json
index 4afc17c9..a8fbeba6 100644
--- a/package.json
+++ b/package.json
@@ -2,7 +2,7 @@
   "name": "crawl-data",
   "author": "@max-ostapenko",
   "dependencies": {
-    "@dataform/core": "^3.0.15"
+    "@dataform/core": "3.0.15"
   },
   "scripts": {
     "format": "npx standard --fix; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint . --fix; terraform -chdir=infra/tf fmt -recursive",
@@ -20,8 +20,8 @@
     ]
   },
   "devDependencies": {
-    "markdownlint-cli": "^0.44.0",
-    "standard": "^17.1.2"
+    "markdownlint-cli": "0.44.0",
+    "standard": "17.1.2"
   },
   "markdownlint": {
     "default": true,

From 492dadbb5dee16417634dbbce288a5fe334c4a9e Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Thu, 13 Mar 2025 20:04:15 +0100
Subject: [PATCH 24/52] remove submodule

---
 .gitmodules                      |  3 ---
 infra/tf/.terraform.lock.hcl     | 26 +++++++++++++-------------
 infra/tf/bigquery_export/main.tf |  2 +-
 infra/tf/main.tf                 |  2 +-
 infra/tf/masthead-deployment     |  1 -
 5 files changed, 15 insertions(+), 19 deletions(-)
 delete mode 100644 .gitmodules
 delete mode 160000 infra/tf/masthead-deployment

diff --git a/.gitmodules b/.gitmodules
deleted file mode 100644
index cdd9adbf..00000000
--- a/.gitmodules
+++ /dev/null
@@ -1,3 +0,0 @@
-[submodule "infra/tf/masthead-deployment"]
-    path = infra/tf/masthead-deployment
-    url = https://github.com/Masthead-Data/masthead-deployment.git
diff --git a/infra/tf/.terraform.lock.hcl b/infra/tf/.terraform.lock.hcl
index 653ae725..da11b295 100644
--- a/infra/tf/.terraform.lock.hcl
+++ b/infra/tf/.terraform.lock.hcl
@@ -42,21 +42,21 @@ provider "registry.terraform.io/hashicorp/google" {
 }
 
 provider "registry.terraform.io/hashicorp/google-beta" {
-  version     = "6.24.0"
+  version     = "6.25.0"
   constraints = ">= 6.13.0"
   hashes = [
-    "h1:rSlf/p6mrMGjIy7yt7H0rFS+W1kLe/JqIPH8nJdLvl0=",
-    "zh:0daad0f1b3e030d6931a1d252d6625851a96927ce3cae86779c213f686f4f848",
-    "zh:24ae128cfbd8b2b6c3a10a3c8719719cafee9f449e4a737404dabc0a2b63845a",
-    "zh:43343015e1fe0653a855a8980b85b4c03e815903a4cde6c615c105c66ef7331d",
-    "zh:5eb97b570b4f9ab59ad3b7700e2a5019805ceb09e7ec557701c849c5c4a2a903",
-    "zh:63cbf916a40c8ac20c2efe80496a97c45d18f748c676faedd235d9d09b8c3616",
-    "zh:74063fba0ae1a951ddd9fef53158e08144901fbe7798d385beb08b5aae847ac4",
-    "zh:7716d4a3c1f7b338f231b9dc856dc83c000f0b44f860d78bda215fc48783d72a",
-    "zh:77709c91c0cc8bee738c59696063e1b1aeb9f672db9ca9b456e3a0bdbc4695c3",
-    "zh:96c337ecc8869c5a67d5a54bea9563e283dab0b4050eec829c3e3b5075399153",
-    "zh:98c5324f0a5b0da29b106d044373b1bc6185de9e8c7481e25c0d2c2a36eb1217",
-    "zh:cb21d979314ed318b20952f3297f53ba15afbe3cdaf430c4f9bbdaced81c827c",
+    "h1:KlIctHHQkbToSXa41DWFQUsTKsfaWe+vjs/CFbnj3bc=",
+    "zh:101a8a679274cae57e8c816164fa28b6d271a8a53b3e007b84153fc63f0f06f7",
+    "zh:413c41e4ee14b10cb815cd01f8b880c12474ecbf63ecfc6415512fb0c18c473b",
+    "zh:4260143c8584a8f338a4d45f65c2f08f12b28a746c43d43a1e3a575f96ce4110",
+    "zh:4785080dd28b79a94ba6e277d247b6bb678d676d99244a8d664ea60802338df7",
+    "zh:673b09d67cb59487095fe7176c194e8c7ced62fc7bb5084016f3c87dd89e685e",
+    "zh:7de0dd9367c723cc203fdc4e70d316c779224483637039de5a3c9ee807c3fc55",
+    "zh:aa49c5955652b21ff2d0b7b2db0e18f59097d3a539efc98ef65deed4293d7327",
+    "zh:b4d73704d88deee367568fd68303878e6e60924c2a65df397f37b868e6aecaac",
+    "zh:cae0c5eff7c37c3f9857de08ca35b3a86d44855ae9c0ec2fab1eda604ff501c8",
+    "zh:e115f6b0c25bdad624fafbb5d685c8a48d9028f9525918274a5e5469f7dfde3d",
+    "zh:e536197371458b0b177a203f8367f99e74770b3dfce597034e880ea5f3f798b8",
     "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
   ]
 }
diff --git a/infra/tf/bigquery_export/main.tf b/infra/tf/bigquery_export/main.tf
index 4347e806..7a492740 100644
--- a/infra/tf/bigquery_export/main.tf
+++ b/infra/tf/bigquery_export/main.tf
@@ -20,7 +20,7 @@ resource "google_cloud_run_v2_job" "bigquery_export" {
   deletion_protection = false
 
   template {
-    parallelism = 5
+    parallelism = 0
     template {
       containers {
         image = "${var.location}.gcr.io/${var.project}/cloud-run/${var.function_name}:latest"
diff --git a/infra/tf/main.tf b/infra/tf/main.tf
index 77ee684d..c95ff40e 100644
--- a/infra/tf/main.tf
+++ b/infra/tf/main.tf
@@ -58,7 +58,7 @@ module "masthead" {
   source = "./masthead"
 
   project = local.project
-  # source = "./masthead-deployment/app"
+  # source = "https://github.com/Masthead-Data/masthead-deployment"
   # project_id = local.project
   # project_number = local.project_number
 }
diff --git a/infra/tf/masthead-deployment b/infra/tf/masthead-deployment
deleted file mode 160000
index 92b5b6db..00000000
--- a/infra/tf/masthead-deployment
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 92b5b6db174e0b7939efcc14f17a7f421a4e81d5

From b13806d8937461649723ec8cd08cf5754da2ed75 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Thu, 13 Mar 2025 20:06:14 +0100
Subject: [PATCH 25/52] update packages

---
 package-lock.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 4154fb02..93345a1d 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -9,8 +9,8 @@
         "@dataform/core": "3.0.15"
       },
       "devDependencies": {
-        "markdownlint-cli": "^0.44.0",
-        "standard": "^17.1.2"
+        "markdownlint-cli": "0.44.0",
+        "standard": "17.1.2"
       }
     },
     "node_modules/@dataform/core": {

From d3fd4b4abeac6f249171b6131553bb8614c8504c Mon Sep 17 00:00:00 2001
From: Max Ostapenko <max@httparchive.org>
Date: Thu, 13 Mar 2025 19:12:28 +0000
Subject: [PATCH 26/52] test

---
 package-lock.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package-lock.json b/package-lock.json
index 4154fb02..32b4ac56 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -6,7 +6,7 @@
     "": {
       "name": "crawl-data",
       "dependencies": {
-        "@dataform/core": "3.0.15"
+        "@dataform/core": "^3.0.15"
       },
       "devDependencies": {
         "markdownlint-cli": "^0.44.0",

From d96e87e14c12bb760ced15966aa29bf148b97aec Mon Sep 17 00:00:00 2001
From: Max Ostapenko <max@httparchive.org>
Date: Thu, 13 Mar 2025 21:49:50 +0000
Subject: [PATCH 27/52] test

---
 package-lock.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package-lock.json b/package-lock.json
index 4154fb02..32b4ac56 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -6,7 +6,7 @@
     "": {
       "name": "crawl-data",
       "dependencies": {
-        "@dataform/core": "3.0.15"
+        "@dataform/core": "^3.0.15"
       },
       "devDependencies": {
         "markdownlint-cli": "^0.44.0",

From 9976f3abdc7d230224bc404fbd56828c22cc9de7 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Thu, 13 Mar 2025 23:46:42 +0100
Subject: [PATCH 28/52] rewrite triggers

---
 definitions/output/reports/reports_dynamic.js      | 14 +++++++++-----
 .../output/reports/tech_report_technologies.js     | 11 +++++++----
 infra/dataform-export/index.js                     |  3 +--
 3 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js
index 9309ba88..be0635d5 100644
--- a/definitions/output/reports/reports_dynamic.js
+++ b/definitions/output/reports/reports_dynamic.js
@@ -29,11 +29,12 @@ if (iterations.length === 1) {
 SELECT
   reports.run_export_job(
     JSON '''{
-      "dataform_trigger": "report_complete",
-      "date": "${params.date}",
-      "name": "${metric.id}",
-      "type": "${sql.type}",
-      "environment": "${constants.environment}"
+      "destination": "cloud_storage",
+      "config": {
+        "bucket": "httparchive",
+        "name": "reports/${constants.environment}/${metric.id}_${sql.type}_${params.date}.json"
+      },
+      "query": "SELECT FORMAT_DATE('%Y_%m_%d', date) AS date, * EXCEPT(date) FROM ${ctx.self()}"
     }'''
   );
       `)
@@ -66,3 +67,6 @@ SELECT
     })
   })
 }
+
+// --"query": "SELECT * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${params.date}'"
+// `${this.storagePath}${date.replaceAll('-', '_')}/${metric}.json`
diff --git a/definitions/output/reports/tech_report_technologies.js b/definitions/output/reports/tech_report_technologies.js
index 5d481ffc..591daad1 100644
--- a/definitions/output/reports/tech_report_technologies.js
+++ b/definitions/output/reports/tech_report_technologies.js
@@ -89,10 +89,13 @@ FROM total_pages
   SELECT
     reports.run_export_job(
       JSON '''{
-        "dataform_trigger": "tech_report_complete",
-        "name": "technologies",
-        "type": "dict",
-        "environment": "${constants.environment}"
+        "destination": "firestore",
+        "config": {
+          "database": "tech-report-api-${constants.environment}"
+          "collection": "technologies",
+          "type": "dict"
+        },
+        "query": "SELECT * FROM ${ctx.self()}"
       }'''
     );
 `)
diff --git a/infra/dataform-export/index.js b/infra/dataform-export/index.js
index 7d4fc47f..fb07269d 100644
--- a/infra/dataform-export/index.js
+++ b/infra/dataform-export/index.js
@@ -29,8 +29,7 @@ async function callRunJob (payload = {}) {
 }
 
 function hasRequiredKeys (obj) {
-  const baseRequiredKeys = ['dataform_trigger', 'name', 'type', 'environment']
-  const requiredKeys = obj.type === 'report' ? [...baseRequiredKeys, 'date'] : baseRequiredKeys
+  const requiredKeys = ['destination', 'config', 'query']
   return requiredKeys.every(key => key in obj)
 }
 

From 4b477c292636b5c61b76ada92959933ebf6c1222 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Thu, 13 Mar 2025 23:47:10 +0100
Subject: [PATCH 29/52] adjust bq export

---
 infra/bigquery-export/index.js   | 42 ++++++++--------
 infra/bigquery-export/reports.js | 85 --------------------------------
 infra/bigquery-export/storage.js | 11 +++--
 3 files changed, 26 insertions(+), 112 deletions(-)
 delete mode 100644 infra/bigquery-export/reports.js

diff --git a/infra/bigquery-export/index.js b/infra/bigquery-export/index.js
index 449ccea4..b38e442d 100644
--- a/infra/bigquery-export/index.js
+++ b/infra/bigquery-export/index.js
@@ -1,34 +1,32 @@
-import { ReportsExporter, TechReportsExporter } from './reports.js'
+import { StorageUpload } from './storage.js'
+import { FirestoreBatch } from './firestore.js'
 
-const exportConfig = process.env.EXPORT_CONFIG && JSON.parse(process.env.EXPORT_CONFIG)
-
-async function main (exportConfig) {
-  if (!exportConfig) {
-    throw new Error('No config received')
+async function main () {
+  const { query, destination, config } = process.env.EXPORT_CONFIG && JSON.parse(process.env.EXPORT_CONFIG)
+  if (!destination) {
+    throw new Error('No destination found')
   }
 
-  const eventName = exportConfig.dataform_trigger
-  if (!eventName) {
-    throw new Error('No trigger name found')
-  }
+  if (destination === 'cloud_storage') {
+    console.info('Cloud Storage export')
+    console.log(query, config)
+
+    const storage = new StorageUpload(config.bucket)
+    await storage.exportToJson(query, config.name)
+  } else if (destination === 'firestore') {
+    console.info('Firestore export')
+    console.log(query, config)
 
-  if (eventName === 'report_complete') {
-    console.info('Report export')
-    console.log(exportConfig)
-    const reports = new ReportsExporter()
-    await reports.export(exportConfig)
-  } else if (eventName === 'tech_report_complete') {
-    console.info('Tech Report export')
-    console.log(exportConfig)
-    const techReports = new TechReportsExporter()
-    await techReports.export(exportConfig)
+    const firestore = new FirestoreBatch()
+    await firestore.export(config, query)
   } else {
-    throw new Error('Bad Request: unknown trigger name')
+    throw new Error('Bad Request: destination unknown')
   }
+  console.info('Export finished successfully')
   return 'OK'
 }
 
-await main(exportConfig).catch((error) => {
+await main().catch((error) => {
   console.error(error)
   process.exit(1)
 })
diff --git a/infra/bigquery-export/reports.js b/infra/bigquery-export/reports.js
deleted file mode 100644
index b621f199..00000000
--- a/infra/bigquery-export/reports.js
+++ /dev/null
@@ -1,85 +0,0 @@
-import { BigQueryExport } from './bigquery.js'
-import { StorageExport } from './storage.js'
-import { FirestoreBatch } from './firestore.js'
-
-export class ReportsExporter {
-  constructor () {
-    this.bigquery = new BigQueryExport()
-    this.storage = new StorageExport()
-  }
-
-  // Export timeseries reports
-  async exportTimeseries (exportConfig) {
-    const metric = exportConfig.name
-    const query = `
-SELECT
-  FORMAT_DATE('%Y_%m_%d', date) AS date,
-  * EXCEPT(date)
-FROM reports.${metric}_timeseries
-`
-    const rows = await this.bigquery.queryResults(query)
-    await this.storage.exportToJson(rows, metric)
-  }
-
-  // Export monthly histogram report
-  async exportHistogram (exportConfig) {
-    const metric = exportConfig.name
-    const date = exportConfig.date
-
-    const query = `
-SELECT * EXCEPT(date)
-FROM reports.${metric}_histogram
-WHERE date = '${date}'
-`
-    const rows = await this.bigquery.queryResults(query)
-    await this.storage.exportToJson(rows, `${this.storagePath}${date.replaceAll('-', '_')}/${metric}.json`)
-  }
-
-  async export (exportConfig) {
-    if (exportConfig.dataform_trigger !== 'report_complete') {
-      console.error('Invalid dataform trigger')
-      return
-    }
-
-    if (exportConfig.type === 'histogram') {
-      await this.exportHistogram(exportConfig)
-    } else if (exportConfig.type === 'timeseries') {
-      await this.exportTimeseries(exportConfig)
-    } else {
-      console.error('Invalid report type')
-    }
-  }
-}
-
-export class TechReportsExporter {
-  constructor () {
-    this.firestore = new FirestoreBatch()
-  }
-
-  async export (exportConfig) {
-    if (exportConfig.dataform_trigger !== 'tech_report_complete') {
-      console.error('Invalid dataform trigger')
-      return
-    }
-
-    let query = ''
-    if (exportConfig.type === 'report') {
-      query = `
-SELECT
-  STRING(date) AS date,
-  * EXCEPT(date)
-FROM httparchive.reports.tech_report_${exportConfig.name}
-WHERE date = '${exportConfig.date}'
-`
-    } else if (exportConfig.type === 'dict') {
-      query = `
-SELECT *
-FROM reports.tech_report_${exportConfig.name}
-`
-    } else {
-      console.error('Invalid export type')
-    }
-
-    await this.firestore.export(exportConfig, query)
-  }
-}
diff --git a/infra/bigquery-export/storage.js b/infra/bigquery-export/storage.js
index a6850905..3a4e9630 100644
--- a/infra/bigquery-export/storage.js
+++ b/infra/bigquery-export/storage.js
@@ -1,21 +1,22 @@
 import { Storage } from '@google-cloud/storage'
+import { BigQueryExport } from './bigquery.js'
 import { Readable } from 'stream'
 import zlib from 'zlib'
 
+const bigquery = new BigQueryExport()
 const storage = new Storage()
 
-export class StorageExport {
-  constructor (bucket = 'httparchive') {
+export class StorageUpload {
+  constructor (bucket) {
     this.bucket = bucket
-    this.storagePath = 'reports/dev/' // TODO change to prod
     this.stream = new Readable({
       objectMode: true,
       read () {}
     })
   }
 
-  async exportToJson (data, fileName) {
-    fileName = this.storagePath + fileName + '.json'
+  async exportToJson (query, fileName) {
+    const data = await bigquery.queryResults(query)
 
     const bucket = storage.bucket(this.bucket)
     const file = bucket.file(fileName)

From 202b345a7209e55f4f467780bb7a5b1da8b38f3a Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Fri, 14 Mar 2025 00:07:17 +0100
Subject: [PATCH 30/52] mh roles update

---
 infra/tf/masthead/main.tf | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/infra/tf/masthead/main.tf b/infra/tf/masthead/main.tf
index 246d933d..31c7dc24 100644
--- a/infra/tf/masthead/main.tf
+++ b/infra/tf/masthead/main.tf
@@ -58,16 +58,12 @@ resource "google_project_iam_custom_role" "masthead_bq_meta_reader" {
   title       = "masthead_bq_meta_reader"
 }
 
-resource "google_project_iam_binding" "masthead_bq_meta_reader_binding" {
-  role    = google_project_iam_custom_role.masthead_bq_meta_reader.id
-  members = ["serviceAccount:masthead-data@masthead-prod.iam.gserviceaccount.com"]
-  project = var.project
-}
-
 resource "google_project_iam_member" "masthead_pubsub_subscriber_member" {
-  role    = "roles/pubsub.subscriber"
-  member  = "serviceAccount:masthead-data@masthead-prod.iam.gserviceaccount.com"
+  for_each = toset(["roles/bigquery.metadataViewer", "roles/bigquery.resourceViewer", "roles/pubsub.subscriber"])
+
   project = var.project
+  role    = each.value
+  member  = "serviceAccount:masthead-data@masthead-prod.iam.gserviceaccount.com"
 }
 
 # 4. Grant Masthead Service Account to quickly onboard from retrospective data

From a0dc7a0613c256aaea3f112213c6196c81204fe8 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Fri, 14 Mar 2025 00:09:39 +0100
Subject: [PATCH 31/52] cleanup

---
 infra/tf/masthead-deployment | 1 -
 1 file changed, 1 deletion(-)
 delete mode 160000 infra/tf/masthead-deployment

diff --git a/infra/tf/masthead-deployment b/infra/tf/masthead-deployment
deleted file mode 160000
index 92b5b6db..00000000
--- a/infra/tf/masthead-deployment
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 92b5b6db174e0b7939efcc14f17a7f421a4e81d5

From 22df13b334dcceb6245f42481ed9d58fd19ad590 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Fri, 14 Mar 2025 00:11:09 +0100
Subject: [PATCH 32/52] packages update

---
 package-lock.json | 1144 +++++++++++++++++++++++++++------------------
 1 file changed, 697 insertions(+), 447 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index b14ef024..86619c43 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -6,7 +6,7 @@
     "": {
       "name": "crawl-data",
       "dependencies": {
-        "@dataform/core": "^3.0.15"
+        "@dataform/core": "3.0.15"
       },
       "devDependencies": {
         "markdownlint-cli": "0.44.0",
@@ -20,9 +20,9 @@
       "license": "Apache-2.0"
     },
     "node_modules/@eslint-community/eslint-utils": {
-      "version": "4.4.1",
-      "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.1.tgz",
-      "integrity": "sha512-s3O3waFUrMV8P/XaF/+ZTp1X9XBZW1a4B97ZnjQF2KYWaFD2A8KyFBsrsfSjEmjn3RGWAIuvlneuZm3CUK3jbA==",
+      "version": "4.5.1",
+      "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.5.1.tgz",
+      "integrity": "sha512-soEIOALTfTK6EjmKMMoLugwaP0rzkad90iIWd1hMO9ARkSAyjfMfkRRhLvD5qH7vvM0Cg72pieUfR6yh6XxC4w==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -291,16 +291,16 @@
       "license": "MIT"
     },
     "node_modules/@ungap/structured-clone": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz",
-      "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==",
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz",
+      "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==",
       "dev": true,
       "license": "ISC"
     },
     "node_modules/acorn": {
-      "version": "8.14.0",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz",
-      "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==",
+      "version": "8.14.1",
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.1.tgz",
+      "integrity": "sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg==",
       "dev": true,
       "license": "MIT",
       "bin": {
@@ -374,14 +374,14 @@
       "license": "Python-2.0"
     },
     "node_modules/array-buffer-byte-length": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.1.tgz",
-      "integrity": "sha512-ahC5W1xgou+KTXix4sAO8Ki12Q+jf4i0+tmk3sC+zgcynshkHxzpXdImBehiUYKKKDwvfFiJl1tZt6ewscS1Mg==",
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.2.tgz",
+      "integrity": "sha512-LHE+8BuR7RYGDKvnrmcuSq3tDcKv9OFEXQt/HpbZhY7V6h0zlUXutnAD82GiFx9rdieCMjkvtcsPqBwgUl1Iiw==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.5",
-        "is-array-buffer": "^3.0.4"
+        "call-bound": "^1.0.3",
+        "is-array-buffer": "^3.0.5"
       },
       "engines": {
         "node": ">= 0.4"
@@ -454,16 +454,16 @@
       }
     },
     "node_modules/array.prototype.flat": {
-      "version": "1.3.2",
-      "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.2.tgz",
-      "integrity": "sha512-djYB+Zx2vLewY8RWlNCUdHjDXs2XOgm602S9E7P/UpHgfeHL00cRiIF+IN/G/aUJ7kGPb6yO/ErDI5V2s8iycA==",
+      "version": "1.3.3",
+      "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.3.tgz",
+      "integrity": "sha512-rwG/ja1neyLqCuGZ5YYrznA62D4mZXg0i1cIskIUKSiqF3Cje9/wXAls9B9s1Wa2fomMsIv8czB8jZcPmxCXFg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.2",
-        "define-properties": "^1.2.0",
-        "es-abstract": "^1.22.1",
-        "es-shim-unscopables": "^1.0.0"
+        "call-bind": "^1.0.8",
+        "define-properties": "^1.2.1",
+        "es-abstract": "^1.23.5",
+        "es-shim-unscopables": "^1.0.2"
       },
       "engines": {
         "node": ">= 0.4"
@@ -473,16 +473,16 @@
       }
     },
     "node_modules/array.prototype.flatmap": {
-      "version": "1.3.2",
-      "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.3.2.tgz",
-      "integrity": "sha512-Ewyx0c9PmpcsByhSW4r+9zDU7sGjFc86qf/kKtuSCRdhfbk0SNLLkaT5qvcHnRGgc5NP/ly/y+qkXkqONX54CQ==",
+      "version": "1.3.3",
+      "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.3.3.tgz",
+      "integrity": "sha512-Y7Wt51eKJSyi80hFrJCePGGNo5ktJCslFuboqJsbf57CCPcm5zztluPlc4/aD8sWsKvlwatezpV4U1efk8kpjg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.2",
-        "define-properties": "^1.2.0",
-        "es-abstract": "^1.22.1",
-        "es-shim-unscopables": "^1.0.0"
+        "call-bind": "^1.0.8",
+        "define-properties": "^1.2.1",
+        "es-abstract": "^1.23.5",
+        "es-shim-unscopables": "^1.0.2"
       },
       "engines": {
         "node": ">= 0.4"
@@ -509,20 +509,19 @@
       }
     },
     "node_modules/arraybuffer.prototype.slice": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.3.tgz",
-      "integrity": "sha512-bMxMKAjg13EBSVscxTaYA4mRc5t1UAXa2kXiGTNfZ079HIWXEkKmkgFrh/nJqamaLSrXO5H4WFFkPEaLJWbs3A==",
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.4.tgz",
+      "integrity": "sha512-BNoCY6SXXPQ7gF2opIP4GBE+Xw7U+pHMYKuzjgCN3GwiaIR09UUeKfheyIry77QtrCBlC0KK0q5/TER/tYh3PQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "array-buffer-byte-length": "^1.0.1",
-        "call-bind": "^1.0.5",
+        "call-bind": "^1.0.8",
         "define-properties": "^1.2.1",
-        "es-abstract": "^1.22.3",
-        "es-errors": "^1.2.1",
-        "get-intrinsic": "^1.2.3",
-        "is-array-buffer": "^3.0.4",
-        "is-shared-array-buffer": "^1.0.2"
+        "es-abstract": "^1.23.5",
+        "es-errors": "^1.3.0",
+        "get-intrinsic": "^1.2.6",
+        "is-array-buffer": "^3.0.4"
       },
       "engines": {
         "node": ">= 0.4"
@@ -531,6 +530,16 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/async-function": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/async-function/-/async-function-1.0.0.tgz",
+      "integrity": "sha512-hsU18Ae8CDTR6Kgu9DYf0EbCr/a5iGL0rytQDobUcdpYOKokk8LEjVphnXkDkgpi0wYVsqrXuP0bZxJaTqdgoA==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/available-typed-arrays": {
       "version": "1.0.7",
       "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz",
@@ -575,9 +584,9 @@
       }
     },
     "node_modules/builtins/node_modules/semver": {
-      "version": "7.6.3",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz",
-      "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==",
+      "version": "7.7.1",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz",
+      "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==",
       "dev": true,
       "license": "ISC",
       "bin": {
@@ -588,17 +597,47 @@
       }
     },
     "node_modules/call-bind": {
-      "version": "1.0.7",
-      "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz",
-      "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==",
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz",
+      "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
+        "call-bind-apply-helpers": "^1.0.0",
         "es-define-property": "^1.0.0",
-        "es-errors": "^1.3.0",
-        "function-bind": "^1.1.2",
         "get-intrinsic": "^1.2.4",
-        "set-function-length": "^1.2.1"
+        "set-function-length": "^1.2.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/call-bind-apply-helpers": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
+      "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "es-errors": "^1.3.0",
+        "function-bind": "^1.1.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/call-bound": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz",
+      "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "call-bind-apply-helpers": "^1.0.2",
+        "get-intrinsic": "^1.3.0"
       },
       "engines": {
         "node": ">= 0.4"
@@ -705,9 +744,9 @@
       "license": "MIT"
     },
     "node_modules/cross-spawn": {
-      "version": "7.0.5",
-      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.5.tgz",
-      "integrity": "sha512-ZVJrKKYunU38/76t0RMOulHOnUcbU9GbpWKAOZ0mhjr7CX6FVrH+4FrAapSOekrgFQ3f/8gwMEuIft0aKq6Hug==",
+      "version": "7.0.6",
+      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
+      "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -720,15 +759,15 @@
       }
     },
     "node_modules/data-view-buffer": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/data-view-buffer/-/data-view-buffer-1.0.1.tgz",
-      "integrity": "sha512-0lht7OugA5x3iJLOWFhWK/5ehONdprk0ISXqVFn/NFrDu+cuc8iADFrGQz5BnRK7LLU3JmkbXSxaqX+/mXYtUA==",
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/data-view-buffer/-/data-view-buffer-1.0.2.tgz",
+      "integrity": "sha512-EmKO5V3OLXh1rtK2wgXRansaK1/mtVdTUEiEI0W8RkvgT05kfxaH29PliLnpLP73yYO6142Q72QNa8Wx/A5CqQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.6",
+        "call-bound": "^1.0.3",
         "es-errors": "^1.3.0",
-        "is-data-view": "^1.0.1"
+        "is-data-view": "^1.0.2"
       },
       "engines": {
         "node": ">= 0.4"
@@ -738,31 +777,31 @@
       }
     },
     "node_modules/data-view-byte-length": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/data-view-byte-length/-/data-view-byte-length-1.0.1.tgz",
-      "integrity": "sha512-4J7wRJD3ABAzr8wP+OcIcqq2dlUKp4DVflx++hs5h5ZKydWMI6/D/fAot+yh6g2tHh8fLFTvNOaVN357NvSrOQ==",
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/data-view-byte-length/-/data-view-byte-length-1.0.2.tgz",
+      "integrity": "sha512-tuhGbE6CfTM9+5ANGf+oQb72Ky/0+s3xKUpHvShfiz2RxMFgFPjsXuRLBVMtvMs15awe45SRb83D6wH4ew6wlQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.7",
+        "call-bound": "^1.0.3",
         "es-errors": "^1.3.0",
-        "is-data-view": "^1.0.1"
+        "is-data-view": "^1.0.2"
       },
       "engines": {
         "node": ">= 0.4"
       },
       "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+        "url": "https://github.com/sponsors/inspect-js"
       }
     },
     "node_modules/data-view-byte-offset": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/data-view-byte-offset/-/data-view-byte-offset-1.0.0.tgz",
-      "integrity": "sha512-t/Ygsytq+R995EJ5PZlD4Cu56sWa8InXySaViRzw9apusqsOO2bQP+SbYzAhR0pFKoB+43lYy8rWban9JSuXnA==",
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/data-view-byte-offset/-/data-view-byte-offset-1.0.1.tgz",
+      "integrity": "sha512-BS8PfmtDGnrgYdOonGZQdLZslWIeCGFP9tpan0hi1Co2Zr2NKADsvGYA8XxuG/4UWgJ6Cjtv+YJnB6MM69QGlQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.6",
+        "call-bound": "^1.0.2",
         "es-errors": "^1.3.0",
         "is-data-view": "^1.0.1"
       },
@@ -774,9 +813,9 @@
       }
     },
     "node_modules/debug": {
-      "version": "4.3.7",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz",
-      "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==",
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
+      "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -792,9 +831,9 @@
       }
     },
     "node_modules/decode-named-character-reference": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz",
-      "integrity": "sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.1.0.tgz",
+      "integrity": "sha512-Wy+JTSbFThEOXQIR2L6mxJvEs+veIzpmqD7ynWxMXGpnk3smkHQOp6forLdHsKpAMW9iJpaBBIxz285t1n1C3w==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -895,6 +934,21 @@
         "node": ">=6.0.0"
       }
     },
+    "node_modules/dunder-proto": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
+      "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "call-bind-apply-helpers": "^1.0.1",
+        "es-errors": "^1.3.0",
+        "gopd": "^1.2.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/eastasianwidth": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
@@ -933,58 +987,63 @@
       }
     },
     "node_modules/es-abstract": {
-      "version": "1.23.3",
-      "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.23.3.tgz",
-      "integrity": "sha512-e+HfNH61Bj1X9/jLc5v1owaLYuHdeHHSQlkhCBiTK8rBvKaULl/beGMxwrMXjpYrv4pz22BlY570vVePA2ho4A==",
+      "version": "1.23.9",
+      "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.23.9.tgz",
+      "integrity": "sha512-py07lI0wjxAC/DcfK1S6G7iANonniZwTISvdPzk9hzeH0IZIshbuuFxLIU96OyF89Yb9hiqWn8M/bY83KY5vzA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "array-buffer-byte-length": "^1.0.1",
-        "arraybuffer.prototype.slice": "^1.0.3",
+        "array-buffer-byte-length": "^1.0.2",
+        "arraybuffer.prototype.slice": "^1.0.4",
         "available-typed-arrays": "^1.0.7",
-        "call-bind": "^1.0.7",
-        "data-view-buffer": "^1.0.1",
-        "data-view-byte-length": "^1.0.1",
-        "data-view-byte-offset": "^1.0.0",
-        "es-define-property": "^1.0.0",
+        "call-bind": "^1.0.8",
+        "call-bound": "^1.0.3",
+        "data-view-buffer": "^1.0.2",
+        "data-view-byte-length": "^1.0.2",
+        "data-view-byte-offset": "^1.0.1",
+        "es-define-property": "^1.0.1",
         "es-errors": "^1.3.0",
         "es-object-atoms": "^1.0.0",
-        "es-set-tostringtag": "^2.0.3",
-        "es-to-primitive": "^1.2.1",
-        "function.prototype.name": "^1.1.6",
-        "get-intrinsic": "^1.2.4",
-        "get-symbol-description": "^1.0.2",
-        "globalthis": "^1.0.3",
-        "gopd": "^1.0.1",
+        "es-set-tostringtag": "^2.1.0",
+        "es-to-primitive": "^1.3.0",
+        "function.prototype.name": "^1.1.8",
+        "get-intrinsic": "^1.2.7",
+        "get-proto": "^1.0.0",
+        "get-symbol-description": "^1.1.0",
+        "globalthis": "^1.0.4",
+        "gopd": "^1.2.0",
         "has-property-descriptors": "^1.0.2",
-        "has-proto": "^1.0.3",
-        "has-symbols": "^1.0.3",
+        "has-proto": "^1.2.0",
+        "has-symbols": "^1.1.0",
         "hasown": "^2.0.2",
-        "internal-slot": "^1.0.7",
-        "is-array-buffer": "^3.0.4",
+        "internal-slot": "^1.1.0",
+        "is-array-buffer": "^3.0.5",
         "is-callable": "^1.2.7",
-        "is-data-view": "^1.0.1",
-        "is-negative-zero": "^2.0.3",
-        "is-regex": "^1.1.4",
-        "is-shared-array-buffer": "^1.0.3",
-        "is-string": "^1.0.7",
-        "is-typed-array": "^1.1.13",
-        "is-weakref": "^1.0.2",
-        "object-inspect": "^1.13.1",
+        "is-data-view": "^1.0.2",
+        "is-regex": "^1.2.1",
+        "is-shared-array-buffer": "^1.0.4",
+        "is-string": "^1.1.1",
+        "is-typed-array": "^1.1.15",
+        "is-weakref": "^1.1.0",
+        "math-intrinsics": "^1.1.0",
+        "object-inspect": "^1.13.3",
         "object-keys": "^1.1.1",
-        "object.assign": "^4.1.5",
-        "regexp.prototype.flags": "^1.5.2",
-        "safe-array-concat": "^1.1.2",
-        "safe-regex-test": "^1.0.3",
-        "string.prototype.trim": "^1.2.9",
-        "string.prototype.trimend": "^1.0.8",
+        "object.assign": "^4.1.7",
+        "own-keys": "^1.0.1",
+        "regexp.prototype.flags": "^1.5.3",
+        "safe-array-concat": "^1.1.3",
+        "safe-push-apply": "^1.0.0",
+        "safe-regex-test": "^1.1.0",
+        "set-proto": "^1.0.0",
+        "string.prototype.trim": "^1.2.10",
+        "string.prototype.trimend": "^1.0.9",
         "string.prototype.trimstart": "^1.0.8",
-        "typed-array-buffer": "^1.0.2",
-        "typed-array-byte-length": "^1.0.1",
-        "typed-array-byte-offset": "^1.0.2",
-        "typed-array-length": "^1.0.6",
-        "unbox-primitive": "^1.0.2",
-        "which-typed-array": "^1.1.15"
+        "typed-array-buffer": "^1.0.3",
+        "typed-array-byte-length": "^1.0.3",
+        "typed-array-byte-offset": "^1.0.4",
+        "typed-array-length": "^1.0.7",
+        "unbox-primitive": "^1.1.0",
+        "which-typed-array": "^1.1.18"
       },
       "engines": {
         "node": ">= 0.4"
@@ -994,14 +1053,11 @@
       }
     },
     "node_modules/es-define-property": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz",
-      "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==",
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
+      "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==",
       "dev": true,
       "license": "MIT",
-      "dependencies": {
-        "get-intrinsic": "^1.2.4"
-      },
       "engines": {
         "node": ">= 0.4"
       }
@@ -1017,36 +1073,37 @@
       }
     },
     "node_modules/es-iterator-helpers": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/es-iterator-helpers/-/es-iterator-helpers-1.2.0.tgz",
-      "integrity": "sha512-tpxqxncxnpw3c93u8n3VOzACmRFoVmWJqbWXvX/JfKbkhBw1oslgPrUfeSt2psuqyEJFD6N/9lg5i7bsKpoq+Q==",
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/es-iterator-helpers/-/es-iterator-helpers-1.2.1.tgz",
+      "integrity": "sha512-uDn+FE1yrDzyC0pCo961B2IHbdM8y/ACZsKD4dG6WqrjV53BADjwa7D+1aom2rsNVfLyDgU/eigvlJGJ08OQ4w==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.7",
+        "call-bind": "^1.0.8",
+        "call-bound": "^1.0.3",
         "define-properties": "^1.2.1",
-        "es-abstract": "^1.23.3",
+        "es-abstract": "^1.23.6",
         "es-errors": "^1.3.0",
         "es-set-tostringtag": "^2.0.3",
         "function-bind": "^1.1.2",
-        "get-intrinsic": "^1.2.4",
+        "get-intrinsic": "^1.2.6",
         "globalthis": "^1.0.4",
-        "gopd": "^1.0.1",
+        "gopd": "^1.2.0",
         "has-property-descriptors": "^1.0.2",
-        "has-proto": "^1.0.3",
-        "has-symbols": "^1.0.3",
-        "internal-slot": "^1.0.7",
-        "iterator.prototype": "^1.1.3",
-        "safe-array-concat": "^1.1.2"
+        "has-proto": "^1.2.0",
+        "has-symbols": "^1.1.0",
+        "internal-slot": "^1.1.0",
+        "iterator.prototype": "^1.1.4",
+        "safe-array-concat": "^1.1.3"
       },
       "engines": {
         "node": ">= 0.4"
       }
     },
     "node_modules/es-object-atoms": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.0.0.tgz",
-      "integrity": "sha512-MZ4iQ6JwHOBQjahnjwaC1ZtIBH+2ohjamzAO3oaHcXYup7qxjF2fixyH+Q71voWHeOkI2q/TnJao/KfXYIZWbw==",
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz",
+      "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -1057,40 +1114,44 @@
       }
     },
     "node_modules/es-set-tostringtag": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.3.tgz",
-      "integrity": "sha512-3T8uNMC3OQTHkFUsFq8r/BwAXLHvU/9O9mE0fBc/MY5iq/8H7ncvO947LmYA6ldWw9Uh8Yhf25zu6n7nML5QWQ==",
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
+      "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "get-intrinsic": "^1.2.4",
+        "es-errors": "^1.3.0",
+        "get-intrinsic": "^1.2.6",
         "has-tostringtag": "^1.0.2",
-        "hasown": "^2.0.1"
+        "hasown": "^2.0.2"
       },
       "engines": {
         "node": ">= 0.4"
       }
     },
     "node_modules/es-shim-unscopables": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/es-shim-unscopables/-/es-shim-unscopables-1.0.2.tgz",
-      "integrity": "sha512-J3yBRXCzDu4ULnQwxyToo/OjdMx6akgVC7K6few0a7F/0wLtmKKN7I73AH5T2836UuXRqN7Qg+IIUw/+YJksRw==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/es-shim-unscopables/-/es-shim-unscopables-1.1.0.tgz",
+      "integrity": "sha512-d9T8ucsEhh8Bi1woXCf+TIKDIROLG5WCkxg8geBCbvk22kzwC5G2OnXVMO6FUsvQlgUUXQ2itephWDLqDzbeCw==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "hasown": "^2.0.0"
+        "hasown": "^2.0.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
       }
     },
     "node_modules/es-to-primitive": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz",
-      "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==",
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.3.0.tgz",
+      "integrity": "sha512-w+5mJ3GuFL+NjVtJlvydShqE1eN3h3PbI7/5LAsYJP/2qtuMXjfL2LpHSRqo4b4eSF5K/DH1JXKUAHSB2UW50g==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "is-callable": "^1.1.4",
-        "is-date-object": "^1.0.1",
-        "is-symbol": "^1.0.2"
+        "is-callable": "^1.2.7",
+        "is-date-object": "^1.0.5",
+        "is-symbol": "^1.0.4"
       },
       "engines": {
         "node": ">= 0.4"
@@ -1462,9 +1523,9 @@
       }
     },
     "node_modules/eslint-plugin-n/node_modules/semver": {
-      "version": "7.6.3",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz",
-      "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==",
+      "version": "7.7.1",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz",
+      "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==",
       "dev": true,
       "license": "ISC",
       "bin": {
@@ -1491,29 +1552,29 @@
       }
     },
     "node_modules/eslint-plugin-react": {
-      "version": "7.37.2",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.37.2.tgz",
-      "integrity": "sha512-EsTAnj9fLVr/GZleBLFbj/sSuXeWmp1eXIN60ceYnZveqEaUCyW4X+Vh4WTdUhCkW4xutXYqTXCUSyqD4rB75w==",
+      "version": "7.37.4",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.37.4.tgz",
+      "integrity": "sha512-BGP0jRmfYyvOyvMoRX/uoUeW+GqNj9y16bPQzqAHf3AYII/tDs+jMN0dBVkl88/OZwNGwrVFxE7riHsXVfy/LQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "array-includes": "^3.1.8",
         "array.prototype.findlast": "^1.2.5",
-        "array.prototype.flatmap": "^1.3.2",
+        "array.prototype.flatmap": "^1.3.3",
         "array.prototype.tosorted": "^1.1.4",
         "doctrine": "^2.1.0",
-        "es-iterator-helpers": "^1.1.0",
+        "es-iterator-helpers": "^1.2.1",
         "estraverse": "^5.3.0",
         "hasown": "^2.0.2",
         "jsx-ast-utils": "^2.4.1 || ^3.0.0",
         "minimatch": "^3.1.2",
         "object.entries": "^1.1.8",
         "object.fromentries": "^2.0.8",
-        "object.values": "^1.2.0",
+        "object.values": "^1.2.1",
         "prop-types": "^15.8.1",
         "resolve": "^2.0.0-next.5",
         "semver": "^6.3.1",
-        "string.prototype.matchall": "^4.0.11",
+        "string.prototype.matchall": "^4.0.12",
         "string.prototype.repeat": "^1.0.0"
       },
       "engines": {
@@ -1780,9 +1841,9 @@
       "license": "MIT"
     },
     "node_modules/fastq": {
-      "version": "1.17.1",
-      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz",
-      "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==",
+      "version": "1.19.1",
+      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.19.1.tgz",
+      "integrity": "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==",
       "dev": true,
       "license": "ISC",
       "dependencies": {
@@ -1835,30 +1896,36 @@
       }
     },
     "node_modules/flatted": {
-      "version": "3.3.1",
-      "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz",
-      "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==",
+      "version": "3.3.3",
+      "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz",
+      "integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==",
       "dev": true,
       "license": "ISC"
     },
     "node_modules/for-each": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz",
-      "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==",
+      "version": "0.3.5",
+      "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz",
+      "integrity": "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "is-callable": "^1.1.3"
+        "is-callable": "^1.2.7"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
     "node_modules/foreground-child": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.0.tgz",
-      "integrity": "sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==",
+      "version": "3.3.1",
+      "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz",
+      "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==",
       "dev": true,
       "license": "ISC",
       "dependencies": {
-        "cross-spawn": "^7.0.0",
+        "cross-spawn": "^7.0.6",
         "signal-exit": "^4.0.1"
       },
       "engines": {
@@ -1886,16 +1953,18 @@
       }
     },
     "node_modules/function.prototype.name": {
-      "version": "1.1.6",
-      "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.6.tgz",
-      "integrity": "sha512-Z5kx79swU5P27WEayXM1tBi5Ze/lbIyiNgU3qyXUOf9b2rgXYyF9Dy9Cx+IQv/Lc8WCG6L82zwUPpSS9hGehIg==",
+      "version": "1.1.8",
+      "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.8.tgz",
+      "integrity": "sha512-e5iwyodOHhbMr/yNrc7fDYG4qlbIvI5gajyzPnb5TCwyhjApznQh1BMFou9b30SevY43gCJKXycoCBjMbsuW0Q==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.2",
-        "define-properties": "^1.2.0",
-        "es-abstract": "^1.22.1",
-        "functions-have-names": "^1.2.3"
+        "call-bind": "^1.0.8",
+        "call-bound": "^1.0.3",
+        "define-properties": "^1.2.1",
+        "functions-have-names": "^1.2.3",
+        "hasown": "^2.0.2",
+        "is-callable": "^1.2.7"
       },
       "engines": {
         "node": ">= 0.4"
@@ -1915,17 +1984,22 @@
       }
     },
     "node_modules/get-intrinsic": {
-      "version": "1.2.4",
-      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz",
-      "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==",
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
+      "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
+        "call-bind-apply-helpers": "^1.0.2",
+        "es-define-property": "^1.0.1",
         "es-errors": "^1.3.0",
+        "es-object-atoms": "^1.1.1",
         "function-bind": "^1.1.2",
-        "has-proto": "^1.0.1",
-        "has-symbols": "^1.0.3",
-        "hasown": "^2.0.0"
+        "get-proto": "^1.0.1",
+        "gopd": "^1.2.0",
+        "has-symbols": "^1.1.0",
+        "hasown": "^2.0.2",
+        "math-intrinsics": "^1.1.0"
       },
       "engines": {
         "node": ">= 0.4"
@@ -1934,16 +2008,43 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/get-proto": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
+      "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "dunder-proto": "^1.0.1",
+        "es-object-atoms": "^1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/get-stdin": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-8.0.0.tgz",
+      "integrity": "sha512-sY22aA6xchAzprjyqmSEQv4UbAAzRN0L2dQB0NlN5acTTK9Don6nhoc3eAbUnpZiCANAMfd/+40kVdKfFygohg==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/get-symbol-description": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.2.tgz",
-      "integrity": "sha512-g0QYk1dZBxGwk+Ngc+ltRH2IBp2f7zBkBMBJZCDerh6EhlhSR6+9irMCuT/09zD6qkarHUSn529sK/yL4S27mg==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.1.0.tgz",
+      "integrity": "sha512-w9UMqWwJxHNOvoNzSJ2oPF5wvYcvP7jUvYzhp67yEhTi17ZDBBC1z9pTdGuzjD+EFIqLSYRweZjqfiPzQ06Ebg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.5",
+        "call-bound": "^1.0.3",
         "es-errors": "^1.3.0",
-        "get-intrinsic": "^1.2.4"
+        "get-intrinsic": "^1.2.6"
       },
       "engines": {
         "node": ">= 0.4"
@@ -2020,13 +2121,13 @@
       }
     },
     "node_modules/gopd": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
-      "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
+      "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==",
       "dev": true,
       "license": "MIT",
-      "dependencies": {
-        "get-intrinsic": "^1.1.3"
+      "engines": {
+        "node": ">= 0.4"
       },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
@@ -2047,11 +2148,14 @@
       "license": "MIT"
     },
     "node_modules/has-bigints": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz",
-      "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.1.0.tgz",
+      "integrity": "sha512-R3pbpkcIqv2Pm3dUwgjclDRVmWpTJW2DcMzcIhEXEx1oh/CEMObMm3KLmRJOdvhM7o4uQBnwr8pzRK2sJWIqfg==",
       "dev": true,
       "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
@@ -2080,11 +2184,14 @@
       }
     },
     "node_modules/has-proto": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz",
-      "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==",
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.2.0.tgz",
+      "integrity": "sha512-KIL7eQPfHQRC8+XluaIw7BHUwwqL19bQn4hzNgdr+1wXoU0KKj6rufu47lhY7KbJR2C6T6+PfyN0Ea7wkSS+qQ==",
       "dev": true,
       "license": "MIT",
+      "dependencies": {
+        "dunder-proto": "^1.0.0"
+      },
       "engines": {
         "node": ">= 0.4"
       },
@@ -2093,9 +2200,9 @@
       }
     },
     "node_modules/has-symbols": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
-      "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
+      "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==",
       "dev": true,
       "license": "MIT",
       "engines": {
@@ -2145,9 +2252,9 @@
       }
     },
     "node_modules/import-fresh": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz",
-      "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==",
+      "version": "3.3.1",
+      "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz",
+      "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -2201,15 +2308,15 @@
       }
     },
     "node_modules/internal-slot": {
-      "version": "1.0.7",
-      "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.7.tgz",
-      "integrity": "sha512-NGnrKwXzSms2qUUih/ILZ5JBqNTSa1+ZmP6flaIp6KmSElgE9qdndzS3cqjrDovwFdmwsGsLdeFgB6suw+1e9g==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.1.0.tgz",
+      "integrity": "sha512-4gd7VpWNQNB4UKKCFFVcp1AVv+FMOgs9NKzjHKusc8jTMhd5eL1NqQqOpE0KzMds804/yHlglp3uxgluOqAPLw==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "es-errors": "^1.3.0",
-        "hasown": "^2.0.0",
-        "side-channel": "^1.0.4"
+        "hasown": "^2.0.2",
+        "side-channel": "^1.1.0"
       },
       "engines": {
         "node": ">= 0.4"
@@ -2242,14 +2349,15 @@
       }
     },
     "node_modules/is-array-buffer": {
-      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.4.tgz",
-      "integrity": "sha512-wcjaerHw0ydZwfhiKbXJWLDY8A7yV7KhjQOpb83hGgGfId/aQa4TOvwyzn2PuswW2gPCYEL/nEAiSVpdOj1lXw==",
+      "version": "3.0.5",
+      "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.5.tgz",
+      "integrity": "sha512-DDfANUiiG2wC1qawP66qlTugJeL5HyzMpfr8lLK+jMQirGzNod0B12cFB/9q838Ru27sBwfw78/rdoU7RERz6A==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.2",
-        "get-intrinsic": "^1.2.1"
+        "call-bind": "^1.0.8",
+        "call-bound": "^1.0.3",
+        "get-intrinsic": "^1.2.6"
       },
       "engines": {
         "node": ">= 0.4"
@@ -2266,13 +2374,17 @@
       "license": "MIT"
     },
     "node_modules/is-async-function": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/is-async-function/-/is-async-function-2.0.0.tgz",
-      "integrity": "sha512-Y1JXKrfykRJGdlDwdKlLpLyMIiWqWvuSd17TvZk68PLAOGOoF4Xyav1z0Xhoi+gCYjZVeC5SI+hYFOfvXmGRCA==",
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/is-async-function/-/is-async-function-2.1.1.tgz",
+      "integrity": "sha512-9dgM/cZBnNvjzaMYHVoxxfPj2QXt22Ev7SuuPrs+xav0ukGB0S6d4ydZdEiM48kLx5kDV+QBPrpVnFyefL8kkQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "has-tostringtag": "^1.0.0"
+        "async-function": "^1.0.0",
+        "call-bound": "^1.0.3",
+        "get-proto": "^1.0.1",
+        "has-tostringtag": "^1.0.2",
+        "safe-regex-test": "^1.1.0"
       },
       "engines": {
         "node": ">= 0.4"
@@ -2282,27 +2394,30 @@
       }
     },
     "node_modules/is-bigint": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz",
-      "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.1.0.tgz",
+      "integrity": "sha512-n4ZT37wG78iz03xPRKJrHTdZbe3IicyucEtdRsV5yglwc3GyUfbAfpSeD0FJ41NbUNSt5wbhqfp1fS+BgnvDFQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "has-bigints": "^1.0.1"
+        "has-bigints": "^1.0.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
       },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
     },
     "node_modules/is-boolean-object": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz",
-      "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==",
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.2.2.tgz",
+      "integrity": "sha512-wa56o2/ElJMYqjCjGkXri7it5FbebW5usLw/nPmCMs5DeZ7eziSYZhSmPRn0txqeW4LnAmQQU7FgqLpsEFKM4A==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.2",
-        "has-tostringtag": "^1.0.0"
+        "call-bound": "^1.0.3",
+        "has-tostringtag": "^1.0.2"
       },
       "engines": {
         "node": ">= 0.4"
@@ -2325,9 +2440,9 @@
       }
     },
     "node_modules/is-core-module": {
-      "version": "2.15.1",
-      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.15.1.tgz",
-      "integrity": "sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ==",
+      "version": "2.16.1",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz",
+      "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -2341,12 +2456,14 @@
       }
     },
     "node_modules/is-data-view": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/is-data-view/-/is-data-view-1.0.1.tgz",
-      "integrity": "sha512-AHkaJrsUVW6wq6JS8y3JnM/GJF/9cf+k20+iDzlSaJrinEo5+7vRiteOSwBhHRiAyQATN1AmY4hwzxJKPmYf+w==",
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/is-data-view/-/is-data-view-1.0.2.tgz",
+      "integrity": "sha512-RKtWF8pGmS87i2D6gqQu/l7EYRlVdfzemCJN/P3UOs//x1QE7mfhvzHIApBTRf7axvT6DMGwSwBXYCT0nfB9xw==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
+        "call-bound": "^1.0.2",
+        "get-intrinsic": "^1.2.6",
         "is-typed-array": "^1.1.13"
       },
       "engines": {
@@ -2357,13 +2474,14 @@
       }
     },
     "node_modules/is-date-object": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz",
-      "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.1.0.tgz",
+      "integrity": "sha512-PwwhEakHVKTdRNVOw+/Gyh0+MzlCl4R6qKvkhuvLtPMggI1WAHt9sOwZxQLSGpUaDnrdyDsomoRgNnCfKNSXXg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "has-tostringtag": "^1.0.0"
+        "call-bound": "^1.0.2",
+        "has-tostringtag": "^1.0.2"
       },
       "engines": {
         "node": ">= 0.4"
@@ -2394,13 +2512,16 @@
       }
     },
     "node_modules/is-finalizationregistry": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/is-finalizationregistry/-/is-finalizationregistry-1.0.2.tgz",
-      "integrity": "sha512-0by5vtUJs8iFQb5TYUHHPudOR+qXYIMKtiUzvLIZITZUjknFmziyBJuLhVRc+Ds0dREFlskDNJKYIdIzu/9pfw==",
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/is-finalizationregistry/-/is-finalizationregistry-1.1.1.tgz",
+      "integrity": "sha512-1pC6N8qWJbWoPtEjgcL2xyhQOP491EQjeUo3qTKcmV8YSDDJrOepfG8pcC7h/QgnQHYSv0mJ3Z/ZWxmatVrysg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.2"
+        "call-bound": "^1.0.3"
+      },
+      "engines": {
+        "node": ">= 0.4"
       },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
@@ -2417,13 +2538,16 @@
       }
     },
     "node_modules/is-generator-function": {
-      "version": "1.0.10",
-      "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz",
-      "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.1.0.tgz",
+      "integrity": "sha512-nPUB5km40q9e8UfN/Zc24eLlzdSf9OfKByBw9CIdw4H1giPMeA0OIJvbchsCu4npfI2QcMVBsGEBHKZ7wLTWmQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "has-tostringtag": "^1.0.0"
+        "call-bound": "^1.0.3",
+        "get-proto": "^1.0.0",
+        "has-tostringtag": "^1.0.2",
+        "safe-regex-test": "^1.1.0"
       },
       "engines": {
         "node": ">= 0.4"
@@ -2469,27 +2593,15 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/is-negative-zero": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.3.tgz",
-      "integrity": "sha512-5KoIu2Ngpyek75jXodFvnafB6DJgr3u8uuK0LEZJjrU19DrMD3EVERaR8sjz8CCGgpZvxPl9SuE1GMVPFHx1mw==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
     "node_modules/is-number-object": {
-      "version": "1.0.7",
-      "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz",
-      "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==",
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.1.1.tgz",
+      "integrity": "sha512-lZhclumE1G6VYD8VHe35wFaIif+CTy5SJIi5+3y4psDgWu4wPDoBhF8NxUOinEc7pHgiTsT6MaBb92rKhhD+Xw==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "has-tostringtag": "^1.0.0"
+        "call-bound": "^1.0.3",
+        "has-tostringtag": "^1.0.2"
       },
       "engines": {
         "node": ">= 0.4"
@@ -2509,14 +2621,16 @@
       }
     },
     "node_modules/is-regex": {
-      "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz",
-      "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==",
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz",
+      "integrity": "sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.2",
-        "has-tostringtag": "^1.0.0"
+        "call-bound": "^1.0.2",
+        "gopd": "^1.2.0",
+        "has-tostringtag": "^1.0.2",
+        "hasown": "^2.0.2"
       },
       "engines": {
         "node": ">= 0.4"
@@ -2539,13 +2653,13 @@
       }
     },
     "node_modules/is-shared-array-buffer": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.3.tgz",
-      "integrity": "sha512-nA2hv5XIhLR3uVzDDfCIknerhx8XUKnstuOERPNNIinXG7v9u+ohXF67vxm4TPTEPU6lm61ZkwP3c9PCB97rhg==",
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.4.tgz",
+      "integrity": "sha512-ISWac8drv4ZGfwKl5slpHG9OwPNty4jOWPRIhBpxOoD+hqITiwuipOQ2bNthAzwA3B4fIjO4Nln74N0S9byq8A==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.7"
+        "call-bound": "^1.0.3"
       },
       "engines": {
         "node": ">= 0.4"
@@ -2555,13 +2669,14 @@
       }
     },
     "node_modules/is-string": {
-      "version": "1.0.7",
-      "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz",
-      "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==",
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.1.1.tgz",
+      "integrity": "sha512-BtEeSsoaQjlSPBemMQIrY1MY0uM6vnS1g5fmufYOtnxLGUZM2178PKbhsk7Ffv58IX+ZtcvoGwccYsh0PglkAA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "has-tostringtag": "^1.0.0"
+        "call-bound": "^1.0.3",
+        "has-tostringtag": "^1.0.2"
       },
       "engines": {
         "node": ">= 0.4"
@@ -2571,13 +2686,15 @@
       }
     },
     "node_modules/is-symbol": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz",
-      "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==",
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.1.1.tgz",
+      "integrity": "sha512-9gGx6GTtCQM73BgmHQXfDmLtfjjTUDSyoxTCbp5WtoixAhfgsDirWIcVQ/IHpvI5Vgd5i/J5F7B9cN/WlVbC/w==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "has-symbols": "^1.0.2"
+        "call-bound": "^1.0.2",
+        "has-symbols": "^1.1.0",
+        "safe-regex-test": "^1.1.0"
       },
       "engines": {
         "node": ">= 0.4"
@@ -2587,13 +2704,13 @@
       }
     },
     "node_modules/is-typed-array": {
-      "version": "1.1.13",
-      "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.13.tgz",
-      "integrity": "sha512-uZ25/bUAlUY5fR4OKT4rZQEBrzQWYV9ZJYGGsUmEJ6thodVJ1HX64ePQ6Z0qPWP+m+Uq6e9UugrE38jeYsDSMw==",
+      "version": "1.1.15",
+      "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.15.tgz",
+      "integrity": "sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "which-typed-array": "^1.1.14"
+        "which-typed-array": "^1.1.16"
       },
       "engines": {
         "node": ">= 0.4"
@@ -2616,27 +2733,30 @@
       }
     },
     "node_modules/is-weakref": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz",
-      "integrity": "sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==",
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.1.1.tgz",
+      "integrity": "sha512-6i9mGWSlqzNMEqpCp93KwRS1uUOodk2OJ6b+sq7ZPDSy2WuI5NFIxp/254TytR8ftefexkWn5xNiHUNpPOfSew==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.2"
+        "call-bound": "^1.0.3"
+      },
+      "engines": {
+        "node": ">= 0.4"
       },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
     },
     "node_modules/is-weakset": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/is-weakset/-/is-weakset-2.0.3.tgz",
-      "integrity": "sha512-LvIm3/KWzS9oRFHugab7d+M/GcBXuXX5xZkzPmN+NxihdQlZUQ4dWuSV1xR/sq6upL1TJEDrfBgRepHFdBtSNQ==",
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/is-weakset/-/is-weakset-2.0.4.tgz",
+      "integrity": "sha512-mfcwb6IzQyOKTs84CQMrOwW4gQcaTOAWJ0zzJCl2WSPDrWk/OzDaImWFH3djXhb24g4eudZfLRozAvPGw4d9hQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.7",
-        "get-intrinsic": "^1.2.4"
+        "call-bound": "^1.0.3",
+        "get-intrinsic": "^1.2.6"
       },
       "engines": {
         "node": ">= 0.4"
@@ -2660,17 +2780,18 @@
       "license": "ISC"
     },
     "node_modules/iterator.prototype": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/iterator.prototype/-/iterator.prototype-1.1.3.tgz",
-      "integrity": "sha512-FW5iMbeQ6rBGm/oKgzq2aW4KvAGpxPzYES8N4g4xNXUKpL1mclMvOe+76AcLDTvD+Ze+sOpVhgdAQEKF4L9iGQ==",
+      "version": "1.1.5",
+      "resolved": "https://registry.npmjs.org/iterator.prototype/-/iterator.prototype-1.1.5.tgz",
+      "integrity": "sha512-H0dkQoCa3b2VEeKQBOxFph+JAbcrQdE7KC0UkqwpLmv2EC4P41QXP+rqo9wYodACiG5/WM5s9oDApTU8utwj9g==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "define-properties": "^1.2.1",
-        "get-intrinsic": "^1.2.1",
-        "has-symbols": "^1.0.3",
-        "reflect.getprototypeof": "^1.0.4",
-        "set-function-name": "^2.0.1"
+        "define-data-property": "^1.1.4",
+        "es-object-atoms": "^1.0.0",
+        "get-intrinsic": "^1.2.6",
+        "get-proto": "^1.0.0",
+        "has-symbols": "^1.1.0",
+        "set-function-name": "^2.0.2"
       },
       "engines": {
         "node": ">= 0.4"
@@ -2984,6 +3105,16 @@
         "node": ">=18"
       }
     },
+    "node_modules/math-intrinsics": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
+      "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/mdurl": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz",
@@ -3471,9 +3602,9 @@
       }
     },
     "node_modules/micromark-util-subtokenize": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.0.4.tgz",
-      "integrity": "sha512-N6hXjrin2GTJDe3MVjf5FuXpm12PGm80BrUAeub9XFXca8JZbP+oIwY4LJSVwFUCL1IPm/WwSVUN7goFHmSGGQ==",
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.1.0.tgz",
+      "integrity": "sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA==",
       "dev": true,
       "funding": [
         {
@@ -3588,9 +3719,9 @@
       }
     },
     "node_modules/object-inspect": {
-      "version": "1.13.3",
-      "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.3.tgz",
-      "integrity": "sha512-kDCGIbxkDSXE3euJZZXzc6to7fCrKHNI/hSRQnRuQ+BWjFNzZwiFF8fj/6o2t2G9/jTj8PSIYTfCLelLZEeRpA==",
+      "version": "1.13.4",
+      "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz",
+      "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==",
       "dev": true,
       "license": "MIT",
       "engines": {
@@ -3611,15 +3742,17 @@
       }
     },
     "node_modules/object.assign": {
-      "version": "4.1.5",
-      "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.5.tgz",
-      "integrity": "sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ==",
+      "version": "4.1.7",
+      "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.7.tgz",
+      "integrity": "sha512-nK28WOo+QIjBkDduTINE4JkF/UJJKyf2EJxvJKfblDpyg0Q+pkOHNTL0Qwy6NP6FhE/EnzV73BxxqcJaXY9anw==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.5",
+        "call-bind": "^1.0.8",
+        "call-bound": "^1.0.3",
         "define-properties": "^1.2.1",
-        "has-symbols": "^1.0.3",
+        "es-object-atoms": "^1.0.0",
+        "has-symbols": "^1.1.0",
         "object-keys": "^1.1.1"
       },
       "engines": {
@@ -3679,13 +3812,14 @@
       }
     },
     "node_modules/object.values": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.2.0.tgz",
-      "integrity": "sha512-yBYjY9QX2hnRmZHAjG/f13MzmBzxzYgQhFrke06TTyKY5zSTEqkOeukBzIdVA3j3ulu8Qa3MbVFShV7T2RmGtQ==",
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.2.1.tgz",
+      "integrity": "sha512-gXah6aZrcUxjWg2zR2MwouP2eHlCBzdV4pygudehaKXSGW4v2AsRQUK+lwwXhii6KFZcunEnmSUoYp5CXibxtA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.7",
+        "call-bind": "^1.0.8",
+        "call-bound": "^1.0.3",
         "define-properties": "^1.2.1",
         "es-object-atoms": "^1.0.0"
       },
@@ -3724,6 +3858,24 @@
         "node": ">= 0.8.0"
       }
     },
+    "node_modules/own-keys": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/own-keys/-/own-keys-1.0.1.tgz",
+      "integrity": "sha512-qFOyK5PjiWZd+QQIh+1jhdb9LpxTF0qs7Pm8o5QHYZ0M3vKqSqzsZaEB6oWlxZ+q2sJBMI/Ktgd2N5ZwQoRHfg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "get-intrinsic": "^1.2.6",
+        "object-keys": "^1.1.1",
+        "safe-push-apply": "^1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/p-limit": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
@@ -3965,9 +4117,9 @@
       }
     },
     "node_modules/possible-typed-array-names": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz",
-      "integrity": "sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz",
+      "integrity": "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==",
       "dev": true,
       "license": "MIT",
       "engines": {
@@ -4045,19 +4197,20 @@
       "license": "MIT"
     },
     "node_modules/reflect.getprototypeof": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.6.tgz",
-      "integrity": "sha512-fmfw4XgoDke3kdI6h4xcUz1dG8uaiv5q9gcEwLS4Pnth2kxT+GZ7YehS1JTMGBQmtV7Y4GFGbs2re2NqhdozUg==",
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.10.tgz",
+      "integrity": "sha512-00o4I+DVrefhv+nX0ulyi3biSHCPDe+yLv5o/p6d/UVlirijB8E16FtfwSAi4g3tcqrQ4lRAqQSoFEZJehYEcw==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.7",
+        "call-bind": "^1.0.8",
         "define-properties": "^1.2.1",
-        "es-abstract": "^1.23.1",
+        "es-abstract": "^1.23.9",
         "es-errors": "^1.3.0",
-        "get-intrinsic": "^1.2.4",
-        "globalthis": "^1.0.3",
-        "which-builtin-type": "^1.1.3"
+        "es-object-atoms": "^1.0.0",
+        "get-intrinsic": "^1.2.7",
+        "get-proto": "^1.0.1",
+        "which-builtin-type": "^1.2.1"
       },
       "engines": {
         "node": ">= 0.4"
@@ -4067,15 +4220,17 @@
       }
     },
     "node_modules/regexp.prototype.flags": {
-      "version": "1.5.3",
-      "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.3.tgz",
-      "integrity": "sha512-vqlC04+RQoFalODCbCumG2xIOvapzVMHwsyIGM/SIE8fRhFFsXeH8/QQ+s0T0kDAhKc4k30s73/0ydkHQz6HlQ==",
+      "version": "1.5.4",
+      "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz",
+      "integrity": "sha512-dYqgNSZbDwkaJ2ceRd9ojCGjBq+mOm9LmtXnAnEGyHhN/5R7iDW2TRw3h+o/jCFxus3P2LfWIIiwowAjANm7IA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.7",
+        "call-bind": "^1.0.8",
         "define-properties": "^1.2.1",
         "es-errors": "^1.3.0",
+        "get-proto": "^1.0.1",
+        "gopd": "^1.2.0",
         "set-function-name": "^2.0.2"
       },
       "engines": {
@@ -4099,19 +4254,22 @@
       }
     },
     "node_modules/resolve": {
-      "version": "1.22.8",
-      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz",
-      "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==",
+      "version": "1.22.10",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.10.tgz",
+      "integrity": "sha512-NPRy+/ncIMeDlTAsuqwKIiferiawhefFJtkNSW0qZJEqMEb+qBt/77B/jGeeek+F0uOeN05CDa6HXbbIgtVX4w==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "is-core-module": "^2.13.0",
+        "is-core-module": "^2.16.0",
         "path-parse": "^1.0.7",
         "supports-preserve-symlinks-flag": "^1.0.0"
       },
       "bin": {
         "resolve": "bin/resolve"
       },
+      "engines": {
+        "node": ">= 0.4"
+      },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
@@ -4127,9 +4285,9 @@
       }
     },
     "node_modules/reusify": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",
-      "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz",
+      "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==",
       "dev": true,
       "license": "MIT",
       "engines": {
@@ -4241,15 +4399,16 @@
       }
     },
     "node_modules/safe-array-concat": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.2.tgz",
-      "integrity": "sha512-vj6RsCsWBCf19jIeHEfkRMw8DPiBb+DMXklQ/1SGDHOMlHdPUkZXFQ2YdplS23zESTijAcurb1aSgJA3AgMu1Q==",
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.3.tgz",
+      "integrity": "sha512-AURm5f0jYEOydBj7VQlVvDrjeFgthDdEF5H1dP+6mNpoXOMo1quQqJ4wvJDyRZ9+pO3kGWoOdmV08cSv2aJV6Q==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.7",
-        "get-intrinsic": "^1.2.4",
-        "has-symbols": "^1.0.3",
+        "call-bind": "^1.0.8",
+        "call-bound": "^1.0.2",
+        "get-intrinsic": "^1.2.6",
+        "has-symbols": "^1.1.0",
         "isarray": "^2.0.5"
       },
       "engines": {
@@ -4259,16 +4418,33 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/safe-push-apply": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/safe-push-apply/-/safe-push-apply-1.0.0.tgz",
+      "integrity": "sha512-iKE9w/Z7xCzUMIZqdBsp6pEQvwuEebH4vdpjcDWnyzaI6yl6O9FHvVpmGelvEHNsoY6wGblkxR6Zty/h00WiSA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "es-errors": "^1.3.0",
+        "isarray": "^2.0.5"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/safe-regex-test": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.3.tgz",
-      "integrity": "sha512-CdASjNJPvRa7roO6Ra/gLYBTzYzzPyyBXxIMdGW3USQLyjWEls2RgW5UBTXaQVp+OrpeCK3bLem8smtmheoRuw==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.1.0.tgz",
+      "integrity": "sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.6",
+        "call-bound": "^1.0.2",
         "es-errors": "^1.3.0",
-        "is-regex": "^1.1.4"
+        "is-regex": "^1.2.1"
       },
       "engines": {
         "node": ">= 0.4"
@@ -4321,6 +4497,21 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/set-proto": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/set-proto/-/set-proto-1.0.0.tgz",
+      "integrity": "sha512-RJRdvCo6IAnPdsvP/7m6bsQqNnn1FCBX5ZNtFL98MmFF/4xAIJTIg1YbHW5DC2W5SKZanrC6i4HsJqlajw/dZw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "dunder-proto": "^1.0.1",
+        "es-errors": "^1.3.0",
+        "es-object-atoms": "^1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/shebang-command": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
@@ -4345,16 +4536,73 @@
       }
     },
     "node_modules/side-channel": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz",
-      "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz",
+      "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.7",
         "es-errors": "^1.3.0",
-        "get-intrinsic": "^1.2.4",
-        "object-inspect": "^1.13.1"
+        "object-inspect": "^1.13.3",
+        "side-channel-list": "^1.0.0",
+        "side-channel-map": "^1.0.1",
+        "side-channel-weakmap": "^1.0.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/side-channel-list": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz",
+      "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "es-errors": "^1.3.0",
+        "object-inspect": "^1.13.3"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/side-channel-map": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz",
+      "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "call-bound": "^1.0.2",
+        "es-errors": "^1.3.0",
+        "get-intrinsic": "^1.2.5",
+        "object-inspect": "^1.13.3"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/side-channel-weakmap": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz",
+      "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "call-bound": "^1.0.2",
+        "es-errors": "^1.3.0",
+        "get-intrinsic": "^1.2.5",
+        "object-inspect": "^1.13.3",
+        "side-channel-map": "^1.0.1"
       },
       "engines": {
         "node": ">= 0.4"
@@ -4381,6 +4629,7 @@
       "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.3.1.tgz",
       "integrity": "sha512-tEYNll18pPKHroYSmLLrksq233j021G0giwW7P3D24jC54pQ5W5BXMsQ/Mvw1OJCmEYDgY+lrzT+3nNUtoNfXQ==",
       "dev": true,
+      "license": "BSD-3-Clause",
       "engines": {
         "node": ">= 18"
       },
@@ -4456,19 +4705,6 @@
         "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
       }
     },
-    "node_modules/standard-engine/node_modules/get-stdin": {
-      "version": "8.0.0",
-      "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-8.0.0.tgz",
-      "integrity": "sha512-sY22aA6xchAzprjyqmSEQv4UbAAzRN0L2dQB0NlN5acTTK9Don6nhoc3eAbUnpZiCANAMfd/+40kVdKfFygohg==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
     "node_modules/string-width": {
       "version": "5.1.2",
       "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz",
@@ -4534,24 +4770,25 @@
       }
     },
     "node_modules/string.prototype.matchall": {
-      "version": "4.0.11",
-      "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.11.tgz",
-      "integrity": "sha512-NUdh0aDavY2og7IbBPenWqR9exH+E26Sv8e0/eTe1tltDGZL+GtBkDAnnyBtmekfK6/Dq3MkcGtzXFEd1LQrtg==",
+      "version": "4.0.12",
+      "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.12.tgz",
+      "integrity": "sha512-6CC9uyBL+/48dYizRf7H7VAYCMCNTBeM78x/VTUe9bFEaxBepPJDa1Ow99LqI/1yF7kuy7Q3cQsYMrcjGUcskA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.7",
+        "call-bind": "^1.0.8",
+        "call-bound": "^1.0.3",
         "define-properties": "^1.2.1",
-        "es-abstract": "^1.23.2",
+        "es-abstract": "^1.23.6",
         "es-errors": "^1.3.0",
         "es-object-atoms": "^1.0.0",
-        "get-intrinsic": "^1.2.4",
-        "gopd": "^1.0.1",
-        "has-symbols": "^1.0.3",
-        "internal-slot": "^1.0.7",
-        "regexp.prototype.flags": "^1.5.2",
+        "get-intrinsic": "^1.2.6",
+        "gopd": "^1.2.0",
+        "has-symbols": "^1.1.0",
+        "internal-slot": "^1.1.0",
+        "regexp.prototype.flags": "^1.5.3",
         "set-function-name": "^2.0.2",
-        "side-channel": "^1.0.6"
+        "side-channel": "^1.1.0"
       },
       "engines": {
         "node": ">= 0.4"
@@ -4572,16 +4809,19 @@
       }
     },
     "node_modules/string.prototype.trim": {
-      "version": "1.2.9",
-      "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.9.tgz",
-      "integrity": "sha512-klHuCNxiMZ8MlsOihJhJEBJAiMVqU3Z2nEXWfWnIqjN0gEFS9J9+IxKozWWtQGcgoa1WUZzLjKPTr4ZHNFTFxw==",
+      "version": "1.2.10",
+      "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.10.tgz",
+      "integrity": "sha512-Rs66F0P/1kedk5lyYyH9uBzuiI/kNRmwJAR9quK6VOtIpZ2G+hMZd+HQbbv25MgCA6gEffoMZYxlTod4WcdrKA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.7",
+        "call-bind": "^1.0.8",
+        "call-bound": "^1.0.2",
+        "define-data-property": "^1.1.4",
         "define-properties": "^1.2.1",
-        "es-abstract": "^1.23.0",
-        "es-object-atoms": "^1.0.0"
+        "es-abstract": "^1.23.5",
+        "es-object-atoms": "^1.0.0",
+        "has-property-descriptors": "^1.0.2"
       },
       "engines": {
         "node": ">= 0.4"
@@ -4591,16 +4831,20 @@
       }
     },
     "node_modules/string.prototype.trimend": {
-      "version": "1.0.8",
-      "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.8.tgz",
-      "integrity": "sha512-p73uL5VCHCO2BZZ6krwwQE3kCzM7NKmis8S//xEC6fQonchbum4eP6kR4DLEjQFO3Wnj3Fuo8NM0kOSjVdHjZQ==",
+      "version": "1.0.9",
+      "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.9.tgz",
+      "integrity": "sha512-G7Ok5C6E/j4SGfyLCloXTrngQIQU3PWtXGst3yM7Bea9FRURf1S42ZHlZZtsNque2FN2PoUhfZXYLNWwEr4dLQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.7",
+        "call-bind": "^1.0.8",
+        "call-bound": "^1.0.2",
         "define-properties": "^1.2.1",
         "es-object-atoms": "^1.0.0"
       },
+      "engines": {
+        "node": ">= 0.4"
+      },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
@@ -4759,32 +5003,32 @@
       }
     },
     "node_modules/typed-array-buffer": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.2.tgz",
-      "integrity": "sha512-gEymJYKZtKXzzBzM4jqa9w6Q1Jjm7x2d+sh19AdsD4wqnMPDYyvwpsIc2Q/835kHuo3BEQ7CjelGhfTsoBb2MQ==",
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz",
+      "integrity": "sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.7",
+        "call-bound": "^1.0.3",
         "es-errors": "^1.3.0",
-        "is-typed-array": "^1.1.13"
+        "is-typed-array": "^1.1.14"
       },
       "engines": {
         "node": ">= 0.4"
       }
     },
     "node_modules/typed-array-byte-length": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/typed-array-byte-length/-/typed-array-byte-length-1.0.1.tgz",
-      "integrity": "sha512-3iMJ9q0ao7WE9tWcaYKIptkNBuOIcZCCT0d4MRvuuH88fEoEH62IuQe0OtraD3ebQEoTRk8XCBoknUNc1Y67pw==",
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/typed-array-byte-length/-/typed-array-byte-length-1.0.3.tgz",
+      "integrity": "sha512-BaXgOuIxz8n8pIq3e7Atg/7s+DpiYrxn4vdot3w9KbnBhcRQq6o3xemQdIfynqSeXeDrF32x+WvfzmOjPiY9lg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.7",
+        "call-bind": "^1.0.8",
         "for-each": "^0.3.3",
-        "gopd": "^1.0.1",
-        "has-proto": "^1.0.3",
-        "is-typed-array": "^1.1.13"
+        "gopd": "^1.2.0",
+        "has-proto": "^1.2.0",
+        "is-typed-array": "^1.1.14"
       },
       "engines": {
         "node": ">= 0.4"
@@ -4794,18 +5038,19 @@
       }
     },
     "node_modules/typed-array-byte-offset": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/typed-array-byte-offset/-/typed-array-byte-offset-1.0.2.tgz",
-      "integrity": "sha512-Ous0vodHa56FviZucS2E63zkgtgrACj7omjwd/8lTEMEPFFyjfixMZ1ZXenpgCFBBt4EC1J2XsyVS2gkG0eTFA==",
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/typed-array-byte-offset/-/typed-array-byte-offset-1.0.4.tgz",
+      "integrity": "sha512-bTlAFB/FBYMcuX81gbL4OcpH5PmlFHqlCCpAl8AlEzMz5k53oNDvN8p1PNOWLEmI2x4orp3raOFB51tv9X+MFQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "available-typed-arrays": "^1.0.7",
-        "call-bind": "^1.0.7",
+        "call-bind": "^1.0.8",
         "for-each": "^0.3.3",
-        "gopd": "^1.0.1",
-        "has-proto": "^1.0.3",
-        "is-typed-array": "^1.1.13"
+        "gopd": "^1.2.0",
+        "has-proto": "^1.2.0",
+        "is-typed-array": "^1.1.15",
+        "reflect.getprototypeof": "^1.0.9"
       },
       "engines": {
         "node": ">= 0.4"
@@ -4815,18 +5060,18 @@
       }
     },
     "node_modules/typed-array-length": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/typed-array-length/-/typed-array-length-1.0.6.tgz",
-      "integrity": "sha512-/OxDN6OtAk5KBpGb28T+HZc2M+ADtvRxXrKKbUwtsLgdoxgX13hyy7ek6bFRl5+aBs2yZzB0c4CnQfAtVypW/g==",
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/typed-array-length/-/typed-array-length-1.0.7.tgz",
+      "integrity": "sha512-3KS2b+kL7fsuk/eJZ7EQdnEmQoaho/r6KUef7hxvltNA5DR8NAUM+8wJMbJyZ4G9/7i3v5zPBIMN5aybAh2/Jg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "call-bind": "^1.0.7",
         "for-each": "^0.3.3",
         "gopd": "^1.0.1",
-        "has-proto": "^1.0.3",
         "is-typed-array": "^1.1.13",
-        "possible-typed-array-names": "^1.0.0"
+        "possible-typed-array-names": "^1.0.0",
+        "reflect.getprototypeof": "^1.0.6"
       },
       "engines": {
         "node": ">= 0.4"
@@ -4843,16 +5088,19 @@
       "license": "MIT"
     },
     "node_modules/unbox-primitive": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz",
-      "integrity": "sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.1.0.tgz",
+      "integrity": "sha512-nWJ91DjeOkej/TA8pXQ3myruKpKEYgqvpw9lz4OPHj/NWFNluYrjbz9j01CJ8yKQd2g4jFoOkINCTW2I5LEEyw==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "call-bind": "^1.0.2",
+        "call-bound": "^1.0.3",
         "has-bigints": "^1.0.2",
-        "has-symbols": "^1.0.3",
-        "which-boxed-primitive": "^1.0.2"
+        "has-symbols": "^1.1.0",
+        "which-boxed-primitive": "^1.1.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
       },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
@@ -4895,41 +5143,45 @@
       }
     },
     "node_modules/which-boxed-primitive": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz",
-      "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==",
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.1.1.tgz",
+      "integrity": "sha512-TbX3mj8n0odCBFVlY8AxkqcHASw3L60jIuF8jFP78az3C2YhmGvqbHBpAjTRH2/xqYunrJ9g1jSyjCjpoWzIAA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "is-bigint": "^1.0.1",
-        "is-boolean-object": "^1.1.0",
-        "is-number-object": "^1.0.4",
-        "is-string": "^1.0.5",
-        "is-symbol": "^1.0.3"
+        "is-bigint": "^1.1.0",
+        "is-boolean-object": "^1.2.1",
+        "is-number-object": "^1.1.1",
+        "is-string": "^1.1.1",
+        "is-symbol": "^1.1.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
       },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
     },
     "node_modules/which-builtin-type": {
-      "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/which-builtin-type/-/which-builtin-type-1.1.4.tgz",
-      "integrity": "sha512-bppkmBSsHFmIMSl8BO9TbsyzsvGjVoppt8xUiGzwiu/bhDCGxnpOKCxgqj6GuyHE0mINMDecBFPlOm2hzY084w==",
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/which-builtin-type/-/which-builtin-type-1.2.1.tgz",
+      "integrity": "sha512-6iBczoX+kDQ7a3+YJBnh3T+KZRxM/iYNPXicqk66/Qfm1b93iu+yOImkg0zHbj5LNOcNv1TEADiZ0xa34B4q6Q==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
+        "call-bound": "^1.0.2",
         "function.prototype.name": "^1.1.6",
         "has-tostringtag": "^1.0.2",
         "is-async-function": "^2.0.0",
-        "is-date-object": "^1.0.5",
-        "is-finalizationregistry": "^1.0.2",
+        "is-date-object": "^1.1.0",
+        "is-finalizationregistry": "^1.1.0",
         "is-generator-function": "^1.0.10",
-        "is-regex": "^1.1.4",
+        "is-regex": "^1.2.1",
         "is-weakref": "^1.0.2",
         "isarray": "^2.0.5",
-        "which-boxed-primitive": "^1.0.2",
+        "which-boxed-primitive": "^1.1.0",
         "which-collection": "^1.0.2",
-        "which-typed-array": "^1.1.15"
+        "which-typed-array": "^1.1.16"
       },
       "engines": {
         "node": ">= 0.4"
@@ -4958,16 +5210,18 @@
       }
     },
     "node_modules/which-typed-array": {
-      "version": "1.1.15",
-      "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.15.tgz",
-      "integrity": "sha512-oV0jmFtUky6CXfkqehVvBP/LSWJ2sy4vWMioiENyJLePrBO/yKyV9OyJySfAKosh+RYkIl5zJCNZ8/4JncrpdA==",
+      "version": "1.1.19",
+      "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.19.tgz",
+      "integrity": "sha512-rEvr90Bck4WZt9HHFC4DJMsjvu7x+r6bImz0/BrbWb7A2djJ8hnZMrWnHo9F8ssv0OMErasDhftrfROTyqSDrw==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "available-typed-arrays": "^1.0.7",
-        "call-bind": "^1.0.7",
-        "for-each": "^0.3.3",
-        "gopd": "^1.0.1",
+        "call-bind": "^1.0.8",
+        "call-bound": "^1.0.4",
+        "for-each": "^0.3.5",
+        "get-proto": "^1.0.1",
+        "gopd": "^1.2.0",
         "has-tostringtag": "^1.0.2"
       },
       "engines": {
@@ -5111,10 +5365,6 @@
       "funding": {
         "url": "https://github.com/sponsors/sindresorhus"
       }
-    },
-    "reports": {
-      "name": "httparchive_reports",
-      "extraneous": true
     }
   }
 }

From 57ca8039fe112e025cd22ace2630b99445510193 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <max@httparchive.org>
Date: Thu, 13 Mar 2025 23:16:04 +0000
Subject: [PATCH 33/52] test

---
 package-lock.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package-lock.json b/package-lock.json
index 4154fb02..32b4ac56 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -6,7 +6,7 @@
     "": {
       "name": "crawl-data",
       "dependencies": {
-        "@dataform/core": "3.0.15"
+        "@dataform/core": "^3.0.15"
       },
       "devDependencies": {
         "markdownlint-cli": "^0.44.0",

From e5aaf33b5486de1b6c4c6d2842ceaaca537c427e Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Fri, 14 Mar 2025 00:18:03 +0100
Subject: [PATCH 34/52] packages update

---
 package-lock.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package-lock.json b/package-lock.json
index f76c7576..86619c43 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -6,7 +6,7 @@
     "": {
       "name": "crawl-data",
       "dependencies": {
-        "@dataform/core": "^3.0.15"
+        "@dataform/core": "3.0.15"
       },
       "devDependencies": {
         "markdownlint-cli": "0.44.0",

From 90add5bcc8c58055eee5fdacad92148bf1e0de8a Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Fri, 14 Mar 2025 00:38:54 +0100
Subject: [PATCH 35/52] arguments renamed

---
 infra/bigquery-export/firestore.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/infra/bigquery-export/firestore.js b/infra/bigquery-export/firestore.js
index bde22a8e..7ae4e457 100644
--- a/infra/bigquery-export/firestore.js
+++ b/infra/bigquery-export/firestore.js
@@ -126,12 +126,12 @@ export class FirestoreBatch {
   }
 
   async export (exportConfig, query) {
-    this.date = exportConfig.date
-    this.collectionName = exportConfig.name
-    this.collectionType = exportConfig.type
     this.firestore.settings({
-      databaseId: 'tech-report-apis-' + exportConfig.environment
+      databaseId: exportConfig.database
     })
+    this.collectionName = exportConfig.collection
+    this.collectionType = exportConfig.type
+    this.date = exportConfig.date
 
     await this.batchDelete()
 

From a1529fa5bd2100e22a244ee8ec6e98ace5977e6a Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Fri, 14 Mar 2025 00:43:18 +0100
Subject: [PATCH 36/52] current nodejs

---
 infra/bigquery-export/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/infra/bigquery-export/Dockerfile b/infra/bigquery-export/Dockerfile
index 47f13182..a50c671b 100644
--- a/infra/bigquery-export/Dockerfile
+++ b/infra/bigquery-export/Dockerfile
@@ -1,5 +1,5 @@
 # checkov:skip=CKV_DOCKER_3:Ensure that a user for the container has been created
-FROM node:20-slim
+FROM node:current-slim
 
 WORKDIR /usr/src/app
 

From 0a38251c01be804f4a6b581747840671ee218f49 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Fri, 14 Mar 2025 07:44:45 +0100
Subject: [PATCH 37/52] Update
 definitions/output/reports/tech_report_technologies.js

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
---
 definitions/output/reports/tech_report_technologies.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/definitions/output/reports/tech_report_technologies.js b/definitions/output/reports/tech_report_technologies.js
index 591daad1..0421376a 100644
--- a/definitions/output/reports/tech_report_technologies.js
+++ b/definitions/output/reports/tech_report_technologies.js
@@ -91,7 +91,7 @@ FROM total_pages
       JSON '''{
         "destination": "firestore",
         "config": {
-          "database": "tech-report-api-${constants.environment}"
+          "database": "tech-report-api-${constants.environment}",
           "collection": "technologies",
           "type": "dict"
         },

From fcf1672a056665f29a748f396017916c892d8fab Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Mon, 17 Mar 2025 08:06:00 +0100
Subject: [PATCH 38/52] remove reservation

---
 definitions/declarations/httparchive.js | 18 ------------------
 definitions/output/crawl/pages.js       |  3 +--
 definitions/output/crawl/parsed_css.js  |  3 +--
 definitions/output/crawl/requests.js    |  3 +--
 4 files changed, 3 insertions(+), 24 deletions(-)

diff --git a/definitions/declarations/httparchive.js b/definitions/declarations/httparchive.js
index f7b2e86d..aa6d183d 100644
--- a/definitions/declarations/httparchive.js
+++ b/definitions/declarations/httparchive.js
@@ -40,21 +40,3 @@ ORDER BY cnt_pages DESC
     name: table
   })
 )
-
-operate('create_reservation_assignment')
-  // .tags(['crawl_complete'])
-  .queries(ctx => `
-CREATE ASSIGNMENT
-\`httparchive.region-us.retrospective-reprocessing.pipeline\`
-OPTIONS (
-  assignee = 'projects/httparchive',
-  job_type = 'QUERY')
-`)
-
-operate('drop_reservation_assignment')
-  // .dependencies(['requests_10k'])
-  // .tags(['crawl_complete'])
-  .queries(ctx => `
-DROP ASSIGNMENT IF EXISTS
-\`httparchive.region-us.retrospective-reprocessing.pipeline\`
-`)
diff --git a/definitions/output/crawl/pages.js b/definitions/output/crawl/pages.js
index bb4a3009..098521e6 100644
--- a/definitions/output/crawl/pages.js
+++ b/definitions/output/crawl/pages.js
@@ -47,8 +47,7 @@ publish('pages', {
     technologies: 'Technologies detected at runtime (see https://www.wappalyzer.com/)',
     metadata: 'Additional metadata about the test'
   },
-  tags: ['crawl_complete'],
-  dependencies: ['create_reservation_assignment']
+  tags: ['crawl_complete']
 }).preOps(ctx => `
 DELETE FROM ${ctx.self()}
 WHERE date = '${constants.currentMonth}' AND
diff --git a/definitions/output/crawl/parsed_css.js b/definitions/output/crawl/parsed_css.js
index b3fbaefb..529bbe06 100644
--- a/definitions/output/crawl/parsed_css.js
+++ b/definitions/output/crawl/parsed_css.js
@@ -7,8 +7,7 @@ publish('parsed_css', {
     clusterBy: ['client', 'is_root_page', 'rank', 'page'],
     requirePartitionFilter: true
   },
-  tags: ['crawl_complete'],
-  dependencies: ['create_reservation_assignment']
+  tags: ['crawl_complete']
 }).preOps(ctx => `
 DELETE FROM ${ctx.self()}
 WHERE date = '${constants.currentMonth}'
diff --git a/definitions/output/crawl/requests.js b/definitions/output/crawl/requests.js
index d7a57993..5a610c88 100644
--- a/definitions/output/crawl/requests.js
+++ b/definitions/output/crawl/requests.js
@@ -36,8 +36,7 @@ publish('requests', {
     },
     response_body: 'Text-based response body'
   },
-  tags: ['crawl_complete'],
-  dependencies: ['create_reservation_assignment']
+  tags: ['crawl_complete']
 }).preOps(ctx => `
 FOR client_value IN (SELECT * FROM UNNEST(['desktop', 'mobile']) AS client) DO
   FOR is_root_page_value IN (SELECT * FROM UNNEST([TRUE, FALSE]) AS is_root_page) DO

From 37fa3b2da338bdf6e109e327d8e8f7b3d081fdeb Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Fri, 11 Apr 2025 01:52:21 +0200
Subject: [PATCH 39/52] switch order

---
 infra/bigquery-export/firestore.js | 2 +-
 infra/bigquery-export/index.js     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/infra/bigquery-export/firestore.js b/infra/bigquery-export/firestore.js
index 7ae4e457..18d8f51e 100644
--- a/infra/bigquery-export/firestore.js
+++ b/infra/bigquery-export/firestore.js
@@ -125,7 +125,7 @@ export class FirestoreBatch {
     console.info(`Transfer to ${this.collectionName} complete. Total rows processed: ${totalRowsProcessed}. Time: ${duration} seconds`)
   }
 
-  async export (exportConfig, query) {
+  async export (query, exportConfig) {
     this.firestore.settings({
       databaseId: exportConfig.database
     })
diff --git a/infra/bigquery-export/index.js b/infra/bigquery-export/index.js
index b38e442d..fe301804 100644
--- a/infra/bigquery-export/index.js
+++ b/infra/bigquery-export/index.js
@@ -18,7 +18,7 @@ async function main () {
     console.log(query, config)
 
     const firestore = new FirestoreBatch()
-    await firestore.export(config, query)
+    await firestore.export(query, config)
   } else {
     throw new Error('Bad Request: destination unknown')
   }

From 8227d9bc47bdae6b2f74100d1f25779d4e9f758d Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Mon, 14 Apr 2025 23:24:29 +0200
Subject: [PATCH 40/52] deactivate spark procedure

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 87599039..794f7389 100644
--- a/Makefile
+++ b/Makefile
@@ -15,5 +15,5 @@ tf_apply:
 bigquery_export_deploy:
 	cd infra/bigquery-export && npm install && npm run buildpack
 
-bigquery_export_spark_deploy:
-	cd infra/bigquery_export_spark && gcloud builds submit --region=global --tag us-docker.pkg.dev/httparchive/bigquery-spark-procedures/firestore_export:latest
+#bigquery_export_spark_deploy:
+#	cd infra/bigquery_export_spark && gcloud builds submit --region=global --tag us-docker.pkg.dev/httparchive/bigquery-spark-procedures/firestore_export:latest

From cd482ba53a7c90e44f95d4a690a71a4c3f357fe1 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Mon, 14 Apr 2025 23:24:44 +0200
Subject: [PATCH 41/52] deactivate standard exports

---
 definitions/output/reports/reports_dynamic.js | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js
index be0635d5..6fa85e75 100644
--- a/definitions/output/reports/reports_dynamic.js
+++ b/definitions/output/reports/reports_dynamic.js
@@ -19,7 +19,7 @@ if (iterations.length === 1) {
         protected: true,
         bigquery: sql.type === 'histogram' ? { partitionBy: 'date', clusterBy: ['client'] } : {},
         schema: 'reports',
-        tags: ['crawl_complete', 'http_reports']
+        //tags: ['crawl_complete', 'http_reports']
       }).preOps(ctx => `
 --DELETE FROM ${ctx.self()}
 --WHERE date = '${params.date}';
@@ -45,21 +45,22 @@ SELECT
     metrics.forEach(metric => {
       metric.SQL.forEach(sql => {
         operate(metric.id + '_' + sql.type + '_' + params.date, {
-          tags: ['crawl_complete']
+          //tags: ['crawl_complete']
         }).queries(ctx => `
 DELETE FROM reports.${metric.id}_${sql.type}
 WHERE date = '${params.date}';
 
 INSERT INTO reports.${metric.id}_${sql.type}` + sql.query(ctx, params)
         ).postOps(ctx => `
-SELECT
+  SELECT
   reports.run_export_job(
     JSON '''{
-      "dataform_trigger": "report_complete",
-      "date": "${params.date}",
-      "name": "${metric.id}",
-      "type": "${sql.type}",
-      "environment": "${constants.environment}"
+      "destination": "cloud_storage",
+      "config": {
+        "bucket": "httparchive",
+        "name": "reports/${constants.environment}/${metric.id}_${sql.type}_${params.date}.json"
+      },
+      "query": "SELECT FORMAT_DATE('%Y_%m_%d', date) AS date, * EXCEPT(date) FROM ${ctx.self()}"
     }'''
   );
         `)

From 3bdd444a8a9fa139eceb5dac5479d0412b446f72 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Mon, 14 Apr 2025 23:48:17 +0200
Subject: [PATCH 42/52] updated function calling

---
 .../output/reports/cwv_tech_adoption.js       | 17 ++++++++++++--
 .../output/reports/cwv_tech_categories.js     | 17 ++++++++++++--
 .../reports/cwv_tech_core_web_vitals.js       | 17 ++++++++++++--
 .../output/reports/cwv_tech_lighthouse.js     | 17 ++++++++++++--
 .../output/reports/cwv_tech_page_weight.js    | 17 ++++++++++++--
 .../output/reports/cwv_tech_technologies.js   | 16 +++++++++++--
 .../output/reports/tech_report_adoption.js    | 23 +++++++++++--------
 .../output/reports/tech_report_audits.js      | 15 +++++++-----
 .../output/reports/tech_report_categories.js  | 13 +++++++----
 .../reports/tech_report_core_web_vitals.js    | 15 +++++++-----
 .../output/reports/tech_report_lighthouse.js  | 15 +++++++-----
 .../output/reports/tech_report_page_weight.js | 15 +++++++-----
 .../reports/tech_report_technologies.js       |  8 +++----
 .../output/reports/tech_report_versions.js    | 13 +++++++----
 14 files changed, 158 insertions(+), 60 deletions(-)

diff --git a/definitions/output/reports/cwv_tech_adoption.js b/definitions/output/reports/cwv_tech_adoption.js
index 1b3e40c1..400d0a37 100644
--- a/definitions/output/reports/cwv_tech_adoption.js
+++ b/definitions/output/reports/cwv_tech_adoption.js
@@ -13,7 +13,6 @@ publish('cwv_tech_adoption', {
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';
 `).query(ctx => `
-/* {"dataform_trigger": "report_cwv_tech_complete", "date": "${pastMonth}", "name": "adoption", "type": "report"} */
 SELECT
   date,
   app AS technology,
@@ -30,4 +29,18 @@ GROUP BY
   app,
   rank,
   geo
-`)
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "destination": "firestore",
+        "config": {
+          "databaseId": "tech-report-apis-{constants.environment}",
+          "collectionName": "adoption",
+          "collectionType": "report",
+          "date": "${pastMonth}"
+        },
+        "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
+      }'''
+    );
+  `)
diff --git a/definitions/output/reports/cwv_tech_categories.js b/definitions/output/reports/cwv_tech_categories.js
index 3368c0b0..effd9cc9 100644
--- a/definitions/output/reports/cwv_tech_categories.js
+++ b/definitions/output/reports/cwv_tech_categories.js
@@ -5,7 +5,6 @@ publish('cwv_tech_categories', {
   type: 'table',
   tags: ['crux_ready']
 }).query(ctx => `
-/* {"dataform_trigger": "report_cwv_tech_complete", "name": "categories", "type": "dict"} */
 WITH pages AS (
   SELECT DISTINCT
     client,
@@ -91,4 +90,18 @@ SELECT
   ) AS origins,
   NULL AS technologies
 FROM total_pages
-`)
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "destination": "firestore",
+        "config": {
+          "databaseId": "tech-report-apis-{constants.environment}",
+          "collectionName": "categories",
+          "collectionType": "dict"
+        },
+        "query": "SELECT * FROM ${ctx.self()}"
+      }'''
+    );
+  `)
+
diff --git a/definitions/output/reports/cwv_tech_core_web_vitals.js b/definitions/output/reports/cwv_tech_core_web_vitals.js
index b19de991..4f5a4237 100644
--- a/definitions/output/reports/cwv_tech_core_web_vitals.js
+++ b/definitions/output/reports/cwv_tech_core_web_vitals.js
@@ -68,7 +68,6 @@ return Object.values(vitals)
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';
 `).query(ctx => `
-/* {"dataform_trigger": "report_cwv_tech_complete", "date": "${pastMonth}", "name": "core_web_vitals", "type": "report"} */
 SELECT
   date,
   app AS technology,
@@ -98,4 +97,18 @@ GROUP BY
   app,
   rank,
   geo
-`)
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "destination": "firestore",
+        "config": {
+          "databaseId": "tech-report-apis-{constants.environment}",
+          "collectionName": "core_web_vitals",
+          "collectionType": "report",
+          "date": "${pastMonth}"
+        },
+        "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
+      }'''
+    );
+  `)
diff --git a/definitions/output/reports/cwv_tech_lighthouse.js b/definitions/output/reports/cwv_tech_lighthouse.js
index bba7b2b6..41bb4033 100644
--- a/definitions/output/reports/cwv_tech_lighthouse.js
+++ b/definitions/output/reports/cwv_tech_lighthouse.js
@@ -54,7 +54,6 @@ return Object.values(lighthouse)
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';
 `).query(ctx => `
-/* {"dataform_trigger": "report_cwv_tech_complete", "date": "${pastMonth}", "name": "lighthouse", "type": "report"} */
 SELECT
   date,
   app AS technology,
@@ -75,4 +74,18 @@ GROUP BY
   app,
   rank,
   geo
-`)
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "destination": "firestore",
+        "config": {
+          "databaseId": "tech-report-apis-{constants.environment}",
+          "collectionName": "lighthouse",
+          "collectionType": "report",
+          "date": "${pastMonth}"
+        },
+        "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
+      }'''
+    );
+  `)
diff --git a/definitions/output/reports/cwv_tech_page_weight.js b/definitions/output/reports/cwv_tech_page_weight.js
index 775462c8..684530fe 100644
--- a/definitions/output/reports/cwv_tech_page_weight.js
+++ b/definitions/output/reports/cwv_tech_page_weight.js
@@ -46,7 +46,6 @@ return Object.values(pageWeight)
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';
 `).query(ctx => `
-/* {"dataform_trigger": "report_cwv_tech_complete", "date": "${pastMonth}", "name": "page_weight", "type": "report"} */
 SELECT
   date,
   app AS technology,
@@ -65,4 +64,18 @@ GROUP BY
   app,
   rank,
   geo
-`)
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "destination": "firestore",
+        "config": {
+          "databaseId": "tech-report-apis-{constants.environment}",
+          "collectionName": "page_weight",
+          "collectionType": "report",
+          "date": "${pastMonth}"
+        },
+        "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
+      }'''
+    );
+  `)
diff --git a/definitions/output/reports/cwv_tech_technologies.js b/definitions/output/reports/cwv_tech_technologies.js
index 07f184c8..4ecfb2ba 100644
--- a/definitions/output/reports/cwv_tech_technologies.js
+++ b/definitions/output/reports/cwv_tech_technologies.js
@@ -5,7 +5,6 @@ publish('cwv_tech_technologies', {
   type: 'table',
   tags: ['crux_ready']
 }).query(ctx => `
-/* {"dataform_trigger": "report_cwv_tech_complete", "name": "technologies", "type": "dict"} */
 WITH pages AS (
   SELECT DISTINCT
     client,
@@ -86,4 +85,17 @@ SELECT
     MAX(IF(client = 'mobile', origins, 0)) AS mobile
   ) AS origins
 FROM total_pages
-`)
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "destination": "firestore",
+        "config": {
+          "databaseId": "tech-report-apis-{constants.environment}",
+          "collectionName": "technologies",
+          "collectionType": "dict"
+        },
+        "query": "SELECT * FROM ${ctx.self()}"
+      }'''
+    );
+  `)
diff --git a/definitions/output/reports/tech_report_adoption.js b/definitions/output/reports/tech_report_adoption.js
index 9c5b8a66..b9a3f83b 100644
--- a/definitions/output/reports/tech_report_adoption.js
+++ b/definitions/output/reports/tech_report_adoption.js
@@ -32,14 +32,17 @@ GROUP BY
   technology,
   version
 `).postOps(ctx => `
-  SELECT
-    reports.run_export_job(
-      JSON '''{
-        "dataform_trigger": "tech_report_complete",
-        "date": "${pastMonth}",
-        "name": "adoption",
-        "type": "report",
-        "environment": "${constants.environment}"
-      }'''
-    );
+SELECT
+  reports.run_export_job(
+    JSON '''{
+      "destination": "firestore",
+      "config": {
+        "databaseId": "tech-report-api-{constants.environment}",
+        "collectionName": "adoption",
+        "collectionType": "report",
+        "date": "${pastMonth}"
+      },
+      "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
+    }'''
+  );
 `)
diff --git a/definitions/output/reports/tech_report_audits.js b/definitions/output/reports/tech_report_audits.js
index f7aa4ac4..6c7a2f43 100644
--- a/definitions/output/reports/tech_report_audits.js
+++ b/definitions/output/reports/tech_report_audits.js
@@ -89,11 +89,14 @@ GROUP BY
   SELECT
     reports.run_export_job(
       JSON '''{
-        "dataform_trigger": "tech_report_complete",
-        "date": "${pastMonth}",
-        "name": "audits",
-        "type": "report",
-        "environment": "${constants.environment}"
+        "destination": "firestore",
+        "config": {
+          "databaseId": "tech-report-api-{constants.environment}",
+          "collectionName": "audits",
+          "collectionType": "report",
+          "date": "${pastMonth}"
+        },
+        "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
       }'''
     );
-`)
+  `)
diff --git a/definitions/output/reports/tech_report_categories.js b/definitions/output/reports/tech_report_categories.js
index 8c50970a..f149a9a1 100644
--- a/definitions/output/reports/tech_report_categories.js
+++ b/definitions/output/reports/tech_report_categories.js
@@ -94,10 +94,13 @@ FROM (
   SELECT
     reports.run_export_job(
       JSON '''{
-        "dataform_trigger": "tech_report_complete",
-        "name": "categories",
-        "type": "dict",
-        "environment": "${constants.environment}"
+        "destination": "firestore",
+        "config": {
+          "databaseId": "tech-report-api-{constants.environment}",
+          "collectionName": "categories",
+          "collectionType": "dict"
+        },
+        "query": "SELECT * FROM ${ctx.self()}"
       }'''
     );
-`)
+  `)
diff --git a/definitions/output/reports/tech_report_core_web_vitals.js b/definitions/output/reports/tech_report_core_web_vitals.js
index 5241ab6a..15ac0f04 100644
--- a/definitions/output/reports/tech_report_core_web_vitals.js
+++ b/definitions/output/reports/tech_report_core_web_vitals.js
@@ -103,11 +103,14 @@ GROUP BY
   SELECT
     reports.run_export_job(
       JSON '''{
-        "dataform_trigger": "tech_report_complete",
-        "date": "${pastMonth}",
-        "name": "core_web_vitals",
-        "type": "report",
-        "environment": "${constants.environment}"
+        "destination": "firestore",
+        "config": {
+          "databaseId": "tech-report-api-{constants.environment}",
+          "collectionName": "core_web_vitals",
+          "collectionType": "report",
+          "date": "${pastMonth}"
+        },
+        "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
       }'''
     );
-`)
+  `)
diff --git a/definitions/output/reports/tech_report_lighthouse.js b/definitions/output/reports/tech_report_lighthouse.js
index 021e317e..f9acf5cf 100644
--- a/definitions/output/reports/tech_report_lighthouse.js
+++ b/definitions/output/reports/tech_report_lighthouse.js
@@ -79,11 +79,14 @@ GROUP BY
   SELECT
     reports.run_export_job(
       JSON '''{
-        "dataform_trigger": "tech_report_complete",
-        "date": "${pastMonth}",
-        "name": "lighthouse",
-        "type": "report",
-        "environment": "${constants.environment}"
+        "destination": "firestore",
+        "config": {
+          "databaseId": "tech-report-api-{constants.environment}",
+          "collectionName": "lighthouse",
+          "collectionType": "report",
+          "date": "${pastMonth}"
+        },
+        "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
       }'''
     );
-`)
+  `)
diff --git a/definitions/output/reports/tech_report_page_weight.js b/definitions/output/reports/tech_report_page_weight.js
index 4a54438f..e65add77 100644
--- a/definitions/output/reports/tech_report_page_weight.js
+++ b/definitions/output/reports/tech_report_page_weight.js
@@ -72,11 +72,14 @@ GROUP BY
   SELECT
     reports.run_export_job(
       JSON '''{
-        "dataform_trigger": "tech_report_complete",
-        "date": "${pastMonth}",
-        "name": "page_weight",
-        "type": "report",
-        "environment": "${constants.environment}"
+        "destination": "firestore",
+        "config": {
+          "databaseId": "tech-report-api-{constants.environment}",
+          "collectionName": "page_weight",
+          "collectionType": "report",
+          "date": "${pastMonth}"
+        },
+        "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
       }'''
     );
-`)
+  `)
diff --git a/definitions/output/reports/tech_report_technologies.js b/definitions/output/reports/tech_report_technologies.js
index 0421376a..7bb385d1 100644
--- a/definitions/output/reports/tech_report_technologies.js
+++ b/definitions/output/reports/tech_report_technologies.js
@@ -91,11 +91,11 @@ FROM total_pages
       JSON '''{
         "destination": "firestore",
         "config": {
-          "database": "tech-report-api-${constants.environment}",
-          "collection": "technologies",
-          "type": "dict"
+          "databaseId": "tech-report-api-{constants.environment}",
+          "collectionName": "technologies",
+          "collectionType": "dict"
         },
         "query": "SELECT * FROM ${ctx.self()}"
       }'''
     );
-`)
+  `)
diff --git a/definitions/output/reports/tech_report_versions.js b/definitions/output/reports/tech_report_versions.js
index bcf6602d..b572e244 100644
--- a/definitions/output/reports/tech_report_versions.js
+++ b/definitions/output/reports/tech_report_versions.js
@@ -64,10 +64,13 @@ FROM total_origins
   SELECT
     reports.run_export_job(
       JSON '''{
-        "dataform_trigger": "tech_report_complete",
-        "name": "versions",
-        "type": "dict",
-        "environment": "${constants.environment}"
+        "destination": "firestore",
+        "config": {
+          "databaseId": "tech-report-api-{constants.environment}",
+          "collectionName": "versions",
+          "collectionType": "dict"
+        },
+        "query": "SELECT * FROM ${ctx.self()}"
       }'''
     );
-`)
+  `)

From 06960f2d52d9d6bd763f50ab3305313083b5bda9 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Tue, 15 Apr 2025 00:27:49 +0200
Subject: [PATCH 43/52] standard reports export draft

---
 definitions/output/reports/reports_dynamic.js | 52 ++++++++++++++-----
 1 file changed, 40 insertions(+), 12 deletions(-)

diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js
index 6fa85e75..ac445fec 100644
--- a/definitions/output/reports/reports_dynamic.js
+++ b/definitions/output/reports/reports_dynamic.js
@@ -1,12 +1,43 @@
 const configs = new reports.HTTPArchiveReports()
 const metrics = configs.listMetrics()
 
+const bucket = 'httparchive'
+const storagePath = '/reports/'
+
+function generateExportQuery(metric, sql, params, ctx) {
+  if (sql.type === 'histogram') {
+    return `
+SELECT
+  * EXCEPT(date)
+FROM ${ctx.self()}
+WHERE date = '${params.date}'
+`
+  } else if (sql.type === 'timeseries') {
+    return `
+SELECT
+  FORMAT_DATE('%Y_%m_%d', date) AS date,
+  * EXCEPT(date)
+FROM reports.${metric}_timeseries
+  `} else {
+    throw new Error('Unknown SQL type')
+  }
+}
+
+function generateExportPath(metric, sql, params) {
+  if (sql.type === 'histogram') {
+    return `${storagePath}${params.date.replaceAll('-', '_')}/${metric}.json`
+  } else if (sql.type === 'timeseries') {
+    return `${storagePath}${metric}.json`
+  } else {
+    throw new Error('Unknown SQL type')
+  }
+}
+
 const iterations = []
 for (
-  let month = constants.currentMonth; month >= constants.currentMonth; month = constants.fnPastMonth(month)) {
+  let date = constants.currentMonth; month >= constants.currentMonth; month = constants.fnPastMonth(month)) {
   iterations.push({
-    date: month,
-    devRankFilter: constants.devRankFilter
+    date: date
   })
 }
 
@@ -31,10 +62,10 @@ SELECT
     JSON '''{
       "destination": "cloud_storage",
       "config": {
-        "bucket": "httparchive",
-        "name": "reports/${constants.environment}/${metric.id}_${sql.type}_${params.date}.json"
+        "bucket": "${bucket}",
+        "name": "${generateExportPath(metric, sql, params)}"
       },
-      "query": "SELECT FORMAT_DATE('%Y_%m_%d', date) AS date, * EXCEPT(date) FROM ${ctx.self()}"
+      "query": "${generateExportQuery(metric, sql, params, ctx)}"}
     }'''
   );
       `)
@@ -57,10 +88,10 @@ INSERT INTO reports.${metric.id}_${sql.type}` + sql.query(ctx, params)
     JSON '''{
       "destination": "cloud_storage",
       "config": {
-        "bucket": "httparchive",
-        "name": "reports/${constants.environment}/${metric.id}_${sql.type}_${params.date}.json"
+        "bucket": "${bucket}",
+        "name": "${generateExportPath(metric, sql, params)}"
       },
-      "query": "SELECT FORMAT_DATE('%Y_%m_%d', date) AS date, * EXCEPT(date) FROM ${ctx.self()}"
+      "query": "${generateExportQuery(metric, sql, params, ctx)}"}
     }'''
   );
         `)
@@ -68,6 +99,3 @@ INSERT INTO reports.${metric.id}_${sql.type}` + sql.query(ctx, params)
     })
   })
 }
-
-// --"query": "SELECT * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${params.date}'"
-// `${this.storagePath}${date.replaceAll('-', '_')}/${metric}.json`

From a43a3dbd2e50e692f0f886ad5f0f3e097b17395f Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Tue, 15 Apr 2025 01:14:31 +0200
Subject: [PATCH 44/52] update description

---
 infra/tf/dataform_export/main.tf | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/infra/tf/dataform_export/main.tf b/infra/tf/dataform_export/main.tf
index 7029c817..282437b5 100644
--- a/infra/tf/dataform_export/main.tf
+++ b/infra/tf/dataform_export/main.tf
@@ -54,9 +54,19 @@ resource "google_bigquery_routine" "run_export_job" {
   routine_type    = "SCALAR_FUNCTION"
   definition_body = ""
   description     = <<EOT
-  Export data from Google BigQuery.
-  Example payload JSON: {"dataform_trigger": "tech_report_complete", "date": "2025-01-01", "name": "adoption", "type": "report"}
-  EOT
+Export data from Google BigQuery.
+Example payload JSON:
+  {
+    "destination": "firestore",
+    "config": {
+      "databaseId": "tech-report-api-dev",
+      "collectionName": "adoption",
+      "collectionType": "report",
+      "date": "2025-01-01"
+    },
+    "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM reports.tech_report_adoption WHERE date = '2025-01-01'"
+  }
+EOT
 
   arguments {
     name      = "payload"

From 95bca5b057bc1e5af1252ea0985e77750c2e2cf3 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Tue, 15 Apr 2025 06:05:02 +0200
Subject: [PATCH 45/52] dependabot update

---
 .github/dependabot.yml              | 14 +++++++++++++-
 infra/bigquery-export/package.json  |  6 +++---
 infra/dataform-export/package.json  |  4 ++--
 infra/dataform-trigger/package.json |  6 +++---
 package.json                        |  2 +-
 5 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index 01a3e3cf..c5dacfd1 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -10,6 +10,18 @@ updates:
     schedule:
       interval: "weekly"
   - package-ecosystem: "npm"
-    directory: "/src"
+    directory: "/infra/bigquery-export"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "npm"
+    directory: "infra/dataform-export"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "npm"
+    directory: "infra/dataform-trigger"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "terraform"
+    directory: "infra/tf/"
     schedule:
       interval: "weekly"
diff --git a/infra/bigquery-export/package.json b/infra/bigquery-export/package.json
index ff1cca66..b42d270b 100644
--- a/infra/bigquery-export/package.json
+++ b/infra/bigquery-export/package.json
@@ -8,9 +8,9 @@
   },
   "type": "module",
   "dependencies": {
-    "@google-cloud/bigquery": "^7.9.1",
-    "@google-cloud/firestore": "^7.10.0",
-    "@google-cloud/storage": "^7.14.0"
+    "@google-cloud/bigquery": "7.9.1",
+    "@google-cloud/firestore": "7.10.0",
+    "@google-cloud/storage": "7.14.0"
   },
   "author": "@max-ostapenko"
 }
diff --git a/infra/dataform-export/package.json b/infra/dataform-export/package.json
index 59c6ce5b..4603d6de 100644
--- a/infra/dataform-export/package.json
+++ b/infra/dataform-export/package.json
@@ -5,8 +5,8 @@
   "main": "index.js",
   "type": "module",
   "dependencies": {
-    "@google-cloud/functions-framework": "^3.4.2",
-    "@google-cloud/run": "^1.5.0"
+    "@google-cloud/functions-framework": "3.4.2",
+    "@google-cloud/run": "1.5.0"
   },
   "scripts": {
     "start": "npx functions-framework --target=dataform-export --signature-type=http --debug"
diff --git a/infra/dataform-trigger/package.json b/infra/dataform-trigger/package.json
index d4c90397..522cba94 100644
--- a/infra/dataform-trigger/package.json
+++ b/infra/dataform-trigger/package.json
@@ -4,9 +4,9 @@
   "author": "@max-ostapenko",
   "main": "index.js",
   "dependencies": {
-    "@google-cloud/bigquery": "^7.9.1",
-    "@google-cloud/dataform": "^1.3.0",
-    "@google-cloud/functions-framework": "^3.4.2"
+    "@google-cloud/bigquery": "7.9.1",
+    "@google-cloud/dataform": "1.3.0",
+    "@google-cloud/functions-framework": "3.4.2"
   },
   "scripts": {
     "start": "npx functions-framework --target=dataform-trigger --signature-type=http --port=8080 --debug"
diff --git a/package.json b/package.json
index 32981f6b..3be04e5e 100644
--- a/package.json
+++ b/package.json
@@ -2,7 +2,7 @@
   "name": "crawl-data",
   "author": "@max-ostapenko",
   "dependencies": {
-    "@dataform/core": "^3.0.19"
+    "@dataform/core": "3.0.19"
   },
   "scripts": {
     "format": "npx standard --fix; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint . --fix; terraform -chdir=infra/tf fmt -recursive",

From 00fae362a926a9a6c8198560108d5eaa588d3988 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Tue, 15 Apr 2025 06:09:49 +0200
Subject: [PATCH 46/52] fix month

---
 .../output/reports/cwv_tech_categories.js       |  1 -
 definitions/output/reports/reports_dynamic.js   | 17 +++++++++--------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/definitions/output/reports/cwv_tech_categories.js b/definitions/output/reports/cwv_tech_categories.js
index effd9cc9..45f61eb5 100644
--- a/definitions/output/reports/cwv_tech_categories.js
+++ b/definitions/output/reports/cwv_tech_categories.js
@@ -104,4 +104,3 @@ FROM total_pages
       }'''
     );
   `)
-
diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js
index ac445fec..2e39f3df 100644
--- a/definitions/output/reports/reports_dynamic.js
+++ b/definitions/output/reports/reports_dynamic.js
@@ -4,7 +4,7 @@ const metrics = configs.listMetrics()
 const bucket = 'httparchive'
 const storagePath = '/reports/'
 
-function generateExportQuery(metric, sql, params, ctx) {
+function generateExportQuery (metric, sql, params, ctx) {
   if (sql.type === 'histogram') {
     return `
 SELECT
@@ -18,12 +18,13 @@ SELECT
   FORMAT_DATE('%Y_%m_%d', date) AS date,
   * EXCEPT(date)
 FROM reports.${metric}_timeseries
-  `} else {
+  `
+  } else {
     throw new Error('Unknown SQL type')
   }
 }
 
-function generateExportPath(metric, sql, params) {
+function generateExportPath (metric, sql, params) {
   if (sql.type === 'histogram') {
     return `${storagePath}${params.date.replaceAll('-', '_')}/${metric}.json`
   } else if (sql.type === 'timeseries') {
@@ -35,9 +36,9 @@ function generateExportPath(metric, sql, params) {
 
 const iterations = []
 for (
-  let date = constants.currentMonth; month >= constants.currentMonth; month = constants.fnPastMonth(month)) {
+  let date = constants.currentMonth; date >= constants.currentMonth; date = constants.fnPastMonth(date)) {
   iterations.push({
-    date: date
+    date
   })
 }
 
@@ -49,8 +50,8 @@ if (iterations.length === 1) {
         type: 'incremental',
         protected: true,
         bigquery: sql.type === 'histogram' ? { partitionBy: 'date', clusterBy: ['client'] } : {},
-        schema: 'reports',
-        //tags: ['crawl_complete', 'http_reports']
+        schema: 'reports'
+        // tags: ['crawl_complete', 'http_reports']
       }).preOps(ctx => `
 --DELETE FROM ${ctx.self()}
 --WHERE date = '${params.date}';
@@ -76,7 +77,7 @@ SELECT
     metrics.forEach(metric => {
       metric.SQL.forEach(sql => {
         operate(metric.id + '_' + sql.type + '_' + params.date, {
-          //tags: ['crawl_complete']
+          // tags: ['crawl_complete']
         }).queries(ctx => `
 DELETE FROM reports.${metric.id}_${sql.type}
 WHERE date = '${params.date}';

From 6b444c8da8e6155a8cc7d98f8252c27300f3922e Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Tue, 15 Apr 2025 06:25:11 +0200
Subject: [PATCH 47/52] fix query generation

---
 definitions/output/reports/reports_dynamic.js | 27 ++++++++++---------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js
index 2e39f3df..da056597 100644
--- a/definitions/output/reports/reports_dynamic.js
+++ b/definitions/output/reports/reports_dynamic.js
@@ -5,30 +5,32 @@ const bucket = 'httparchive'
 const storagePath = '/reports/'
 
 function generateExportQuery (metric, sql, params, ctx) {
+  let query = ''
   if (sql.type === 'histogram') {
-    return `
+    query = `
 SELECT
   * EXCEPT(date)
 FROM ${ctx.self()}
 WHERE date = '${params.date}'
-`
-  } else if (sql.type === 'timeseries') {
-    return `
+`} else if (sql.type === 'timeseries') {
+    query = `
 SELECT
   FORMAT_DATE('%Y_%m_%d', date) AS date,
   * EXCEPT(date)
-FROM reports.${metric}_timeseries
-  `
-  } else {
+FROM ${ctx.self()}
+`} else {
     throw new Error('Unknown SQL type')
   }
+
+  const queryOutput = query.replace(/[\r\n]+/g, ' ');
+  return queryOutput
 }
 
 function generateExportPath (metric, sql, params) {
   if (sql.type === 'histogram') {
-    return `${storagePath}${params.date.replaceAll('-', '_')}/${metric}.json`
+    return `${storagePath}${params.date.replaceAll('-', '_')}/${metric.id}.json`
   } else if (sql.type === 'timeseries') {
-    return `${storagePath}${metric}.json`
+    return `${storagePath}${metric.id}.json`
   } else {
     throw new Error('Unknown SQL type')
   }
@@ -38,7 +40,8 @@ const iterations = []
 for (
   let date = constants.currentMonth; date >= constants.currentMonth; date = constants.fnPastMonth(date)) {
   iterations.push({
-    date
+    date,
+    devRankFilter: constants.devRankFilter
   })
 }
 
@@ -66,7 +69,7 @@ SELECT
         "bucket": "${bucket}",
         "name": "${generateExportPath(metric, sql, params)}"
       },
-      "query": "${generateExportQuery(metric, sql, params, ctx)}"}
+      "query": "${generateExportQuery(metric, sql, params, ctx)}"
     }'''
   );
       `)
@@ -92,7 +95,7 @@ INSERT INTO reports.${metric.id}_${sql.type}` + sql.query(ctx, params)
         "bucket": "${bucket}",
         "name": "${generateExportPath(metric, sql, params)}"
       },
-      "query": "${generateExportQuery(metric, sql, params, ctx)}"}
+      "query": "${generateExportQuery(metric, sql, params, ctx)}"
     }'''
   );
         `)

From 77f355da06d92f032712e3ec8e6f688306395026 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Tue, 15 Apr 2025 06:55:50 +0200
Subject: [PATCH 48/52] update

---
 infra/bigquery-export/package-lock.json | 12 +++---
 infra/tf/.terraform.lock.hcl            | 52 ++++++++++++-------------
 2 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/infra/bigquery-export/package-lock.json b/infra/bigquery-export/package-lock.json
index d520e3f5..65641f19 100644
--- a/infra/bigquery-export/package-lock.json
+++ b/infra/bigquery-export/package-lock.json
@@ -8,9 +8,9 @@
       "name": "bigquery-export",
       "version": "1.0.0",
       "dependencies": {
-        "@google-cloud/bigquery": "^7.9.1",
-        "@google-cloud/firestore": "^7.10.0",
-        "@google-cloud/storage": "^7.14.0"
+        "@google-cloud/bigquery": "7.9.1",
+        "@google-cloud/firestore": "7.10.0",
+        "@google-cloud/storage": "7.14.0"
       }
     },
     "node_modules/@google-cloud/bigquery": {
@@ -56,9 +56,9 @@
       }
     },
     "node_modules/@google-cloud/firestore": {
-      "version": "7.11.0",
-      "resolved": "https://registry.npmjs.org/@google-cloud/firestore/-/firestore-7.11.0.tgz",
-      "integrity": "sha512-88uZ+jLsp1aVMj7gh3EKYH1aulTAMFAp8sH/v5a9w8q8iqSG27RiWLoxSAFr/XocZ9hGiWH1kEnBw+zl3xAgNA==",
+      "version": "7.10.0",
+      "resolved": "https://registry.npmjs.org/@google-cloud/firestore/-/firestore-7.10.0.tgz",
+      "integrity": "sha512-VFNhdHvfnmqcHHs6YhmSNHHxQqaaD64GwiL0c+e1qz85S8SWZPC2XFRf8p9yHRTF40Kow424s1KBU9f0fdQa+Q==",
       "license": "Apache-2.0",
       "dependencies": {
         "@opentelemetry/api": "^1.3.0",
diff --git a/infra/tf/.terraform.lock.hcl b/infra/tf/.terraform.lock.hcl
index da11b295..f3153a4a 100644
--- a/infra/tf/.terraform.lock.hcl
+++ b/infra/tf/.terraform.lock.hcl
@@ -22,41 +22,41 @@ provider "registry.terraform.io/hashicorp/archive" {
 }
 
 provider "registry.terraform.io/hashicorp/google" {
-  version     = "6.25.0"
+  version     = "6.29.0"
   constraints = ">= 6.13.0"
   hashes = [
-    "h1:rvsmJc5J5OMMFXj9th8Yqfwm7cLZdRM/stPo0B8Datg=",
-    "zh:115ce3e84a02412a9a42c7c8f1c4568ab470bdd93c9a3a1f9202d4b77d7bc236",
-    "zh:623a32acea92d98a8bb66481fa9489e0a1e4dd6fc57ab70dbb53fae5732c1c63",
-    "zh:6440fc959b5e316152e26c916d55566311dafbe5b64e45d4b9c9931a5b29fa13",
-    "zh:91edb056638e723b1c7802d0e6e293611208e2825f645572ddba98122723f11c",
-    "zh:9b57ee44172677d2c2df03b22cf4b40e184ac8bd8facdd456ccbfdb7fee4684a",
-    "zh:9e2a97dd09b78b36caecdd990fff80bdb47a6a6d45c8b2dd4d23885d6bc89e65",
-    "zh:b33e40e9ba745f15b1c2e9ebcdccbac185788a4eb57450820cec7b9585858522",
-    "zh:d9e051bb703597384d83d924c49770e3bcdc1b68583d3def33a607ab168c634b",
-    "zh:e84253140fc3b0bd5cf7a1ebb54f993bacc6d5ee33c7b5e714ad834d5b2d35d0",
-    "zh:ebb624504c6f4297e691b6e3c00f789a6729461e5aa80fc165ef2ecd878e2d87",
+    "h1:U0Ca/+zZMMuea+r80qu9SRzWu8Waxny5aWGZXn+kVhc=",
+    "zh:01a501df2fb9ecbf0935b27e588bc7b6bcaf4ab0043747f4229c25e4ba47dadb",
+    "zh:056f8ab2b73755cf5a67228ab4a07882e76265fa25b07f2794d9939288164f48",
+    "zh:0dbdfa564f7db8a2e6f7e76437a9850b6101450120c08e87cf9846330736c0c6",
+    "zh:3c3e4ee801de22812bd07cb3d36b227f8057d7825998fb4d97051764a565b89b",
+    "zh:4e440eb4c60da9cd7d23b3b99be54c869472fd70006c39639a04b9a51248929c",
+    "zh:659490efd20b3e98e4166b2925baa18549d82e4c16751bc92baed0185d22d108",
+    "zh:9ae24b98a3a3346b8004c6b87e3b59decba2f64c7407e106263859c275105ef8",
+    "zh:c64cff9c17e302236bb9e0a6d5eff4c92540ce3947269f3db94e2b9a1b1a3f4e",
+    "zh:d2fd0aecbbbba463bcb0640f54c8df5e7a63918bdd44236dcd36dff33bb8de09",
+    "zh:f022316369ea676f5f9d11768b4c621abd3304c1e6d1f0c2361e4e2620c4b65d",
     "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
-    "zh:f8c3e8192e98eff09a9453b52424c0d9727ae284de4fafbc498f2d527e026850",
+    "zh:feb7d4fdbebdfabac2aaa73376f754736ccf089fa90adf6388701f89801188e6",
   ]
 }
 
 provider "registry.terraform.io/hashicorp/google-beta" {
-  version     = "6.25.0"
+  version     = "6.29.0"
   constraints = ">= 6.13.0"
   hashes = [
-    "h1:KlIctHHQkbToSXa41DWFQUsTKsfaWe+vjs/CFbnj3bc=",
-    "zh:101a8a679274cae57e8c816164fa28b6d271a8a53b3e007b84153fc63f0f06f7",
-    "zh:413c41e4ee14b10cb815cd01f8b880c12474ecbf63ecfc6415512fb0c18c473b",
-    "zh:4260143c8584a8f338a4d45f65c2f08f12b28a746c43d43a1e3a575f96ce4110",
-    "zh:4785080dd28b79a94ba6e277d247b6bb678d676d99244a8d664ea60802338df7",
-    "zh:673b09d67cb59487095fe7176c194e8c7ced62fc7bb5084016f3c87dd89e685e",
-    "zh:7de0dd9367c723cc203fdc4e70d316c779224483637039de5a3c9ee807c3fc55",
-    "zh:aa49c5955652b21ff2d0b7b2db0e18f59097d3a539efc98ef65deed4293d7327",
-    "zh:b4d73704d88deee367568fd68303878e6e60924c2a65df397f37b868e6aecaac",
-    "zh:cae0c5eff7c37c3f9857de08ca35b3a86d44855ae9c0ec2fab1eda604ff501c8",
-    "zh:e115f6b0c25bdad624fafbb5d685c8a48d9028f9525918274a5e5469f7dfde3d",
-    "zh:e536197371458b0b177a203f8367f99e74770b3dfce597034e880ea5f3f798b8",
+    "h1:NoShkbRyzLHxafFJJ10sw1cpzvJNrQCMuW3lc3cJ6GU=",
+    "zh:40ed1d1c359ec1551db1818ac249da01d3be24601e888b1861eb1c2f69c404e4",
+    "zh:4efdf4206e20b36f00d431410ab9158c53c36395528f99cc3b8b38261e63f667",
+    "zh:92449d1720fe60e5f52fe2e29cbdb38f7489823f571326e72cba5639bbd609c6",
+    "zh:941c8f24614a3af5834ede642cc7b5359eaaad044ee91e2acf5e7b91409b2e2a",
+    "zh:a3a66470e11cee61c8613cb318f196b5ada6686e98a7638f08b1136e532afc78",
+    "zh:b92bd0daa14bdec1901430b918149128f992f5a9d2a9327bc58ed15732bb4882",
+    "zh:ce440e06b452e51cab002e35b8dc3fa4b16f05b0733efd6a64611dd3176a5362",
+    "zh:d08de0b0b2a12c8696be1bb6d633cca735879f4fed708843a6dd07483ff435a0",
+    "zh:d220c78f0d2d51313465d5cf8c94f7bb225153e654349b79c280aa13e2ea549e",
+    "zh:d37e191cd6e41f9ca7b247cdad413c827f9a31bc58232c36712cfffd9b69f3f1",
+    "zh:d92b2a020b11306bf7afd8f9d97451eb38628747da044515f1f32e9146159e35",
     "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
   ]
 }

From 71fa9e737770a230950770ca733a5f0dcda35a03 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Tue, 15 Apr 2025 06:56:20 +0200
Subject: [PATCH 49/52] fix env var

---
 definitions/output/reports/cwv_tech_adoption.js           | 2 +-
 definitions/output/reports/cwv_tech_categories.js         | 2 +-
 definitions/output/reports/cwv_tech_core_web_vitals.js    | 2 +-
 definitions/output/reports/cwv_tech_lighthouse.js         | 2 +-
 definitions/output/reports/cwv_tech_page_weight.js        | 2 +-
 definitions/output/reports/cwv_tech_technologies.js       | 2 +-
 definitions/output/reports/tech_report_adoption.js        | 2 +-
 definitions/output/reports/tech_report_audits.js          | 2 +-
 definitions/output/reports/tech_report_categories.js      | 2 +-
 definitions/output/reports/tech_report_core_web_vitals.js | 2 +-
 definitions/output/reports/tech_report_lighthouse.js      | 2 +-
 definitions/output/reports/tech_report_page_weight.js     | 2 +-
 definitions/output/reports/tech_report_technologies.js    | 2 +-
 definitions/output/reports/tech_report_versions.js        | 2 +-
 14 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/definitions/output/reports/cwv_tech_adoption.js b/definitions/output/reports/cwv_tech_adoption.js
index 400d0a37..dff3a04d 100644
--- a/definitions/output/reports/cwv_tech_adoption.js
+++ b/definitions/output/reports/cwv_tech_adoption.js
@@ -35,7 +35,7 @@ GROUP BY
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-apis-{constants.environment}",
+          "databaseId": "tech-report-apis-${constants.environment}",
           "collectionName": "adoption",
           "collectionType": "report",
           "date": "${pastMonth}"
diff --git a/definitions/output/reports/cwv_tech_categories.js b/definitions/output/reports/cwv_tech_categories.js
index 45f61eb5..4b38a3bc 100644
--- a/definitions/output/reports/cwv_tech_categories.js
+++ b/definitions/output/reports/cwv_tech_categories.js
@@ -96,7 +96,7 @@ FROM total_pages
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-apis-{constants.environment}",
+          "databaseId": "tech-report-apis-${constants.environment}",
           "collectionName": "categories",
           "collectionType": "dict"
         },
diff --git a/definitions/output/reports/cwv_tech_core_web_vitals.js b/definitions/output/reports/cwv_tech_core_web_vitals.js
index 4f5a4237..2da4014d 100644
--- a/definitions/output/reports/cwv_tech_core_web_vitals.js
+++ b/definitions/output/reports/cwv_tech_core_web_vitals.js
@@ -103,7 +103,7 @@ GROUP BY
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-apis-{constants.environment}",
+          "databaseId": "tech-report-apis-${constants.environment}",
           "collectionName": "core_web_vitals",
           "collectionType": "report",
           "date": "${pastMonth}"
diff --git a/definitions/output/reports/cwv_tech_lighthouse.js b/definitions/output/reports/cwv_tech_lighthouse.js
index 41bb4033..d46f172d 100644
--- a/definitions/output/reports/cwv_tech_lighthouse.js
+++ b/definitions/output/reports/cwv_tech_lighthouse.js
@@ -80,7 +80,7 @@ GROUP BY
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-apis-{constants.environment}",
+          "databaseId": "tech-report-apis-${constants.environment}",
           "collectionName": "lighthouse",
           "collectionType": "report",
           "date": "${pastMonth}"
diff --git a/definitions/output/reports/cwv_tech_page_weight.js b/definitions/output/reports/cwv_tech_page_weight.js
index 684530fe..2e84e43b 100644
--- a/definitions/output/reports/cwv_tech_page_weight.js
+++ b/definitions/output/reports/cwv_tech_page_weight.js
@@ -70,7 +70,7 @@ GROUP BY
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-apis-{constants.environment}",
+          "databaseId": "tech-report-apis-${constants.environment}",
           "collectionName": "page_weight",
           "collectionType": "report",
           "date": "${pastMonth}"
diff --git a/definitions/output/reports/cwv_tech_technologies.js b/definitions/output/reports/cwv_tech_technologies.js
index 4ecfb2ba..826a6bfd 100644
--- a/definitions/output/reports/cwv_tech_technologies.js
+++ b/definitions/output/reports/cwv_tech_technologies.js
@@ -91,7 +91,7 @@ FROM total_pages
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-apis-{constants.environment}",
+          "databaseId": "tech-report-apis-${constants.environment}",
           "collectionName": "technologies",
           "collectionType": "dict"
         },
diff --git a/definitions/output/reports/tech_report_adoption.js b/definitions/output/reports/tech_report_adoption.js
index b9a3f83b..df58c5ab 100644
--- a/definitions/output/reports/tech_report_adoption.js
+++ b/definitions/output/reports/tech_report_adoption.js
@@ -37,7 +37,7 @@ SELECT
     JSON '''{
       "destination": "firestore",
       "config": {
-        "databaseId": "tech-report-api-{constants.environment}",
+        "databaseId": "tech-report-api-${constants.environment}",
         "collectionName": "adoption",
         "collectionType": "report",
         "date": "${pastMonth}"
diff --git a/definitions/output/reports/tech_report_audits.js b/definitions/output/reports/tech_report_audits.js
index 6c7a2f43..c628c87f 100644
--- a/definitions/output/reports/tech_report_audits.js
+++ b/definitions/output/reports/tech_report_audits.js
@@ -91,7 +91,7 @@ GROUP BY
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-api-{constants.environment}",
+          "databaseId": "tech-report-api-${constants.environment}",
           "collectionName": "audits",
           "collectionType": "report",
           "date": "${pastMonth}"
diff --git a/definitions/output/reports/tech_report_categories.js b/definitions/output/reports/tech_report_categories.js
index f149a9a1..6f588af9 100644
--- a/definitions/output/reports/tech_report_categories.js
+++ b/definitions/output/reports/tech_report_categories.js
@@ -96,7 +96,7 @@ FROM (
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-api-{constants.environment}",
+          "databaseId": "tech-report-api-${constants.environment}",
           "collectionName": "categories",
           "collectionType": "dict"
         },
diff --git a/definitions/output/reports/tech_report_core_web_vitals.js b/definitions/output/reports/tech_report_core_web_vitals.js
index 15ac0f04..b3a7b819 100644
--- a/definitions/output/reports/tech_report_core_web_vitals.js
+++ b/definitions/output/reports/tech_report_core_web_vitals.js
@@ -105,7 +105,7 @@ GROUP BY
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-api-{constants.environment}",
+          "databaseId": "tech-report-api-${constants.environment}",
           "collectionName": "core_web_vitals",
           "collectionType": "report",
           "date": "${pastMonth}"
diff --git a/definitions/output/reports/tech_report_lighthouse.js b/definitions/output/reports/tech_report_lighthouse.js
index f9acf5cf..c464ad58 100644
--- a/definitions/output/reports/tech_report_lighthouse.js
+++ b/definitions/output/reports/tech_report_lighthouse.js
@@ -81,7 +81,7 @@ GROUP BY
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-api-{constants.environment}",
+          "databaseId": "tech-report-api-${constants.environment}",
           "collectionName": "lighthouse",
           "collectionType": "report",
           "date": "${pastMonth}"
diff --git a/definitions/output/reports/tech_report_page_weight.js b/definitions/output/reports/tech_report_page_weight.js
index e65add77..2046434f 100644
--- a/definitions/output/reports/tech_report_page_weight.js
+++ b/definitions/output/reports/tech_report_page_weight.js
@@ -74,7 +74,7 @@ GROUP BY
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-api-{constants.environment}",
+          "databaseId": "tech-report-api-${constants.environment}",
           "collectionName": "page_weight",
           "collectionType": "report",
           "date": "${pastMonth}"
diff --git a/definitions/output/reports/tech_report_technologies.js b/definitions/output/reports/tech_report_technologies.js
index 7bb385d1..1a0941ea 100644
--- a/definitions/output/reports/tech_report_technologies.js
+++ b/definitions/output/reports/tech_report_technologies.js
@@ -91,7 +91,7 @@ FROM total_pages
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-api-{constants.environment}",
+          "databaseId": "tech-report-api-${constants.environment}",
           "collectionName": "technologies",
           "collectionType": "dict"
         },
diff --git a/definitions/output/reports/tech_report_versions.js b/definitions/output/reports/tech_report_versions.js
index b572e244..3e6f1389 100644
--- a/definitions/output/reports/tech_report_versions.js
+++ b/definitions/output/reports/tech_report_versions.js
@@ -66,7 +66,7 @@ FROM total_origins
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-api-{constants.environment}",
+          "databaseId": "tech-report-api-${constants.environment}",
           "collectionName": "versions",
           "collectionType": "dict"
         },

From 461077a56f21ca177c2a2db981231f5746b19203 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Tue, 15 Apr 2025 07:11:24 +0200
Subject: [PATCH 50/52] fix export config

---
 definitions/output/reports/cwv_tech_adoption.js           | 6 +++---
 definitions/output/reports/cwv_tech_categories.js         | 6 +++---
 definitions/output/reports/cwv_tech_core_web_vitals.js    | 6 +++---
 definitions/output/reports/cwv_tech_lighthouse.js         | 6 +++---
 definitions/output/reports/cwv_tech_page_weight.js        | 6 +++---
 definitions/output/reports/cwv_tech_technologies.js       | 6 +++---
 definitions/output/reports/tech_report_adoption.js        | 6 +++---
 definitions/output/reports/tech_report_audits.js          | 6 +++---
 definitions/output/reports/tech_report_categories.js      | 6 +++---
 definitions/output/reports/tech_report_core_web_vitals.js | 6 +++---
 definitions/output/reports/tech_report_lighthouse.js      | 6 +++---
 definitions/output/reports/tech_report_page_weight.js     | 6 +++---
 definitions/output/reports/tech_report_technologies.js    | 6 +++---
 definitions/output/reports/tech_report_versions.js        | 6 +++---
 infra/tf/dataform_export/main.tf                          | 6 +++---
 15 files changed, 45 insertions(+), 45 deletions(-)

diff --git a/definitions/output/reports/cwv_tech_adoption.js b/definitions/output/reports/cwv_tech_adoption.js
index dff3a04d..b5cb4749 100644
--- a/definitions/output/reports/cwv_tech_adoption.js
+++ b/definitions/output/reports/cwv_tech_adoption.js
@@ -35,9 +35,9 @@ GROUP BY
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-apis-${constants.environment}",
-          "collectionName": "adoption",
-          "collectionType": "report",
+          "database": "tech-report-apis-${constants.environment}",
+          "collection": "adoption",
+          "type": "report",
           "date": "${pastMonth}"
         },
         "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
diff --git a/definitions/output/reports/cwv_tech_categories.js b/definitions/output/reports/cwv_tech_categories.js
index 4b38a3bc..4324622d 100644
--- a/definitions/output/reports/cwv_tech_categories.js
+++ b/definitions/output/reports/cwv_tech_categories.js
@@ -96,9 +96,9 @@ FROM total_pages
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-apis-${constants.environment}",
-          "collectionName": "categories",
-          "collectionType": "dict"
+          "database": "tech-report-apis-${constants.environment}",
+          "collection": "categories",
+          "type": "dict"
         },
         "query": "SELECT * FROM ${ctx.self()}"
       }'''
diff --git a/definitions/output/reports/cwv_tech_core_web_vitals.js b/definitions/output/reports/cwv_tech_core_web_vitals.js
index 2da4014d..04336ec0 100644
--- a/definitions/output/reports/cwv_tech_core_web_vitals.js
+++ b/definitions/output/reports/cwv_tech_core_web_vitals.js
@@ -103,9 +103,9 @@ GROUP BY
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-apis-${constants.environment}",
-          "collectionName": "core_web_vitals",
-          "collectionType": "report",
+          "database": "tech-report-apis-${constants.environment}",
+          "collection": "core_web_vitals",
+          "type": "report",
           "date": "${pastMonth}"
         },
         "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
diff --git a/definitions/output/reports/cwv_tech_lighthouse.js b/definitions/output/reports/cwv_tech_lighthouse.js
index d46f172d..775a482d 100644
--- a/definitions/output/reports/cwv_tech_lighthouse.js
+++ b/definitions/output/reports/cwv_tech_lighthouse.js
@@ -80,9 +80,9 @@ GROUP BY
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-apis-${constants.environment}",
-          "collectionName": "lighthouse",
-          "collectionType": "report",
+          "database": "tech-report-apis-${constants.environment}",
+          "collection": "lighthouse",
+          "type": "report",
           "date": "${pastMonth}"
         },
         "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
diff --git a/definitions/output/reports/cwv_tech_page_weight.js b/definitions/output/reports/cwv_tech_page_weight.js
index 2e84e43b..ea9e58e6 100644
--- a/definitions/output/reports/cwv_tech_page_weight.js
+++ b/definitions/output/reports/cwv_tech_page_weight.js
@@ -70,9 +70,9 @@ GROUP BY
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-apis-${constants.environment}",
-          "collectionName": "page_weight",
-          "collectionType": "report",
+          "database": "tech-report-apis-${constants.environment}",
+          "collection": "page_weight",
+          "type": "report",
           "date": "${pastMonth}"
         },
         "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
diff --git a/definitions/output/reports/cwv_tech_technologies.js b/definitions/output/reports/cwv_tech_technologies.js
index 826a6bfd..9b7cb985 100644
--- a/definitions/output/reports/cwv_tech_technologies.js
+++ b/definitions/output/reports/cwv_tech_technologies.js
@@ -91,9 +91,9 @@ FROM total_pages
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-apis-${constants.environment}",
-          "collectionName": "technologies",
-          "collectionType": "dict"
+          "database": "tech-report-apis-${constants.environment}",
+          "collection": "technologies",
+          "type": "dict"
         },
         "query": "SELECT * FROM ${ctx.self()}"
       }'''
diff --git a/definitions/output/reports/tech_report_adoption.js b/definitions/output/reports/tech_report_adoption.js
index df58c5ab..ecef670f 100644
--- a/definitions/output/reports/tech_report_adoption.js
+++ b/definitions/output/reports/tech_report_adoption.js
@@ -37,9 +37,9 @@ SELECT
     JSON '''{
       "destination": "firestore",
       "config": {
-        "databaseId": "tech-report-api-${constants.environment}",
-        "collectionName": "adoption",
-        "collectionType": "report",
+        "database": "tech-report-api-${constants.environment}",
+        "collection": "adoption",
+        "type": "report",
         "date": "${pastMonth}"
       },
       "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
diff --git a/definitions/output/reports/tech_report_audits.js b/definitions/output/reports/tech_report_audits.js
index c628c87f..5d512209 100644
--- a/definitions/output/reports/tech_report_audits.js
+++ b/definitions/output/reports/tech_report_audits.js
@@ -91,9 +91,9 @@ GROUP BY
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-api-${constants.environment}",
-          "collectionName": "audits",
-          "collectionType": "report",
+          "database": "tech-report-api-${constants.environment}",
+          "collection": "audits",
+          "type": "report",
           "date": "${pastMonth}"
         },
         "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
diff --git a/definitions/output/reports/tech_report_categories.js b/definitions/output/reports/tech_report_categories.js
index 6f588af9..62ffe257 100644
--- a/definitions/output/reports/tech_report_categories.js
+++ b/definitions/output/reports/tech_report_categories.js
@@ -96,9 +96,9 @@ FROM (
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-api-${constants.environment}",
-          "collectionName": "categories",
-          "collectionType": "dict"
+          "database": "tech-report-api-${constants.environment}",
+          "collection": "categories",
+          "type": "dict"
         },
         "query": "SELECT * FROM ${ctx.self()}"
       }'''
diff --git a/definitions/output/reports/tech_report_core_web_vitals.js b/definitions/output/reports/tech_report_core_web_vitals.js
index b3a7b819..01a2092f 100644
--- a/definitions/output/reports/tech_report_core_web_vitals.js
+++ b/definitions/output/reports/tech_report_core_web_vitals.js
@@ -105,9 +105,9 @@ GROUP BY
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-api-${constants.environment}",
-          "collectionName": "core_web_vitals",
-          "collectionType": "report",
+          "database": "tech-report-api-${constants.environment}",
+          "collection": "core_web_vitals",
+          "type": "report",
           "date": "${pastMonth}"
         },
         "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
diff --git a/definitions/output/reports/tech_report_lighthouse.js b/definitions/output/reports/tech_report_lighthouse.js
index c464ad58..ab28ccb4 100644
--- a/definitions/output/reports/tech_report_lighthouse.js
+++ b/definitions/output/reports/tech_report_lighthouse.js
@@ -81,9 +81,9 @@ GROUP BY
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-api-${constants.environment}",
-          "collectionName": "lighthouse",
-          "collectionType": "report",
+          "database": "tech-report-api-${constants.environment}",
+          "collection": "lighthouse",
+          "type": "report",
           "date": "${pastMonth}"
         },
         "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
diff --git a/definitions/output/reports/tech_report_page_weight.js b/definitions/output/reports/tech_report_page_weight.js
index 2046434f..63d269d5 100644
--- a/definitions/output/reports/tech_report_page_weight.js
+++ b/definitions/output/reports/tech_report_page_weight.js
@@ -74,9 +74,9 @@ GROUP BY
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-api-${constants.environment}",
-          "collectionName": "page_weight",
-          "collectionType": "report",
+          "database": "tech-report-api-${constants.environment}",
+          "collection": "page_weight",
+          "type": "report",
           "date": "${pastMonth}"
         },
         "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'"
diff --git a/definitions/output/reports/tech_report_technologies.js b/definitions/output/reports/tech_report_technologies.js
index 1a0941ea..9a1b7ae2 100644
--- a/definitions/output/reports/tech_report_technologies.js
+++ b/definitions/output/reports/tech_report_technologies.js
@@ -91,9 +91,9 @@ FROM total_pages
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-api-${constants.environment}",
-          "collectionName": "technologies",
-          "collectionType": "dict"
+          "database": "tech-report-api-${constants.environment}",
+          "collection": "technologies",
+          "type": "dict"
         },
         "query": "SELECT * FROM ${ctx.self()}"
       }'''
diff --git a/definitions/output/reports/tech_report_versions.js b/definitions/output/reports/tech_report_versions.js
index 3e6f1389..2ddd8266 100644
--- a/definitions/output/reports/tech_report_versions.js
+++ b/definitions/output/reports/tech_report_versions.js
@@ -66,9 +66,9 @@ FROM total_origins
       JSON '''{
         "destination": "firestore",
         "config": {
-          "databaseId": "tech-report-api-${constants.environment}",
-          "collectionName": "versions",
-          "collectionType": "dict"
+          "database": "tech-report-api-${constants.environment}",
+          "collection": "versions",
+          "type": "dict"
         },
         "query": "SELECT * FROM ${ctx.self()}"
       }'''
diff --git a/infra/tf/dataform_export/main.tf b/infra/tf/dataform_export/main.tf
index 282437b5..79949554 100644
--- a/infra/tf/dataform_export/main.tf
+++ b/infra/tf/dataform_export/main.tf
@@ -59,9 +59,9 @@ Example payload JSON:
   {
     "destination": "firestore",
     "config": {
-      "databaseId": "tech-report-api-dev",
-      "collectionName": "adoption",
-      "collectionType": "report",
+      "database": "tech-report-api-dev",
+      "collection": "adoption",
+      "type": "report",
       "date": "2025-01-01"
     },
     "query": "SELECT STRING(date) AS date, * EXCEPT(date) FROM reports.tech_report_adoption WHERE date = '2025-01-01'"

From a722e6081262ae447831c29c8cf676eafd760858 Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Tue, 15 Apr 2025 07:33:40 +0200
Subject: [PATCH 51/52] formatting

---
 definitions/output/reports/reports_dynamic.js | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js
index da056597..b1765b8a 100644
--- a/definitions/output/reports/reports_dynamic.js
+++ b/definitions/output/reports/reports_dynamic.js
@@ -12,17 +12,19 @@ SELECT
   * EXCEPT(date)
 FROM ${ctx.self()}
 WHERE date = '${params.date}'
-`} else if (sql.type === 'timeseries') {
+`
+  } else if (sql.type === 'timeseries') {
     query = `
 SELECT
   FORMAT_DATE('%Y_%m_%d', date) AS date,
   * EXCEPT(date)
 FROM ${ctx.self()}
-`} else {
+`
+  } else {
     throw new Error('Unknown SQL type')
   }
 
-  const queryOutput = query.replace(/[\r\n]+/g, ' ');
+  const queryOutput = query.replace(/[\r\n]+/g, ' ')
   return queryOutput
 }
 

From 793fc64da076706e0c8451edeb48752323571d2c Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Tue, 15 Apr 2025 08:13:54 +0200
Subject: [PATCH 52/52] fix

---
 definitions/output/reports/cwv_tech_categories.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/definitions/output/reports/cwv_tech_categories.js b/definitions/output/reports/cwv_tech_categories.js
index 4324622d..057c8083 100644
--- a/definitions/output/reports/cwv_tech_categories.js
+++ b/definitions/output/reports/cwv_tech_categories.js
@@ -49,7 +49,7 @@ technology_stats AS (
   SELECT
     technology,
     category_obj AS categories,
-    SUM(origins.dektop + origins.mobile) AS total_origins
+    SUM(origins.desktop + origins.mobile) AS total_origins
   FROM ${ctx.ref('reports', 'cwv_tech_technologies')}
   GROUP BY
     technology,