From 255c911f24d83e6963d9b29b629543c4b41e0eca Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Mon, 30 Jun 2025 22:10:29 +0200 Subject: [PATCH 1/6] memory management --- infra/bigquery-export/firestore.js | 131 +++++++++++++++++++++++------ infra/tf/bigquery_export/main.tf | 25 +++++- 2 files changed, 128 insertions(+), 28 deletions(-) diff --git a/infra/bigquery-export/firestore.js b/infra/bigquery-export/firestore.js index 38225983..fcacd81c 100644 --- a/infra/bigquery-export/firestore.js +++ b/infra/bigquery-export/firestore.js @@ -20,22 +20,64 @@ export class FirestoreBatch { // Configuration constants this.config = { timeout: 10 * 60 * 1000, // 10 minutes - progressReportInterval: 200000, // Report progress every N operations - flushThreshold: 200000 // Flush BulkWriter every N operations + progressReportInterval: 100000, // Report progress every N operations + flushThreshold: 50000, // Flush BulkWriter every N operations + gcInterval: 50000 // Force garbage collection interval } this.reset() } + // Memory monitoring utility + logMemoryUsage (operation = '') { + const used = process.memoryUsage() + const memoryInfo = { + rss: Math.round(used.rss / 1024 / 1024 * 100) / 100, + heapTotal: Math.round(used.heapTotal / 1024 / 1024 * 100) / 100, + heapUsed: Math.round(used.heapUsed / 1024 / 1024 * 100) / 100, + external: Math.round(used.external / 1024 / 1024 * 100) / 100 + } + + console.log(`Memory usage ${operation}: RSS ${memoryInfo.rss}MB, Heap Used ${memoryInfo.heapUsed}MB, Heap Total ${memoryInfo.heapTotal}MB, External ${memoryInfo.external}MB`) + + // Configurable memory warning threshold from environment + const warningThreshold = parseInt(process.env.MEMORY_WARNING_THRESHOLD_MB || '1500') + if (memoryInfo.heapUsed > warningThreshold) { + console.warn(`⚠️ High memory usage detected: ${memoryInfo.heapUsed}MB heap used (threshold: ${warningThreshold}MB)`) + } + + return memoryInfo + } + + // Enhanced reset with memory cleanup reset () { this.processedDocs = 0 this.totalDocs = 0 + + // Clean up existing BulkWriter if it exists + if (this.bulkWriter) { + try { + this.bulkWriter.close() + } catch (error) { + console.warn('Error closing existing BulkWriter:', error.message) + } + } this.bulkWriter = null + + // Force garbage collection if available + if (global.gc) { + global.gc() + } + + // Log memory usage after reset + this.logMemoryUsage('after reset') } createBulkWriter (operation) { const bulkWriter = this.firestore.bulkWriter() + bulkWriter.maxBatchSize = 500 // Reduce batch size for memory efficiency + // Configure error handling with progress info bulkWriter.onWriteError((error) => { const progressInfo = this.totalDocs > 0 ? ` (${this.processedDocs}/${this.totalDocs})` : '' @@ -54,6 +96,11 @@ export class FirestoreBatch { if (this.processedDocs % this.config.progressReportInterval === 0) { const progressInfo = this.totalDocs > 0 ? ` (${this.processedDocs}/${this.totalDocs})` : ` (${this.processedDocs} processed)` console.log(`Progress${progressInfo} - ${operation}ing documents in ${this.collectionName}`) + + // Force garbage collection periodically + if (this.processedDocs % this.config.gcInterval === 0 && global.gc) { + global.gc() + } } }) @@ -115,20 +162,26 @@ export class FirestoreBatch { if (snapshot.empty) break // Add all delete operations to BulkWriter - snapshot.docs.forEach(doc => { + for (const doc of snapshot.docs) { this.bulkWriter.delete(doc.ref) deletedCount++ - }) - // Periodically flush to manage memory - // if (deletedCount % this.config.flushThreshold === 0) { - console.log(`Flushing BulkWriter at ${deletedCount} operations...`) - await this.bulkWriter.flush() - // } + // Frequent flushing to prevent memory buildup + if (deletedCount % this.config.flushThreshold === 0) { + console.log(`Flushing BulkWriter at ${deletedCount} operations...`) + await this.bulkWriter.flush() + + // Force garbage collection after flush + if (global.gc) { + global.gc() + } + } + } } // Final flush and close console.log('Finalizing deletion operations...') + await this.bulkWriter.flush() await this.bulkWriter.close() const duration = (Date.now() - startTime) / 1000 @@ -144,32 +197,52 @@ export class FirestoreBatch { this.bulkWriter = this.createBulkWriter('writ') let rowCount = 0 + let batchCount = 0 const collectionRef = this.firestore.collection(this.collectionName) - for await (const row of rowStream) { - // Add document to BulkWriter - const docRef = collectionRef.doc() - this.bulkWriter.set(docRef, row) - - rowCount++ - this.totalDocs = rowCount // Update total as we go since we can't predict BigQuery result size - - // Periodically flush to manage memory - if (rowCount % this.config.flushThreshold === 0) { - console.log(`Flushing BulkWriter at ${rowCount} operations...`) - await this.bulkWriter.flush() + try { + for await (const row of rowStream) { + // Add document to BulkWriter + const docRef = collectionRef.doc() + this.bulkWriter.set(docRef, row) + + rowCount++ + this.totalDocs = rowCount // Update totalDocs for progress tracking + + if (rowCount % this.config.flushThreshold === 0) { + console.log(`Flushing BulkWriter at ${rowCount} operations...`) + await this.bulkWriter.flush() + batchCount++ + + if (batchCount % 5 === 0 && global.gc) { + console.log(`Forcing garbage collection at batch ${batchCount}...`) + global.gc() + } + } } + } catch (error) { + console.error('Error during BigQuery streaming:', error) + throw error } // Final flush and close console.log('Finalizing write operations...') + await this.bulkWriter.flush() await this.bulkWriter.close() + // Final garbage collection + if (global.gc) { + global.gc() + } + const duration = (Date.now() - startTime) / 1000 console.info(`Transfer to ${this.collectionName} complete. Total rows processed: ${this.processedDocs}. Time: ${duration} seconds`) } async export (query, exportConfig) { + console.log(`Starting export to ${exportConfig.collection}...`) + this.logMemoryUsage('at start') + // Configure Firestore settings this.firestore.settings({ databaseId: exportConfig.database, @@ -183,9 +256,19 @@ export class FirestoreBatch { date: exportConfig.date }) - await this.batchDelete() + try { + await this.batchDelete() + this.logMemoryUsage('after deletion') - const rowStream = await this.bigquery.queryResultsStream(query) - await this.streamFromBigQuery(rowStream) + const rowStream = await this.bigquery.queryResultsStream(query) + await this.streamFromBigQuery(rowStream) + + this.logMemoryUsage('at completion') + console.log(`✅ Export to ${exportConfig.collection} completed successfully`) + } catch (error) { + this.logMemoryUsage('on error') + console.error(`❌ Export to ${exportConfig.collection} failed:`, error) + throw error + } } } diff --git a/infra/tf/bigquery_export/main.tf b/infra/tf/bigquery_export/main.tf index 76ab75a9..7c753439 100644 --- a/infra/tf/bigquery_export/main.tf +++ b/infra/tf/bigquery_export/main.tf @@ -20,8 +20,14 @@ resource "google_cloud_run_v2_job" "bigquery_export" { deletion_protection = false template { - parallelism = 0 + parallelism = 1 + task_count = 1 # Ensure single task execution + template { + timeout = "10800s" # 3 hours + service_account = var.function_identity + max_retries = 0 # No retries + containers { image = "${var.location}.gcr.io/${var.project}/cloud-run/${var.function_name}:latest" resources { @@ -34,10 +40,21 @@ resource "google_cloud_run_v2_job" "bigquery_export" { name = "EXPORT_CONFIG" value = "" } + env { + name = "NODE_OPTIONS" + value = "--expose-gc --max-old-space-size=6144" # 6GB heap limit with GC + } + + env { + name = "MEMORY_WARNING_THRESHOLD_MB" + value = "4915" # 80% of max heap size = 6144MB + } + + env { + name = "LOG_LEVEL" + value = "info" + } } - timeout = "7200s" - service_account = var.function_identity - max_retries = 1 } } } From 629c32c914aa8da61caee8764d5ed6d68290c9d7 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 6 Jul 2025 18:33:00 +0200 Subject: [PATCH 2/6] trigger with docker --- docs/infrastructure.md | 1 - infra/bigquery-export/.dockerignore | 1 + infra/bigquery-export/Dockerfile | 11 +- infra/bigquery-export/package.json | 2 +- infra/bigquery_export_spark/Dockerfile | 37 - infra/bigquery_export_spark/requirements.txt | 2 - infra/bigquery_export_spark/src/firestore.py | 158 -- infra/dataform-export/index.js | 72 - infra/dataform-export/package-lock.json | 2413 ----------------- infra/dataform-export/package.json | 14 - infra/dataform-service/.dockerignore | 5 + infra/dataform-service/Dockerfile | 22 + infra/dataform-service/README.md | 117 + infra/dataform-service/SUMMARY.md | 67 + infra/dataform-service/cloud_run.js | 25 + .../dataform.js | 8 +- .../index.js | 116 +- .../package-lock.json | 584 ++-- infra/dataform-service/package.json | 18 + infra/dataform-trigger/package.json | 14 - infra/tf/.terraform.lock.hcl | 41 +- infra/tf/bigquery_export/docker.tf | 41 + infra/tf/bigquery_export/main.tf | 10 +- infra/tf/dataform_export/main.tf | 82 - infra/tf/dataform_export/variables.tf | 19 - infra/tf/dataform_service/docker.tf | 41 + infra/tf/dataform_service/export_resources.tf | 46 + infra/tf/dataform_service/main.tf | 56 + .../tf/dataform_service/trigger_resources.tf | 63 + infra/tf/dataform_service/variables.tf | 24 + infra/tf/dataform_trigger/main.tf | 134 - infra/tf/dataform_trigger/variables.tf | 19 - infra/tf/functions/main.tf | 14 - infra/tf/functions/output.tf | 9 - infra/tf/functions/variables.tf | 4 - infra/tf/main.tf | 21 +- 36 files changed, 1061 insertions(+), 3250 deletions(-) delete mode 100644 infra/bigquery_export_spark/Dockerfile delete mode 100644 infra/bigquery_export_spark/requirements.txt delete mode 100644 infra/bigquery_export_spark/src/firestore.py delete mode 100644 infra/dataform-export/index.js delete mode 100644 infra/dataform-export/package-lock.json delete mode 100644 infra/dataform-export/package.json create mode 100644 infra/dataform-service/.dockerignore create mode 100644 infra/dataform-service/Dockerfile create mode 100644 infra/dataform-service/README.md create mode 100644 infra/dataform-service/SUMMARY.md create mode 100644 infra/dataform-service/cloud_run.js rename infra/{dataform-trigger => dataform-service}/dataform.js (84%) rename infra/{dataform-trigger => dataform-service}/index.js (60%) rename infra/{dataform-trigger => dataform-service}/package-lock.json (84%) create mode 100644 infra/dataform-service/package.json delete mode 100644 infra/dataform-trigger/package.json create mode 100644 infra/tf/bigquery_export/docker.tf delete mode 100644 infra/tf/dataform_export/main.tf delete mode 100644 infra/tf/dataform_export/variables.tf create mode 100644 infra/tf/dataform_service/docker.tf create mode 100644 infra/tf/dataform_service/export_resources.tf create mode 100644 infra/tf/dataform_service/main.tf create mode 100644 infra/tf/dataform_service/trigger_resources.tf create mode 100644 infra/tf/dataform_service/variables.tf delete mode 100644 infra/tf/dataform_trigger/main.tf delete mode 100644 infra/tf/dataform_trigger/variables.tf delete mode 100644 infra/tf/functions/output.tf diff --git a/docs/infrastructure.md b/docs/infrastructure.md index aac2ccb9..0ef40ae6 100644 --- a/docs/infrastructure.md +++ b/docs/infrastructure.md @@ -52,7 +52,6 @@ graph LR; dataformTrigger --> cloud_run_logs dataform_repo_workflow --> dataform_logs bq_jobs --> bq_logs - ``` ## Triggering pipelines diff --git a/infra/bigquery-export/.dockerignore b/infra/bigquery-export/.dockerignore index 5805cc92..9a69e312 100644 --- a/infra/bigquery-export/.dockerignore +++ b/infra/bigquery-export/.dockerignore @@ -7,3 +7,4 @@ npm-debug.log coverage *.md .DS_Store +cloudbuild.yaml diff --git a/infra/bigquery-export/Dockerfile b/infra/bigquery-export/Dockerfile index 45773da4..61fc25a9 100644 --- a/infra/bigquery-export/Dockerfile +++ b/infra/bigquery-export/Dockerfile @@ -1,13 +1,14 @@ # checkov:skip=CKV_DOCKER_3:Ensure that a user for the container has been created -FROM node:current-slim +FROM node:22-slim -WORKDIR /usr/src/app +# Set the working directory +WORKDIR /app -# Copy package files first for better caching +# Copy package files first for better layer caching COPY package*.json ./ -# Install dependencies -RUN npm ci --only=production --quiet --no-fund --no-audit +# Install dependencies (this layer will be cached unless package files change) +RUN npm ci --only=production --quiet --no-fund --no-audit && npm cache clean --force # Copy source code COPY . . diff --git a/infra/bigquery-export/package.json b/infra/bigquery-export/package.json index 968a822e..71810842 100644 --- a/infra/bigquery-export/package.json +++ b/infra/bigquery-export/package.json @@ -4,7 +4,7 @@ "main": "index.js", "scripts": { "start": "node index.js", - "build": "gcloud builds submit" + "build": "docker build -t bigquery-export ." }, "type": "module", "dependencies": { diff --git a/infra/bigquery_export_spark/Dockerfile b/infra/bigquery_export_spark/Dockerfile deleted file mode 100644 index 49b16fd0..00000000 --- a/infra/bigquery_export_spark/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -# Dataproc image example: https://cloud.google.com/dataproc-serverless/docs/guides/custom-containers -# Recommendation: Use Debian 12. -FROM debian:12-slim -# python:3.12-slim - -# Suppress interactive prompts -ENV DEBIAN_FRONTEND=noninteractive - -# Install utilities required by Spark scripts. -RUN apt-get update && apt-get install -y procps=\* tini=\* libjemalloc2=\* \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Enable jemalloc2 as default memory allocator -ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2 - -# Install and configure Miniconda3. -ENV CONDA_HOME=/opt/miniforge3 -ENV PYSPARK_PYTHON=${CONDA_HOME}/bin/python -ENV PATH=${CONDA_HOME}/bin:${PATH} -ADD https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh . -RUN bash Miniforge3-Linux-x86_64.sh -b -p /opt/miniforge3 \ - && ${CONDA_HOME}/bin/conda config --system --set always_yes True \ - && ${CONDA_HOME}/bin/conda config --system --set auto_update_conda False \ - && ${CONDA_HOME}/bin/conda config --system --set channel_priority strict - -WORKDIR /app -COPY requirements.txt . - -# Install pip packages. -RUN ${PYSPARK_PYTHON} -m pip install --no-cache-dir -r requirements.txt - -# Create the 'spark' group/user. -# The GID and UID must be 1099. Home directory is required. -RUN groupadd -g 1099 spark -RUN useradd -u 1099 -g 1099 -d /home/spark -m spark -USER spark diff --git a/infra/bigquery_export_spark/requirements.txt b/infra/bigquery_export_spark/requirements.txt deleted file mode 100644 index be4c0b9b..00000000 --- a/infra/bigquery_export_spark/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -google-cloud-firestore==2.20.1 -# pyspark==3.5.5 diff --git a/infra/bigquery_export_spark/src/firestore.py b/infra/bigquery_export_spark/src/firestore.py deleted file mode 100644 index 0e6d1e0c..00000000 --- a/infra/bigquery_export_spark/src/firestore.py +++ /dev/null @@ -1,158 +0,0 @@ -"""This module processes Firestore documents from BigQuery using Spark.""" - -import json -import os - -from google.cloud import firestore # type: ignore -from pyspark.sql import SparkSession # type: ignore - -PROJECT = "httparchive" -DATABASE = "tech-report-api" - - -# pylint: disable=too-many-instance-attributes -class FirestoreBatch: - """Handles Firestore data batching from BigQuery using Spark.""" - - def __init__(self, export_config): - """Initialize FirestoreBatch with default settings.""" - self.config = { - "collection_name": export_config["name"], - "date": getattr(export_config, "date", ""), - "collection_type": export_config["type"], - } - self.firestore = firestore.Client(project=PROJECT, database=DATABASE) - self.batch_size = 500 - self.max_concurrent_batches = 200 - self.current_batch = [] - self.batch_promises = [] - self.spark = SparkSession.builder.appName( - "FirestoreBatchProcessor" - ).getOrCreate() - - def queue_batch(self, operation): - """Queue a batch commit operation for Firestore.""" - batch = self.firestore.batch() - - for doc in self.current_batch: - if operation == "delete": - batch.delete(doc.reference) - elif operation == "set": - doc_ref = self.firestore.collection( - self.config["collection_name"] - ).document() - batch.set(doc_ref, doc) - else: - raise ValueError("Invalid operation") - self.batch_promises.append(batch.commit()) - self.current_batch = [] - - def commit_batches(self): - """Commit all queued batch promises.""" - print( - f"Committing {len(self.batch_promises)} " - f"batches to {self.config['collection_name']}" - ) - for batch_promise in self.batch_promises: - try: - batch_promise - except Exception as e: - print(f"Error committing batch: {e}") - raise - self.batch_promises = [] - - def final_flush(self, operation): - """Flush any pending batch operations.""" - if self.current_batch: - self.queue_batch(operation) - if self.batch_promises: - self.commit_batches() - - def batch_delete(self): - """Delete Firestore documents in batches.""" - print("Starting batch deletion...") - start_time = self.spark.sparkContext.startTime - self.current_batch = [] - self.batch_promises = [] - total_docs_deleted = 0 - - collection_ref = self.firestore.collection(self.config["collection_name"]) - if self.config["collection_type"] == "report": - print( - f"Deleting documents from {self.config['collection_name']} " - f"for date {self.config['date']}" - ) - collection_query = collection_ref.where("date", "==", self.config["date"]) - elif self.config["collection_type"] == "dict": - print(f"Deleting documents from {self.config['collection_name']}") - collection_query = collection_ref - else: - raise ValueError("Invalid collection type") - while True: - docs = list( - collection_query.limit( - self.batch_size * self.max_concurrent_batches - ).stream() - ) - if not docs: - break - - for doc in docs: - self.current_batch.append(doc) - if len(self.current_batch) >= self.batch_size: - self.queue_batch("delete") - if len(self.batch_promises) >= self.max_concurrent_batches: - self.commit_batches() - total_docs_deleted += 1 - - self.final_flush("delete") - duration = (self.spark.sparkContext.startTime - start_time) / 1000 - print( - f"Deletion complete. " - f"Total docs deleted: {total_docs_deleted}. " - f"Time: {duration} seconds" - ) - - def stream_from_bigquery(self, query_str): - """Stream data from BigQuery to Firestore.""" - print("Starting BigQuery to Firestore transfer...") - start_time = self.spark.sparkContext.startTime - total_rows_processed = 0 - - df = self.spark.read.format("bigquery").option("query", query_str).load() - - for row in df.collect(): - self.current_batch.append(row.asDict()) - if len(self.current_batch) >= self.batch_size: - self.queue_batch("set") - if len(self.batch_promises) >= self.max_concurrent_batches: - self.commit_batches() - total_rows_processed += 1 - - self.final_flush("set") - duration = (self.spark.sparkContext.startTime - start_time) / 1000 - print( - f"Transfer to {self.config['collection_name']} " - f"complete. " - f"Total rows processed: " - f"{total_rows_processed}. " - f"Time: {duration} " - f"seconds" - ) - - def export(self, query_str): - """Export data from BigQuery to Firestore.""" - - self.batch_delete() - self.stream_from_bigquery(query_str) - - -if __name__ == "__main__": - # config_data = json.loads('{"name": "technologies", "type": "dict", "environment": "dev"}') - # QUERY_STR = str(json.loads("SELECT * FROM report.tech_report_technologies")) - - config_data = json.loads(os.environ["BIGQUERY_PROC_PARAM.export_config"]) - QUERY_STR = str(json.loads(os.environ["BIGQUERY_PROC_PARAM.query"])) - - processor = FirestoreBatch(config_data) - processor.export(QUERY_STR) diff --git a/infra/dataform-export/index.js b/infra/dataform-export/index.js deleted file mode 100644 index fb07269d..00000000 --- a/infra/dataform-export/index.js +++ /dev/null @@ -1,72 +0,0 @@ -import functions from '@google-cloud/functions-framework' -import run from '@google-cloud/run' - -const projectId = 'httparchive' -const location = 'us-central1' -const jobId = 'bigquery-export' - -async function callRunJob (payload = {}) { - const client = new run.v2.JobsClient() - const name = `projects/${projectId}/locations/${location}/jobs/${jobId}` - - const request = { - name, - overrides: { - containerOverrides: [{ - env: [ - { - name: 'EXPORT_CONFIG', - value: JSON.stringify(payload) - } - ] - }] - } - } - - const [operation] = await client.runJob(request) - - console.info(`Job initialized: ${operation.name}`) -} - -function hasRequiredKeys (obj) { - const requiredKeys = ['destination', 'config', 'query'] - return requiredKeys.every(key => key in obj) -} - -/** - * Handle incoming message and trigger the appropriate action. - * - * @param {object} req Cloud Function request context. - * @param {object} res Cloud Function response context. - */ -functions.http('dataform-export', async (req, res) => { - console.log(JSON.stringify(req.body)) - try { - const payload = req.body.calls[0][0] - if (!payload) { - res.status(400).json({ - replies: [400], - errorMessage: 'Bad Request: no payload received, expected JSON object' - }) - } - - if (!hasRequiredKeys(payload)) { - res.status(400).json({ - replies: [400], - errorMessage: 'Bad Request: unexpected payload structure, required keys: dataform_trigger, name, type, environment, (optional)date' - }) - } - - await callRunJob(payload) - - res.status(200).json({ - replies: [200], - message: 'Export job initialized' - }) - } catch (error) { - res.status(400).json({ - replies: [400], - errorMessage: error - }) - } -}) diff --git a/infra/dataform-export/package-lock.json b/infra/dataform-export/package-lock.json deleted file mode 100644 index 49ab9c8a..00000000 --- a/infra/dataform-export/package-lock.json +++ /dev/null @@ -1,2413 +0,0 @@ -{ - "name": "dataform-export", - "version": "1.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "dataform-export", - "version": "1.0.0", - "dependencies": { - "@google-cloud/functions-framework": "4.0.0", - "@google-cloud/run": "2.1.0" - } - }, - "node_modules/@babel/code-frame": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", - "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", - "license": "MIT", - "dependencies": { - "@babel/helper-validator-identifier": "^7.27.1", - "js-tokens": "^4.0.0", - "picocolors": "^1.1.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-identifier": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz", - "integrity": "sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==", - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@google-cloud/functions-framework": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/functions-framework/-/functions-framework-4.0.0.tgz", - "integrity": "sha512-CNcYrz0/hw35Oq0D9RipHUB8KzH4ixq7o12L//qoOg0TFYv4953KrzCo0L2VP++19P39RShKTftDKMFmQhCeEw==", - "license": "Apache-2.0", - "dependencies": { - "@types/express": "^4.17.21", - "body-parser": "1.20.3", - "cloudevents": "^8.0.2", - "express": "^4.21.2", - "minimist": "^1.2.8", - "on-finished": "^2.3.0", - "read-package-up": "^11.0.0", - "semver": "^7.7.1" - }, - "bin": { - "functions-framework": "build/src/main.js", - "functions-framework-nodejs": "build/src/main.js" - }, - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/@google-cloud/run": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@google-cloud/run/-/run-2.1.0.tgz", - "integrity": "sha512-P8/KJDKvY0VlkRL4MnleEgy+hXoXzDFqAjw5LYcXCs5fj0leq3WYtDxL4EQSVEPtVmQ5m92sDEHb5WSnZCH+eA==", - "license": "Apache-2.0", - "dependencies": { - "google-gax": "^5.0.1-rc.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@grpc/grpc-js": { - "version": "1.13.3", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.13.3.tgz", - "integrity": "sha512-FTXHdOoPbZrBjlVLHuKbDZnsTxXv2BlHF57xw6LuThXacXvtkahEPED0CKMk6obZDf65Hv4k3z62eyPNpvinIg==", - "license": "Apache-2.0", - "dependencies": { - "@grpc/proto-loader": "^0.7.13", - "@js-sdsl/ordered-map": "^4.4.2" - }, - "engines": { - "node": ">=12.10.0" - } - }, - "node_modules/@grpc/proto-loader": { - "version": "0.7.13", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.13.tgz", - "integrity": "sha512-AiXO/bfe9bmxBjxxtYxFAXGZvMaN5s8kO+jBHAJCON8rJoB5YS/D6X7ZNc6XQkuHNmyl4CYaMI1fJ/Gn27RGGw==", - "license": "Apache-2.0", - "dependencies": { - "lodash.camelcase": "^4.3.0", - "long": "^5.0.0", - "protobufjs": "^7.2.5", - "yargs": "^17.7.2" - }, - "bin": { - "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/@js-sdsl/ordered-map": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", - "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", - "license": "MIT", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/js-sdsl" - } - }, - "node_modules/@protobufjs/aspromise": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/base64": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/codegen": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", - "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/eventemitter": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/fetch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", - "license": "BSD-3-Clause", - "dependencies": { - "@protobufjs/aspromise": "^1.1.1", - "@protobufjs/inquire": "^1.1.0" - } - }, - "node_modules/@protobufjs/float": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/inquire": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/path": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/utf8": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", - "license": "BSD-3-Clause" - }, - "node_modules/@tootallnate/once": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", - "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", - "license": "MIT", - "engines": { - "node": ">= 10" - } - }, - "node_modules/@types/body-parser": { - "version": "1.19.5", - "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.5.tgz", - "integrity": "sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==", - "license": "MIT", - "dependencies": { - "@types/connect": "*", - "@types/node": "*" - } - }, - "node_modules/@types/caseless": { - "version": "0.12.5", - "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", - "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==", - "license": "MIT" - }, - "node_modules/@types/connect": { - "version": "3.4.38", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", - "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/express": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz", - "integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==", - "license": "MIT", - "dependencies": { - "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.33", - "@types/qs": "*", - "@types/serve-static": "*" - } - }, - "node_modules/@types/express-serve-static-core": { - "version": "4.19.6", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.6.tgz", - "integrity": "sha512-N4LZ2xG7DatVqhCZzOGb1Yi5lMbXSZcmdLDe9EzSndPV2HpWYWzRbaerl2n27irrm94EPpprqa8KpskPT085+A==", - "license": "MIT", - "dependencies": { - "@types/node": "*", - "@types/qs": "*", - "@types/range-parser": "*", - "@types/send": "*" - } - }, - "node_modules/@types/http-errors": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz", - "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==", - "license": "MIT" - }, - "node_modules/@types/long": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@types/long/-/long-5.0.0.tgz", - "integrity": "sha512-eQs9RsucA/LNjnMoJvWG/nXa7Pot/RbBzilF/QRIU/xRl+0ApxrSUFsV5lmf01SvSlqMzJ7Zwxe440wmz2SJGA==", - "deprecated": "This is a stub types definition. long provides its own type definitions, so you do not need this installed.", - "license": "MIT", - "dependencies": { - "long": "*" - } - }, - "node_modules/@types/mime": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", - "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", - "license": "MIT" - }, - "node_modules/@types/node": { - "version": "22.10.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.2.tgz", - "integrity": "sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ==", - "license": "MIT", - "dependencies": { - "undici-types": "~6.20.0" - } - }, - "node_modules/@types/normalize-package-data": { - "version": "2.4.4", - "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz", - "integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==", - "license": "MIT" - }, - "node_modules/@types/qs": { - "version": "6.9.17", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.17.tgz", - "integrity": "sha512-rX4/bPcfmvxHDv0XjfJELTTr+iB+tn032nPILqHm5wbthUUUuVtNGGqzhya9XUxjTP8Fpr0qYgSZZKxGY++svQ==", - "license": "MIT" - }, - "node_modules/@types/range-parser": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", - "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", - "license": "MIT" - }, - "node_modules/@types/request": { - "version": "2.48.12", - "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.12.tgz", - "integrity": "sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==", - "license": "MIT", - "dependencies": { - "@types/caseless": "*", - "@types/node": "*", - "@types/tough-cookie": "*", - "form-data": "^2.5.0" - } - }, - "node_modules/@types/send": { - "version": "0.17.4", - "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", - "integrity": "sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==", - "license": "MIT", - "dependencies": { - "@types/mime": "^1", - "@types/node": "*" - } - }, - "node_modules/@types/serve-static": { - "version": "1.15.7", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.7.tgz", - "integrity": "sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==", - "license": "MIT", - "dependencies": { - "@types/http-errors": "*", - "@types/node": "*", - "@types/send": "*" - } - }, - "node_modules/@types/tough-cookie": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", - "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==", - "license": "MIT" - }, - "node_modules/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "license": "MIT", - "dependencies": { - "event-target-shim": "^5.0.0" - }, - "engines": { - "node": ">=6.5" - } - }, - "node_modules/accepts": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", - "license": "MIT", - "dependencies": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/agent-base": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", - "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/ajv": { - "version": "8.17.1", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", - "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/ajv-formats": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", - "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", - "license": "MIT", - "dependencies": { - "ajv": "^8.0.0" - }, - "peerDependencies": { - "ajv": "^8.0.0" - }, - "peerDependenciesMeta": { - "ajv": { - "optional": true - } - } - }, - "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", - "license": "MIT" - }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "license": "MIT" - }, - "node_modules/available-typed-arrays": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", - "integrity": "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==", - "license": "MIT", - "dependencies": { - "possible-typed-array-names": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/bignumber.js": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", - "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==", - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/body-parser": { - "version": "1.20.3", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", - "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", - "license": "MIT", - "dependencies": { - "bytes": "3.1.2", - "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.13.0", - "raw-body": "2.5.2", - "type-is": "~1.6.18", - "unpipe": "1.0.0" - }, - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, - "node_modules/buffer-equal-constant-time": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", - "license": "BSD-3-Clause" - }, - "node_modules/bytes": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", - "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/call-bind": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", - "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.0", - "es-define-property": "^1.0.0", - "get-intrinsic": "^1.2.4", - "set-function-length": "^1.2.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/cliui": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", - "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "license": "ISC", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.1", - "wrap-ansi": "^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/cloudevents": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/cloudevents/-/cloudevents-8.0.2.tgz", - "integrity": "sha512-93KKRR61D2NNE+2lg2HmLbl17beVTKpf1UYd/8BcXpuiDxbU2fb8gAfriSmVGmj1xX/Oh2t5Fh/xGOWFdu6F4A==", - "license": "Apache-2.0", - "dependencies": { - "ajv": "^8.11.0", - "ajv-formats": "^2.1.1", - "json-bigint": "^1.0.0", - "process": "^0.11.10", - "util": "^0.12.4", - "uuid": "^8.3.2" - }, - "engines": { - "node": ">=16 <=22" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "license": "MIT" - }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", - "license": "MIT", - "dependencies": { - "safe-buffer": "5.2.1" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/content-type": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", - "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/cookie": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", - "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/cookie-signature": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", - "license": "MIT" - }, - "node_modules/data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", - "license": "MIT", - "engines": { - "node": ">= 12" - } - }, - "node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/define-data-property": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", - "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", - "license": "MIT", - "dependencies": { - "es-define-property": "^1.0.0", - "es-errors": "^1.3.0", - "gopd": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/destroy": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", - "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", - "license": "MIT", - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/duplexify": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", - "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", - "license": "MIT", - "dependencies": { - "end-of-stream": "^1.4.1", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1", - "stream-shift": "^1.0.2" - } - }, - "node_modules/ecdsa-sig-formatter": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", - "license": "Apache-2.0", - "dependencies": { - "safe-buffer": "^5.0.1" - } - }, - "node_modules/ee-first": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", - "license": "MIT" - }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "license": "MIT" - }, - "node_modules/encodeurl": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", - "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "license": "MIT", - "dependencies": { - "once": "^1.4.0" - } - }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/escalade": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", - "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/escape-html": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", - "license": "MIT" - }, - "node_modules/etag": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/event-target-shim": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/express": { - "version": "4.21.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", - "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==", - "license": "MIT", - "dependencies": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "1.20.3", - "content-disposition": "0.5.4", - "content-type": "~1.0.4", - "cookie": "0.7.1", - "cookie-signature": "1.0.6", - "debug": "2.6.9", - "depd": "2.0.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "1.3.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "merge-descriptors": "1.0.3", - "methods": "~1.1.2", - "on-finished": "2.4.1", - "parseurl": "~1.3.3", - "path-to-regexp": "0.1.12", - "proxy-addr": "~2.0.7", - "qs": "6.13.0", - "range-parser": "~1.2.1", - "safe-buffer": "5.2.1", - "send": "0.19.0", - "serve-static": "1.16.2", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" - }, - "engines": { - "node": ">= 0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", - "license": "MIT" - }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "license": "MIT" - }, - "node_modules/fast-uri": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.3.tgz", - "integrity": "sha512-aLrHthzCjH5He4Z2H9YZ+v6Ujb9ocRuW6ZzkJQOrTxleEijANq4v1TsaPaVG1PZcuurEzrLcWRyYBYXD5cEiaw==", - "license": "BSD-3-Clause" - }, - "node_modules/fetch-blob": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "paypal", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "dependencies": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - }, - "engines": { - "node": "^12.20 || >= 14.13" - } - }, - "node_modules/finalhandler": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", - "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", - "license": "MIT", - "dependencies": { - "debug": "2.6.9", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "on-finished": "2.4.1", - "parseurl": "~1.3.3", - "statuses": "2.0.1", - "unpipe": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/find-up-simple": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/find-up-simple/-/find-up-simple-1.0.1.tgz", - "integrity": "sha512-afd4O7zpqHeRyg4PfDQsXmlDe2PfdHtJt6Akt8jOWaApLOZk5JXs6VMR29lz03pRe9mpykrRCYIYxaJYcfpncQ==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/for-each": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", - "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", - "license": "MIT", - "dependencies": { - "is-callable": "^1.1.3" - } - }, - "node_modules/form-data": { - "version": "2.5.3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.3.tgz", - "integrity": "sha512-XHIrMD0NpDrNM/Ckf7XJiBbLl57KEhT3+i3yY+eWm+cqYZJQTZrKo8Y8AWKnuV5GT4scfuUGt9LzNoIx3dU1nQ==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "mime-types": "^2.1.35", - "safe-buffer": "^5.2.1" - }, - "engines": { - "node": ">= 0.12" - } - }, - "node_modules/formdata-polyfill": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", - "license": "MIT", - "dependencies": { - "fetch-blob": "^3.1.2" - }, - "engines": { - "node": ">=12.20.0" - } - }, - "node_modules/forwarded": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", - "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/gaxios": { - "version": "7.0.0-rc.4", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.0.0-rc.4.tgz", - "integrity": "sha512-fwQMwbs3o8Odl/nc/rkQJwyHeOXdderOwmybUl0gkyTdZXMK1oSTWj4Em7gSogVJsRWDeHPXLY06+e8Rkr01iw==", - "license": "Apache-2.0", - "dependencies": { - "extend": "^3.0.2", - "https-proxy-agent": "^7.0.1", - "node-fetch": "^3.3.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/gcp-metadata": { - "version": "7.0.0-rc.1", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-7.0.0-rc.1.tgz", - "integrity": "sha512-E6c+AdIaK1LNA839OyotiTca+B2IG1nDlMjnlcck8JjXn3fVgx57Ib9i6iL1/iqN7bA3EUQdcRRu+HqOCOABIg==", - "license": "Apache-2.0", - "dependencies": { - "gaxios": "^7.0.0-rc.1", - "google-logging-utils": "^1.0.0", - "json-bigint": "^1.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "license": "ISC", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/google-auth-library": { - "version": "10.0.0-rc.1", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.0.0-rc.1.tgz", - "integrity": "sha512-Ri8Yk7bMhaPcqzwyW5XHS4scc5KL+AdyUVxA5YGw9BUxYcL2P/tdEfj13O9KpV03k5sUqlaTL+HPxb/deGVjxw==", - "license": "Apache-2.0", - "dependencies": { - "base64-js": "^1.3.0", - "ecdsa-sig-formatter": "^1.0.11", - "gaxios": "^7.0.0-rc.4", - "gcp-metadata": "^7.0.0-rc.1", - "gtoken": "^8.0.0-rc.1", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/google-gax": { - "version": "5.0.1-rc.0", - "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-5.0.1-rc.0.tgz", - "integrity": "sha512-7dfJz21VjUYfVB8XKfXw2ETvjUp1Oa2DUFiNqnJ2k+AY0BDDeqIe4H5ROmLw8dRfv+TwmCL4DJELPRuOh8XAGA==", - "license": "Apache-2.0", - "dependencies": { - "@grpc/grpc-js": "^1.12.6", - "@grpc/proto-loader": "^0.7.13", - "@types/long": "^5.0.0", - "abort-controller": "^3.0.0", - "duplexify": "^4.1.3", - "google-auth-library": "^10.0.0-rc.1", - "google-logging-utils": "^1.1.1", - "node-fetch": "^3.3.2", - "object-hash": "^3.0.0", - "proto3-json-serializer": "^3.0.0", - "protobufjs": "^7.4.0", - "retry-request": "^8.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/google-logging-utils": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.1.tgz", - "integrity": "sha512-rcX58I7nqpu4mbKztFeOAObbomBbHU2oIb/d3tJfF3dizGSApqtSwYJigGCooHdnMyQBIw8BrWyK96w3YXgr6A==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/gtoken": { - "version": "8.0.0-rc.1", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-8.0.0-rc.1.tgz", - "integrity": "sha512-UjE/egX6ixArdcCKOkheuFQ4XN4/0gX92nd2JPVEYuRU2sWHAWuOVGnowm1fQUdQtaxqn1n8H0hOb2LCaUhJ3A==", - "license": "MIT", - "dependencies": { - "gaxios": "^7.0.0-rc.1", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/has-property-descriptors": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", - "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", - "license": "MIT", - "dependencies": { - "es-define-property": "^1.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "license": "MIT", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/hosted-git-info": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-7.0.2.tgz", - "integrity": "sha512-puUZAUKT5m8Zzvs72XWy3HtvVbTWljRE66cP60bxJzAqf2DgICo7lYTY2IHUmLnNpjYvw5bvmoHvPc0QO2a62w==", - "license": "ISC", - "dependencies": { - "lru-cache": "^10.0.1" - }, - "engines": { - "node": "^16.14.0 || >=18.0.0" - } - }, - "node_modules/http-errors": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", - "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", - "license": "MIT", - "dependencies": { - "depd": "2.0.0", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "toidentifier": "1.0.1" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/http-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", - "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", - "license": "MIT", - "dependencies": { - "@tootallnate/once": "2", - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/http-proxy-agent/node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "license": "MIT", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/http-proxy-agent/node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/http-proxy-agent/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/https-proxy-agent": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", - "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/https-proxy-agent/node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/https-proxy-agent/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/index-to-position": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/index-to-position/-/index-to-position-1.1.0.tgz", - "integrity": "sha512-XPdx9Dq4t9Qk1mTMbWONJqU7boCoumEH7fRET37HX5+khDUl3J2W6PdALxhILYlIYx2amlwYcRPp28p0tSiojg==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "license": "ISC" - }, - "node_modules/ipaddr.js": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", - "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", - "license": "MIT", - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/is-arguments": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", - "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", - "license": "MIT", - "dependencies": { - "call-bind": "^1.0.2", - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-callable": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", - "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/is-generator-function": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", - "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", - "license": "MIT", - "dependencies": { - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-typed-array": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.13.tgz", - "integrity": "sha512-uZ25/bUAlUY5fR4OKT4rZQEBrzQWYV9ZJYGGsUmEJ6thodVJ1HX64ePQ6Z0qPWP+m+Uq6e9UugrE38jeYsDSMw==", - "license": "MIT", - "dependencies": { - "which-typed-array": "^1.1.14" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "license": "MIT" - }, - "node_modules/json-bigint": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", - "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", - "license": "MIT", - "dependencies": { - "bignumber.js": "^9.0.0" - } - }, - "node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "license": "MIT" - }, - "node_modules/jwa": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", - "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", - "license": "MIT", - "dependencies": { - "buffer-equal-constant-time": "1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", - "license": "MIT", - "dependencies": { - "jwa": "^2.0.0", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/lodash.camelcase": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", - "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", - "license": "MIT" - }, - "node_modules/long": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/long/-/long-5.3.1.tgz", - "integrity": "sha512-ka87Jz3gcx/I7Hal94xaN2tZEOPoUOEVftkQqZx2EeQRN7LGdfLlI3FvZ+7WDplm+vK2Urx9ULrvSowtdCieng==", - "license": "Apache-2.0" - }, - "node_modules/lru-cache": { - "version": "10.4.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", - "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", - "license": "ISC" - }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/merge-descriptors": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", - "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "license": "MIT", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/minimist": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, - "node_modules/negotiator": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "license": "MIT", - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } - }, - "node_modules/normalize-package-data": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-6.0.2.tgz", - "integrity": "sha512-V6gygoYb/5EmNI+MEGrWkC+e6+Rr7mTmfHrxDbLzxQogBkgzo76rkok0Am6thgSF7Mv2nLOajAJj5vDJZEFn7g==", - "license": "BSD-2-Clause", - "dependencies": { - "hosted-git-info": "^7.0.0", - "semver": "^7.3.5", - "validate-npm-package-license": "^3.0.4" - }, - "engines": { - "node": "^16.14.0 || >=18.0.0" - } - }, - "node_modules/object-hash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", - "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", - "license": "MIT", - "engines": { - "node": ">= 6" - } - }, - "node_modules/object-inspect": { - "version": "1.13.3", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.3.tgz", - "integrity": "sha512-kDCGIbxkDSXE3euJZZXzc6to7fCrKHNI/hSRQnRuQ+BWjFNzZwiFF8fj/6o2t2G9/jTj8PSIYTfCLelLZEeRpA==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/on-finished": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", - "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", - "license": "MIT", - "dependencies": { - "ee-first": "1.1.1" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "license": "ISC", - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/parse-json": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-8.3.0.tgz", - "integrity": "sha512-ybiGyvspI+fAoRQbIPRddCcSTV9/LsJbf0e/S85VLowVGzRmokfneg2kwVW/KU5rOXrPSbF1qAKPMgNTqqROQQ==", - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.26.2", - "index-to-position": "^1.1.0", - "type-fest": "^4.39.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/parseurl": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", - "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/path-to-regexp": { - "version": "0.1.12", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", - "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", - "license": "MIT" - }, - "node_modules/picocolors": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "license": "ISC" - }, - "node_modules/possible-typed-array-names": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz", - "integrity": "sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/process": { - "version": "0.11.10", - "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", - "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", - "license": "MIT", - "engines": { - "node": ">= 0.6.0" - } - }, - "node_modules/proto3-json-serializer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-3.0.0.tgz", - "integrity": "sha512-mHPIc7zaJc26HMpgX5J7vXjliYv4Rnn5ICUyINudz76iY4zFMQHTaQXrTFn0EoHnRsLD6BE+OuHhQHFUU93I9A==", - "license": "Apache-2.0", - "dependencies": { - "protobufjs": "^7.4.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/protobufjs": { - "version": "7.4.0", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz", - "integrity": "sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==", - "hasInstallScript": true, - "license": "BSD-3-Clause", - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/node": ">=13.7.0", - "long": "^5.0.0" - }, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/proxy-addr": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", - "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", - "license": "MIT", - "dependencies": { - "forwarded": "0.2.0", - "ipaddr.js": "1.9.1" - }, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/qs": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", - "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", - "license": "BSD-3-Clause", - "dependencies": { - "side-channel": "^1.0.6" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/range-parser": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", - "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/raw-body": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", - "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", - "license": "MIT", - "dependencies": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/read-package-up": { - "version": "11.0.0", - "resolved": "https://registry.npmjs.org/read-package-up/-/read-package-up-11.0.0.tgz", - "integrity": "sha512-MbgfoNPANMdb4oRBNg5eqLbB2t2r+o5Ua1pNt8BqGp4I0FJZhuVSOj3PaBPni4azWuSzEdNn2evevzVmEk1ohQ==", - "license": "MIT", - "dependencies": { - "find-up-simple": "^1.0.0", - "read-pkg": "^9.0.0", - "type-fest": "^4.6.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/read-pkg": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-9.0.1.tgz", - "integrity": "sha512-9viLL4/n1BJUCT1NXVTdS1jtm80yDEgR5T4yCelII49Mbj0v1rZdKqj7zCiYdbB0CuCgdrvHcNogAKTFPBocFA==", - "license": "MIT", - "dependencies": { - "@types/normalize-package-data": "^2.4.3", - "normalize-package-data": "^6.0.0", - "parse-json": "^8.0.0", - "type-fest": "^4.6.0", - "unicorn-magic": "^0.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "license": "MIT", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/retry-request": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-8.0.0.tgz", - "integrity": "sha512-dJkZNmyV9C8WKUmbdj1xcvVlXBSvsUQCkg89TCK8rD72RdSn9A2jlXlS2VuYSTHoPJjJEfUHhjNYrlvuksF9cg==", - "license": "MIT", - "dependencies": { - "@types/request": "^2.48.12", - "extend": "^3.0.2", - "teeny-request": "^10.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT" - }, - "node_modules/semver": { - "version": "7.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz", - "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==", - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/send": { - "version": "0.19.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", - "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", - "license": "MIT", - "dependencies": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "2.4.1", - "range-parser": "~1.2.1", - "statuses": "2.0.1" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/send/node_modules/encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/send/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/serve-static": { - "version": "1.16.2", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", - "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", - "license": "MIT", - "dependencies": { - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "0.19.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/set-function-length": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", - "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", - "license": "MIT", - "dependencies": { - "define-data-property": "^1.1.4", - "es-errors": "^1.3.0", - "function-bind": "^1.1.2", - "get-intrinsic": "^1.2.4", - "gopd": "^1.0.1", - "has-property-descriptors": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/setprototypeof": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", - "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", - "license": "ISC" - }, - "node_modules/side-channel": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", - "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", - "license": "MIT", - "dependencies": { - "call-bind": "^1.0.7", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.4", - "object-inspect": "^1.13.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/spdx-correct": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.2.0.tgz", - "integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==", - "license": "Apache-2.0", - "dependencies": { - "spdx-expression-parse": "^3.0.0", - "spdx-license-ids": "^3.0.0" - } - }, - "node_modules/spdx-exceptions": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz", - "integrity": "sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==", - "license": "CC-BY-3.0" - }, - "node_modules/spdx-expression-parse": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz", - "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", - "license": "MIT", - "dependencies": { - "spdx-exceptions": "^2.1.0", - "spdx-license-ids": "^3.0.0" - } - }, - "node_modules/spdx-license-ids": { - "version": "3.0.21", - "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.21.tgz", - "integrity": "sha512-Bvg/8F5XephndSK3JffaRqdT+gyhfqIPwDHpX80tJrF8QQRYMo8sNMeaZ2Dp5+jhwKnUmIOyFFQfHRkjJm5nXg==", - "license": "CC0-1.0" - }, - "node_modules/statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/stream-events": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", - "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", - "license": "MIT", - "dependencies": { - "stubs": "^3.0.0" - } - }, - "node_modules/stream-shift": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", - "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==", - "license": "MIT" - }, - "node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "license": "MIT", - "dependencies": { - "safe-buffer": "~5.2.0" - } - }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/stubs": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", - "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==", - "license": "MIT" - }, - "node_modules/teeny-request": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.0.tgz", - "integrity": "sha512-3ZnLvgWF29jikg1sAQ1g0o+lr5JX6sVgYvfUJazn7ZjJroDBUTWp44/+cFVX0bULjv4vci+rBD+oGVAkWqhUbw==", - "license": "Apache-2.0", - "dependencies": { - "http-proxy-agent": "^5.0.0", - "https-proxy-agent": "^5.0.0", - "node-fetch": "^3.3.2", - "stream-events": "^1.0.5" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/teeny-request/node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "license": "MIT", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/teeny-request/node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/teeny-request/node_modules/https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "license": "MIT", - "dependencies": { - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/teeny-request/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/toidentifier": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", - "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", - "license": "MIT", - "engines": { - "node": ">=0.6" - } - }, - "node_modules/type-fest": { - "version": "4.40.1", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.40.1.tgz", - "integrity": "sha512-9YvLNnORDpI+vghLU/Nf+zSv0kL47KbVJ1o3sKgoTefl6i+zebxbiDQWoe/oWWqPhIgQdRZRT1KA9sCPL810SA==", - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", - "license": "MIT", - "dependencies": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/undici-types": { - "version": "6.20.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", - "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", - "license": "MIT" - }, - "node_modules/unicorn-magic": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/unicorn-magic/-/unicorn-magic-0.1.0.tgz", - "integrity": "sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/unpipe": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/util": { - "version": "0.12.5", - "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", - "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", - "license": "MIT", - "dependencies": { - "inherits": "^2.0.3", - "is-arguments": "^1.0.4", - "is-generator-function": "^1.0.7", - "is-typed-array": "^1.1.3", - "which-typed-array": "^1.1.2" - } - }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "license": "MIT" - }, - "node_modules/utils-merge": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", - "license": "MIT", - "engines": { - "node": ">= 0.4.0" - } - }, - "node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/validate-npm-package-license": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", - "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", - "license": "Apache-2.0", - "dependencies": { - "spdx-correct": "^3.0.0", - "spdx-expression-parse": "^3.0.0" - } - }, - "node_modules/vary": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/web-streams-polyfill": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", - "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, - "node_modules/which-typed-array": { - "version": "1.1.16", - "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.16.tgz", - "integrity": "sha512-g+N+GAWiRj66DngFwHvISJd+ITsyphZvD1vChfVg6cEdnzy53GzB3oy0fUNlvhz7H7+MiqhYr26qxQShCpKTTQ==", - "license": "MIT", - "dependencies": { - "available-typed-arrays": "^1.0.7", - "call-bind": "^1.0.7", - "for-each": "^0.3.3", - "gopd": "^1.0.1", - "has-tostringtag": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "license": "ISC" - }, - "node_modules/y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "license": "ISC", - "engines": { - "node": ">=10" - } - }, - "node_modules/yargs": { - "version": "17.7.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", - "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "license": "MIT", - "dependencies": { - "cliui": "^8.0.1", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.3", - "y18n": "^5.0.5", - "yargs-parser": "^21.1.1" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/yargs-parser": { - "version": "21.1.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", - "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "license": "ISC", - "engines": { - "node": ">=12" - } - } - } -} diff --git a/infra/dataform-export/package.json b/infra/dataform-export/package.json deleted file mode 100644 index 3044d85b..00000000 --- a/infra/dataform-export/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "dataform-export", - "version": "1.0.0", - "author": "@max-ostapenko", - "main": "index.js", - "type": "module", - "dependencies": { - "@google-cloud/functions-framework": "4.0.0", - "@google-cloud/run": "2.1.0" - }, - "scripts": { - "start": "npx functions-framework --target=dataform-export --signature-type=http --debug" - } -} diff --git a/infra/dataform-service/.dockerignore b/infra/dataform-service/.dockerignore new file mode 100644 index 00000000..c3bf1e3e --- /dev/null +++ b/infra/dataform-service/.dockerignore @@ -0,0 +1,5 @@ +# Ignore node_modules and other files that shouldn't be in the Docker context +node_modules +.git +.gitignore +.DS_Store diff --git a/infra/dataform-service/Dockerfile b/infra/dataform-service/Dockerfile new file mode 100644 index 00000000..914849c7 --- /dev/null +++ b/infra/dataform-service/Dockerfile @@ -0,0 +1,22 @@ +FROM node:22-slim + +# Set the working directory +WORKDIR /app + +# Copy package files first for better layer caching +COPY package*.json ./ + +# Install dependencies (this layer will be cached unless package files change) +RUN npm ci --only=production --quiet --no-fund --no-audit && npm cache clean --force + +# Copy source code +COPY . . + +# Set default port (Cloud Run will override this) +ENV PORT=8080 + +# Expose port for Cloud Run +EXPOSE 8080 + +# Start the function +CMD ["npm", "start"] diff --git a/infra/dataform-service/README.md b/infra/dataform-service/README.md new file mode 100644 index 00000000..9c92441f --- /dev/null +++ b/infra/dataform-service/README.md @@ -0,0 +1,117 @@ +# Dataform Service + +A unified Cloud Run service that combines the functionality of both dataform-trigger and dataform-export services. This service provides two main endpoints for different operations: + +## Endpoints + +### `/trigger` + +Handles Dataform workflow triggers based on events or polling conditions. + +**Example Request:** + +```json +{ + "message": { + "name": "crux_ready" + } +} +``` + +### `/export` + +Handles BigQuery export job initialization. + +**Example Request:** + +```json +{ + "calls": [[{ + "destination": "...", + "config": "...", + "query": "..." + }]] +} +``` + +## Supported Triggers + +- `crux_ready`: Polls for Chrome UX Report data availability and triggers processing when conditions are met +- `crawl_complete`: Event-based trigger for when crawl data processing is complete + +## Environment Variables + +- `PORT`: Port number for the service (default: 8080) + +## Development + +```bash +# Install dependencies +npm install + +# Start development server +npm run start_dev + +# Build Docker image +npm run build +``` + +## Deployment + +The service is deployed to Google Cloud Run using Cloud Build: + +```bash +gcloud builds submit --config cloudbuild.yaml +``` + +## Migration from Separate Services + +This service replaces the following separate services: + +- `dataform-trigger`: Now accessible via `/trigger` endpoint +- `dataform-export`: Now accessible via `/export` endpoint + +Update your service calls to use the new endpoint paths when migrating. + +```js +// Example usage of the merged dataform-service + +// Trigger example +const triggerPayload = { + message: { + name: "crux_ready" + } +} + +// Export example +const exportPayload = { + calls: [[{ + destination: "gs://httparchive-reports/tech-report-2024", + config: { + format: "PARQUET", + compression: "SNAPPY" + }, + query: "SELECT * FROM httparchive.reports.tech_report_categories WHERE _TABLE_SUFFIX = '2024_01_01'" + }]] +} + +// Usage examples: + +// POST to /trigger endpoint +fetch('https://dataform-service-url/trigger', { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(triggerPayload) +}) + +// POST to /export endpoint +fetch('https://dataform-service-url/export', { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(exportPayload) +}) +``` diff --git a/infra/dataform-service/SUMMARY.md b/infra/dataform-service/SUMMARY.md new file mode 100644 index 00000000..4bcf999b --- /dev/null +++ b/infra/dataform-service/SUMMARY.md @@ -0,0 +1,67 @@ +# Dataform Service Merge - Summary + +## What Was Created + +I've successfully merged the `dataform-trigger` and `dataform-export` services into a unified `dataform-service` with path-based routing. + +### New Service Structure + +``` +/infra/dataform-service/ +├── index.js # Main service with /trigger and /export endpoints +├── dataform.js # Dataform client utilities (converted to ES modules) +├── package.json # Combined dependencies from both services +├── Dockerfile # Docker configuration +├── cloudbuild.yaml # Cloud Build configuration +├── README.md # Service documentation +├── MIGRATION.md # Detailed migration guide +└── examples.js # Usage examples +``` + +### New Terraform Configuration + +``` +/infra/tf/dataform_service/ +├── main.tf # Unified infrastructure configuration +└── variables.tf # Variable definitions +``` + +## Key Features + +### Path-Based Routing +- **`/trigger`** - Handles Dataform workflow triggers (Pub/Sub events, scheduler) +- **`/export`** - Handles BigQuery export jobs (remote functions) + +### Unified Functionality +- Combined all trigger logic (crux_ready, crawl_complete events) +- Combined all export logic (BigQuery to various destinations) +- Shared dependencies and infrastructure +- Single Docker image and deployment + +### Infrastructure Updates +- Pub/Sub subscriptions point to `/trigger` endpoint +- BigQuery remote functions point to `/export` endpoint +- Cloud Scheduler jobs point to `/trigger` endpoint +- Single Cloud Run service with appropriate memory allocation + +## Next Steps + +1. **Deploy the new service:** + ```bash + cd infra/dataform-service + gcloud builds submit --config cloudbuild.yaml + ``` + +2. **Update Terraform to use the new module:** + ```terraform + module "dataform_service" { + source = "./dataform_service" + # ... variables + } + ``` + +3. **Test both endpoints** using the examples in `examples.js` + +4. **Monitor the service** to ensure both functionalities work correctly + +The original services have been marked with migration notes but are preserved for rollback if needed. diff --git a/infra/dataform-service/cloud_run.js b/infra/dataform-service/cloud_run.js new file mode 100644 index 00000000..8fb5ddf0 --- /dev/null +++ b/infra/dataform-service/cloud_run.js @@ -0,0 +1,25 @@ +import run from '@google-cloud/run' + + +// Export functionality +export async function callRunJob(name, payload = {}) { + const client = new run.v2.JobsClient() + + const request = { + name, + overrides: { + containerOverrides: [{ + env: [ + { + name: 'EXPORT_CONFIG', + value: JSON.stringify(payload) + } + ] + }] + } + } + + const [operation] = await client.runJob(request) + + console.info(`Job initialized: ${operation.name}`) +} diff --git a/infra/dataform-trigger/dataform.js b/infra/dataform-service/dataform.js similarity index 84% rename from infra/dataform-trigger/dataform.js rename to infra/dataform-service/dataform.js index 4d873921..0ddabb6b 100644 --- a/infra/dataform-trigger/dataform.js +++ b/infra/dataform-service/dataform.js @@ -1,4 +1,4 @@ -const { DataformClient } = require('@google-cloud/dataform').v1beta1 +import { DataformClient } from '@google-cloud/dataform' const dataformClient = new DataformClient() @@ -8,7 +8,7 @@ const dataformClient = new DataformClient() * @param {string} repoURI Dataform repository URI. * @returns {object} Compilation result. */ -async function getCompilationResults (repoURI) { +export async function getCompilationResults (repoURI) { const request = { parent: repoURI, compilationResult: { @@ -30,7 +30,7 @@ async function getCompilationResults (repoURI) { * @param {object} tags Dataform tags. * @returns */ -async function runWorkflow (repoURI, compilationResult, tags) { +export async function runWorkflow (repoURI, compilationResult, tags) { const request = { parent: repoURI, workflowInvocation: { @@ -48,5 +48,3 @@ async function runWorkflow (repoURI, compilationResult, tags) { const [response] = await dataformClient.createWorkflowInvocation(request) console.info(`Workflow invoked: ${response.name}`) } - -module.exports = { getCompilationResults, runWorkflow } diff --git a/infra/dataform-trigger/index.js b/infra/dataform-service/index.js similarity index 60% rename from infra/dataform-trigger/index.js rename to infra/dataform-service/index.js index 97fd5e90..b9aaefac 100644 --- a/infra/dataform-trigger/index.js +++ b/infra/dataform-service/index.js @@ -1,6 +1,13 @@ -const functions = require('@google-cloud/functions-framework') -const { BigQuery } = require('@google-cloud/bigquery') -const { getCompilationResults, runWorkflow } = require('./dataform') +import functions from '@google-cloud/functions-framework' +import { BigQuery } from '@google-cloud/bigquery' + +import { callRunJob } from './cloud_run.js' +import { getCompilationResults, runWorkflow } from './dataform.js' + + +const projectId = 'httparchive' +const location = 'us-central1' +const jobId = 'bigquery-export' const TRIGGERS = { crux_ready: { @@ -46,13 +53,60 @@ FROM crux, report; } } +function hasRequiredKeys(obj) { + const requiredKeys = ['destination', 'config', 'query'] + return requiredKeys.every(key => key in obj) +} + +/** + * Handle export requests. + * + * @param {object} req Cloud Function request context. + * @param {object} res Cloud Function response context. + */ +async function handleExport(req, res) { + console.log(JSON.stringify(req.body)) + try { + const payload = req.body.calls[0][0] + if (!payload) { + res.status(400).json({ + replies: [400], + errorMessage: 'Bad Request: no payload received, expected JSON object' + }) + return + } + + if (!hasRequiredKeys(payload)) { + res.status(400).json({ + replies: [400], + errorMessage: 'Bad Request: unexpected payload structure, required keys: destination, config, query' + }) + return + } + + const jobName = `projects/${projectId}/locations/${location}/jobs/${jobId}` + await callRunJob(jobName, payload) + + res.status(200).json({ + replies: [200], + message: 'Export job initialized' + }) + } catch (error) { + res.status(400).json({ + replies: [400], + errorMessage: error + }) + } +} + +// Trigger functionality /** - * Handle incoming message and trigger the appropriate action. + * Handle trigger messages and trigger the appropriate action. * * @param {object} req Cloud Function request context. * @param {object} res Cloud Function response context. */ -async function messageHandler (req, res) { +async function handleTrigger(req, res) { try { const message = req.body.message if (!message) { @@ -94,8 +148,9 @@ async function messageHandler (req, res) { } else { console.error(`No action found for event: ${eventName}`) res.status(404).send(`No action found for event: ${eventName}`) + return } - res.status(200).send('Event processed sucessfully') + res.status(200).send('Event processed successfully') } else { console.error(`No action found for event: ${eventName}`) res.status(404).send(`No action found for event: ${eventName}`) @@ -112,7 +167,7 @@ async function messageHandler (req, res) { * @param {string} query Polling query. * @returns {boolean} Query result. */ -async function runQuery (query) { +async function runQuery(query) { const bigquery = new BigQuery() const [job] = await bigquery.createQueryJob({ query }) @@ -128,7 +183,7 @@ async function runQuery (query) { * @param {string} actionName Action to execute. * @param {object} actionArgs Action arguments. */ -async function executeAction (actionName, actionArgs) { +async function executeAction(actionName, actionArgs) { if (actionName === 'runDataformRepo') { console.info(`Executing action: ${actionName}`) await runDataformRepo(actionArgs) @@ -140,7 +195,7 @@ async function executeAction (actionName, actionArgs) { * * @param {object} args Action arguments. */ -async function runDataformRepo (args) { +async function runDataformRepo(args) { const project = 'httparchive' const location = 'us-central1' const { repoName, tags } = args @@ -153,16 +208,53 @@ async function runDataformRepo (args) { } /** - * Trigger function for Dataform workflows. + * Main HTTP handler that routes requests based on path. + * + * @param {object} req Cloud Function request context. + * @param {object} res Cloud Function response context. + */ +async function mainHandler(req, res) { + const path = req.path || req.url + + console.info(`Received request for path: ${path}`) + + if (path === '/trigger' || path.startsWith('/trigger/')) { + await handleTrigger(req, res) + } else if (path === '/') { + await handleExport(req, res) + } else { + res.status(404).json({ + error: 'Not Found', + message: 'Available endpoints: /, /export' + }) + } +} + +/** + * Main entry point for the combined Dataform service. + * Handles both trigger and export operations based on the request path. + * + * Routes: + * - /trigger: Handles Dataform workflow triggers + * - /: Handles BigQuery export jobs * * @param {object} req Cloud Function request context. * @param {object} res Cloud Function response context. * - * Example request payload: + * Example trigger request payload: * { * "message": { * "name": "crux_ready" * } * } + * + * Example export request payload: + * { + * "calls": [[{ + * "destination": "...", + * "config": "...", + * "query": "..." + * }]] + * } */ -functions.http('dataform-trigger', (req, res) => messageHandler(req, res)) +functions.http('dataform-service', mainHandler) diff --git a/infra/dataform-trigger/package-lock.json b/infra/dataform-service/package-lock.json similarity index 84% rename from infra/dataform-trigger/package-lock.json rename to infra/dataform-service/package-lock.json index 450c41eb..3b2a6cfb 100644 --- a/infra/dataform-trigger/package-lock.json +++ b/infra/dataform-service/package-lock.json @@ -1,16 +1,17 @@ { - "name": "dataform-trigger", + "name": "dataform-service", "version": "1.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "dataform-trigger", + "name": "dataform-service", "version": "1.0.0", "dependencies": { "@google-cloud/bigquery": "8.1.0", "@google-cloud/dataform": "2.1.0", - "@google-cloud/functions-framework": "4.0.0" + "@google-cloud/functions-framework": "4.0.0", + "@google-cloud/run": "2.1.0" } }, "node_modules/@babel/code-frame": { @@ -170,10 +171,22 @@ "node": ">=18" } }, + "node_modules/@google-cloud/run": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@google-cloud/run/-/run-2.1.0.tgz", + "integrity": "sha512-P8/KJDKvY0VlkRL4MnleEgy+hXoXzDFqAjw5LYcXCs5fj0leq3WYtDxL4EQSVEPtVmQ5m92sDEHb5WSnZCH+eA==", + "license": "Apache-2.0", + "dependencies": { + "google-gax": "^5.0.1-rc.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@grpc/grpc-js": { - "version": "1.13.3", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.13.3.tgz", - "integrity": "sha512-FTXHdOoPbZrBjlVLHuKbDZnsTxXv2BlHF57xw6LuThXacXvtkahEPED0CKMk6obZDf65Hv4k3z62eyPNpvinIg==", + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.13.4.tgz", + "integrity": "sha512-GsFaMXCkMqkKIvwCQjCrwH+GHbPKBjhwo/8ZuUkWHqbI73Kky9I+pQltrlT0+MWpedCoosda53lgjYfyEPgxBg==", "license": "Apache-2.0", "dependencies": { "@grpc/proto-loader": "^0.7.13", @@ -184,9 +197,9 @@ } }, "node_modules/@grpc/proto-loader": { - "version": "0.7.13", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.13.tgz", - "integrity": "sha512-AiXO/bfe9bmxBjxxtYxFAXGZvMaN5s8kO+jBHAJCON8rJoB5YS/D6X7ZNc6XQkuHNmyl4CYaMI1fJ/Gn27RGGw==", + "version": "0.7.15", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.15.tgz", + "integrity": "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==", "license": "Apache-2.0", "dependencies": { "lodash.camelcase": "^4.3.0", @@ -285,9 +298,9 @@ } }, "node_modules/@types/body-parser": { - "version": "1.19.5", - "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.5.tgz", - "integrity": "sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==", + "version": "1.19.6", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz", + "integrity": "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==", "license": "MIT", "dependencies": { "@types/connect": "*", @@ -310,9 +323,9 @@ } }, "node_modules/@types/express": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz", - "integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==", + "version": "4.17.23", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.23.tgz", + "integrity": "sha512-Crp6WY9aTYP3qPi2wGDo9iUe/rceX01UMhnF1jmwDcKCFM6cx7YhGP/Mpr3y9AASpfHixIG0E6azCcL5OcDHsQ==", "license": "MIT", "dependencies": { "@types/body-parser": "*", @@ -334,21 +347,11 @@ } }, "node_modules/@types/http-errors": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz", - "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==", + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.5.tgz", + "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==", "license": "MIT" }, - "node_modules/@types/long": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@types/long/-/long-5.0.0.tgz", - "integrity": "sha512-eQs9RsucA/LNjnMoJvWG/nXa7Pot/RbBzilF/QRIU/xRl+0ApxrSUFsV5lmf01SvSlqMzJ7Zwxe440wmz2SJGA==", - "deprecated": "This is a stub types definition. long provides its own type definitions, so you do not need this installed.", - "license": "MIT", - "dependencies": { - "long": "*" - } - }, "node_modules/@types/mime": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", @@ -356,12 +359,12 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "22.9.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.9.0.tgz", - "integrity": "sha512-vuyHg81vvWA1Z1ELfvLko2c8f34gyA0zaic0+Rllc5lbCnbSyuvb2Oxpm6TAUAC/2xZN3QGqxBNggD1nNR2AfQ==", + "version": "24.0.10", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.0.10.tgz", + "integrity": "sha512-ENHwaH+JIRTDIEEbDK6QSQntAYGtbvdDXnMXnZaZ6k13Du1dPMmprkEHIL7ok2Wl2aZevetwTAb5S+7yIF+enA==", "license": "MIT", "dependencies": { - "undici-types": "~6.19.8" + "undici-types": "~7.8.0" } }, "node_modules/@types/normalize-package-data": { @@ -371,9 +374,9 @@ "license": "MIT" }, "node_modules/@types/qs": { - "version": "6.9.17", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.17.tgz", - "integrity": "sha512-rX4/bPcfmvxHDv0XjfJELTTr+iB+tn032nPILqHm5wbthUUUuVtNGGqzhya9XUxjTP8Fpr0qYgSZZKxGY++svQ==", + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.14.0.tgz", + "integrity": "sha512-eOunJqu0K1923aExK6y8p6fsihYEn/BYuQ4g0CxAAgFc4b/ZLN4CrsRZ55srTdqoiLzU2B2evC+apEIxprEzkQ==", "license": "MIT" }, "node_modules/@types/range-parser": { @@ -395,9 +398,9 @@ } }, "node_modules/@types/send": { - "version": "0.17.4", - "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", - "integrity": "sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==", + "version": "0.17.5", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.5.tgz", + "integrity": "sha512-z6F2D3cOStZvuk2SaP6YrwkNO65iTZcwA2ZkSABegdkAh/lf+Aa/YQndZVfmEXT5vgAp6zv06VQ3ejSVjAny4w==", "license": "MIT", "dependencies": { "@types/mime": "^1", @@ -405,9 +408,9 @@ } }, "node_modules/@types/serve-static": { - "version": "1.15.7", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.7.tgz", - "integrity": "sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==", + "version": "1.15.8", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.8.tgz", + "integrity": "sha512-roei0UY3LhpOJvjbIP6ZZFngyLKl5dskOtDhxY5THRSpO+ZI+nzJ+m5yUMzGrp89YRa7lvknKkMYjqQFGwA7Sg==", "license": "MIT", "dependencies": { "@types/http-errors": "*", @@ -585,9 +588,9 @@ } }, "node_modules/bignumber.js": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", - "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==", + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.0.tgz", + "integrity": "sha512-EM7aMFTXbptt/wZdMlBv2t8IViwQL+h6SLHosp8Yf0dqJMTnY6iL32opnAB6kAdL0SZPuvcAzFr31o0c/R3/RA==", "license": "MIT", "engines": { "node": "*" @@ -633,16 +636,44 @@ } }, "node_modules/call-bind": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", - "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", + "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", "license": "MIT", "dependencies": { + "call-bind-apply-helpers": "^1.0.0", "es-define-property": "^1.0.0", - "es-errors": "^1.3.0", - "function-bind": "^1.1.2", "get-intrinsic": "^1.2.4", - "set-function-length": "^1.2.1" + "set-function-length": "^1.2.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" }, "engines": { "node": ">= 0.4" @@ -666,9 +697,9 @@ } }, "node_modules/cloudevents": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/cloudevents/-/cloudevents-8.0.2.tgz", - "integrity": "sha512-93KKRR61D2NNE+2lg2HmLbl17beVTKpf1UYd/8BcXpuiDxbU2fb8gAfriSmVGmj1xX/Oh2t5Fh/xGOWFdu6F4A==", + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/cloudevents/-/cloudevents-8.0.3.tgz", + "integrity": "sha512-wTixKNjfLeyj9HQpESvLVVO4xgdqdvX4dTeg1IZ2SCunu/fxVzCamcIZneEyj31V82YolFCKwVeSkr8zResB0Q==", "license": "Apache-2.0", "dependencies": { "ajv": "^8.11.0", @@ -682,15 +713,6 @@ "node": ">=16 <=22" } }, - "node_modules/cloudevents/node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -820,6 +842,20 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/duplexify": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", @@ -863,22 +899,19 @@ } }, "node_modules/end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", "license": "MIT", "dependencies": { "once": "^1.4.0" } }, "node_modules/es-define-property": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", - "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", "license": "MIT", - "dependencies": { - "get-intrinsic": "^1.2.4" - }, "engines": { "node": ">= 0.4" } @@ -892,6 +925,33 @@ "node": ">= 0.4" } }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -984,9 +1044,19 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.3.tgz", - "integrity": "sha512-aLrHthzCjH5He4Z2H9YZ+v6Ujb9ocRuW6ZzkJQOrTxleEijANq4v1TsaPaVG1PZcuurEzrLcWRyYBYXD5cEiaw==", + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.6.tgz", + "integrity": "sha512-Atfo14OibSv5wAp4VWNsFYE1AchQRTv9cBGWET4pZWHzYshFSS9NQI6I57rdKn9croWVMbYFbLhJ+yJvmZIIHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], "license": "BSD-3-Clause" }, "node_modules/fetch-blob": { @@ -1043,23 +1113,30 @@ } }, "node_modules/for-each": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", - "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", + "integrity": "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==", "license": "MIT", "dependencies": { - "is-callable": "^1.1.3" + "is-callable": "^1.2.7" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/form-data": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.2.tgz", - "integrity": "sha512-GgwY0PS7DbXqajuGf4OYlsrIu3zgxD6Vvql43IBhm6MahqA5SK/7mwhtNj2AdH2z35YR34ujJ7BN+3fFC3jP5Q==", + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.3.tgz", + "integrity": "sha512-XHIrMD0NpDrNM/Ckf7XJiBbLl57KEhT3+i3yY+eWm+cqYZJQTZrKo8Y8AWKnuV5GT4scfuUGt9LzNoIx3dU1nQ==", "license": "MIT", "dependencies": { "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "mime-types": "^2.1.35", "safe-buffer": "^5.2.1" }, "engines": { @@ -1106,9 +1183,9 @@ } }, "node_modules/gaxios": { - "version": "7.0.0-rc.4", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.0.0-rc.4.tgz", - "integrity": "sha512-fwQMwbs3o8Odl/nc/rkQJwyHeOXdderOwmybUl0gkyTdZXMK1oSTWj4Em7gSogVJsRWDeHPXLY06+e8Rkr01iw==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.1.tgz", + "integrity": "sha512-Odju3uBUJyVCkW64nLD4wKLhbh93bh6vIg/ZIXkWiLPBrdgtc65+tls/qml+un3pr6JqYVFDZbbmLDQT68rTOQ==", "license": "Apache-2.0", "dependencies": { "extend": "^3.0.2", @@ -1120,12 +1197,12 @@ } }, "node_modules/gcp-metadata": { - "version": "7.0.0-rc.1", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-7.0.0-rc.1.tgz", - "integrity": "sha512-E6c+AdIaK1LNA839OyotiTca+B2IG1nDlMjnlcck8JjXn3fVgx57Ib9i6iL1/iqN7bA3EUQdcRRu+HqOCOABIg==", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-7.0.1.tgz", + "integrity": "sha512-UcO3kefx6dCcZkgcTGgVOTFb7b1LlQ02hY1omMjjrrBzkajRMCFgYOjs7J71WqnuG1k2b+9ppGL7FsOfhZMQKQ==", "license": "Apache-2.0", "dependencies": { - "gaxios": "^7.0.0-rc.1", + "gaxios": "^7.0.0", "google-logging-utils": "^1.0.0", "json-bigint": "^1.0.0" }, @@ -1143,16 +1220,21 @@ } }, "node_modules/get-intrinsic": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", - "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", "license": "MIT", "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", - "has-proto": "^1.0.1", - "has-symbols": "^1.0.3", - "hasown": "^2.0.0" + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" }, "engines": { "node": ">= 0.4" @@ -1161,17 +1243,31 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/google-auth-library": { - "version": "10.0.0-rc.1", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.0.0-rc.1.tgz", - "integrity": "sha512-Ri8Yk7bMhaPcqzwyW5XHS4scc5KL+AdyUVxA5YGw9BUxYcL2P/tdEfj13O9KpV03k5sUqlaTL+HPxb/deGVjxw==", + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.1.0.tgz", + "integrity": "sha512-GspVjZj1RbyRWpQ9FbAXMKjFGzZwDKnUHi66JJ+tcjcu5/xYAP1pdlWotCuIkMwjfVsxxDvsGZXGLzRt72D0sQ==", "license": "Apache-2.0", "dependencies": { "base64-js": "^1.3.0", "ecdsa-sig-formatter": "^1.0.11", - "gaxios": "^7.0.0-rc.4", - "gcp-metadata": "^7.0.0-rc.1", - "gtoken": "^8.0.0-rc.1", + "gaxios": "^7.0.0", + "gcp-metadata": "^7.0.0", + "google-logging-utils": "^1.0.0", + "gtoken": "^8.0.0", "jws": "^4.0.0" }, "engines": { @@ -1179,22 +1275,21 @@ } }, "node_modules/google-gax": { - "version": "5.0.1-rc.0", - "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-5.0.1-rc.0.tgz", - "integrity": "sha512-7dfJz21VjUYfVB8XKfXw2ETvjUp1Oa2DUFiNqnJ2k+AY0BDDeqIe4H5ROmLw8dRfv+TwmCL4DJELPRuOh8XAGA==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-5.0.1.tgz", + "integrity": "sha512-I8fTFXvIG8tYpiDxDXwCXoFsTVsvHJ2GA7DToH+eaRccU8r3nqPMFghVb2GdHSVcu4pq9ScRyB2S1BjO+vsa1Q==", "license": "Apache-2.0", "dependencies": { "@grpc/grpc-js": "^1.12.6", "@grpc/proto-loader": "^0.7.13", - "@types/long": "^5.0.0", "abort-controller": "^3.0.0", "duplexify": "^4.1.3", - "google-auth-library": "^10.0.0-rc.1", + "google-auth-library": "^10.1.0", "google-logging-utils": "^1.1.1", "node-fetch": "^3.3.2", "object-hash": "^3.0.0", "proto3-json-serializer": "^3.0.0", - "protobufjs": "^7.4.0", + "protobufjs": "^7.5.3", "retry-request": "^8.0.0" }, "engines": { @@ -1211,24 +1306,24 @@ } }, "node_modules/gopd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", "license": "MIT", - "dependencies": { - "get-intrinsic": "^1.1.3" + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" } }, "node_modules/gtoken": { - "version": "8.0.0-rc.1", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-8.0.0-rc.1.tgz", - "integrity": "sha512-UjE/egX6ixArdcCKOkheuFQ4XN4/0gX92nd2JPVEYuRU2sWHAWuOVGnowm1fQUdQtaxqn1n8H0hOb2LCaUhJ3A==", + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-8.0.0.tgz", + "integrity": "sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw==", "license": "MIT", "dependencies": { - "gaxios": "^7.0.0-rc.1", + "gaxios": "^7.0.0", "jws": "^4.0.0" }, "engines": { @@ -1247,22 +1342,10 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/has-proto": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", - "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/has-symbols": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", "license": "MIT", "engines": { "node": ">= 0.4" @@ -1369,9 +1452,9 @@ } }, "node_modules/http-proxy-agent/node_modules/debug": { - "version": "4.3.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", - "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -1405,9 +1488,9 @@ } }, "node_modules/https-proxy-agent/node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -1476,13 +1559,13 @@ } }, "node_modules/is-arguments": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", - "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.2.0.tgz", + "integrity": "sha512-7bVbi0huj/wrIAOzb8U1aszg9kdi3KN/CyU19CTI7tAoZYEZoL9yCDXpbXN+uPsuWnP02cyug1gleqq+TU+YCA==", "license": "MIT", "dependencies": { - "call-bind": "^1.0.2", - "has-tostringtag": "^1.0.0" + "call-bound": "^1.0.2", + "has-tostringtag": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -1513,12 +1596,33 @@ } }, "node_modules/is-generator-function": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", - "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.1.0.tgz", + "integrity": "sha512-nPUB5km40q9e8UfN/Zc24eLlzdSf9OfKByBw9CIdw4H1giPMeA0OIJvbchsCu4npfI2QcMVBsGEBHKZ7wLTWmQ==", "license": "MIT", "dependencies": { - "has-tostringtag": "^1.0.0" + "call-bound": "^1.0.3", + "get-proto": "^1.0.0", + "has-tostringtag": "^1.0.2", + "safe-regex-test": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-regex": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", + "integrity": "sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" }, "engines": { "node": ">= 0.4" @@ -1528,12 +1632,12 @@ } }, "node_modules/is-typed-array": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.13.tgz", - "integrity": "sha512-uZ25/bUAlUY5fR4OKT4rZQEBrzQWYV9ZJYGGsUmEJ6thodVJ1HX64ePQ6Z0qPWP+m+Uq6e9UugrE38jeYsDSMw==", + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.15.tgz", + "integrity": "sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ==", "license": "MIT", "dependencies": { - "which-typed-array": "^1.1.14" + "which-typed-array": "^1.1.16" }, "engines": { "node": ">= 0.4" @@ -1564,12 +1668,12 @@ "license": "MIT" }, "node_modules/jwa": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", - "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", "license": "MIT", "dependencies": { - "buffer-equal-constant-time": "1.0.1", + "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } @@ -1591,9 +1695,9 @@ "license": "MIT" }, "node_modules/long": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/long/-/long-5.3.1.tgz", - "integrity": "sha512-ka87Jz3gcx/I7Hal94xaN2tZEOPoUOEVftkQqZx2EeQRN7LGdfLlI3FvZ+7WDplm+vK2Urx9ULrvSowtdCieng==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", "license": "Apache-2.0" }, "node_modules/lru-cache": { @@ -1602,6 +1706,15 @@ "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", "license": "ISC" }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -1690,6 +1803,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", "funding": [ { "type": "github", @@ -1747,9 +1861,9 @@ } }, "node_modules/object-inspect": { - "version": "1.13.3", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.3.tgz", - "integrity": "sha512-kDCGIbxkDSXE3euJZZXzc6to7fCrKHNI/hSRQnRuQ+BWjFNzZwiFF8fj/6o2t2G9/jTj8PSIYTfCLelLZEeRpA==", + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", "license": "MIT", "engines": { "node": ">= 0.4" @@ -1818,9 +1932,9 @@ "license": "ISC" }, "node_modules/possible-typed-array-names": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz", - "integrity": "sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", + "integrity": "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==", "license": "MIT", "engines": { "node": ">= 0.4" @@ -1836,9 +1950,9 @@ } }, "node_modules/proto3-json-serializer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-3.0.0.tgz", - "integrity": "sha512-mHPIc7zaJc26HMpgX5J7vXjliYv4Rnn5ICUyINudz76iY4zFMQHTaQXrTFn0EoHnRsLD6BE+OuHhQHFUU93I9A==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-3.0.1.tgz", + "integrity": "sha512-Rug90pDIefARAG9MgaFjd0yR/YP4bN3Fov00kckXMjTZa0x86c4WoWfCQFdSeWi9DvRXjhfLlPDIvODB5LOTfg==", "license": "Apache-2.0", "dependencies": { "protobufjs": "^7.4.0" @@ -1848,9 +1962,9 @@ } }, "node_modules/protobufjs": { - "version": "7.4.0", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz", - "integrity": "sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==", + "version": "7.5.3", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.3.tgz", + "integrity": "sha512-sildjKwVqOI2kmFDiXQ6aEB0fjYTafpEvIBs8tOR8qI4spuL9OPROLVu2qZqi/xgCfsHIwVqlaF8JBjWFHnKbw==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -2025,6 +2139,23 @@ ], "license": "MIT" }, + "node_modules/safe-regex-test": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.1.0.tgz", + "integrity": "sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "is-regex": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -2032,9 +2163,9 @@ "license": "MIT" }, "node_modules/semver": { - "version": "7.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz", - "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==", + "version": "7.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", + "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -2121,15 +2252,69 @@ "license": "ISC" }, "node_modules/side-channel": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", - "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", "license": "MIT", "dependencies": { - "call-bind": "^1.0.7", "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.4", - "object-inspect": "^1.13.1" + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" }, "engines": { "node": ">= 0.4" @@ -2263,9 +2448,9 @@ } }, "node_modules/teeny-request/node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -2308,9 +2493,9 @@ } }, "node_modules/type-fest": { - "version": "4.40.1", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.40.1.tgz", - "integrity": "sha512-9YvLNnORDpI+vghLU/Nf+zSv0kL47KbVJ1o3sKgoTefl6i+zebxbiDQWoe/oWWqPhIgQdRZRT1KA9sCPL810SA==", + "version": "4.41.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", + "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", "license": "(MIT OR CC0-1.0)", "engines": { "node": ">=16" @@ -2333,9 +2518,9 @@ } }, "node_modules/undici-types": { - "version": "6.19.8", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", - "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.8.0.tgz", + "integrity": "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw==", "license": "MIT" }, "node_modules/unicorn-magic": { @@ -2387,6 +2572,15 @@ "node": ">= 0.4.0" } }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/validate-npm-package-license": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", @@ -2416,15 +2610,17 @@ } }, "node_modules/which-typed-array": { - "version": "1.1.15", - "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.15.tgz", - "integrity": "sha512-oV0jmFtUky6CXfkqehVvBP/LSWJ2sy4vWMioiENyJLePrBO/yKyV9OyJySfAKosh+RYkIl5zJCNZ8/4JncrpdA==", + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.19.tgz", + "integrity": "sha512-rEvr90Bck4WZt9HHFC4DJMsjvu7x+r6bImz0/BrbWb7A2djJ8hnZMrWnHo9F8ssv0OMErasDhftrfROTyqSDrw==", "license": "MIT", "dependencies": { "available-typed-arrays": "^1.0.7", - "call-bind": "^1.0.7", - "for-each": "^0.3.3", - "gopd": "^1.0.1", + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", + "for-each": "^0.3.5", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", "has-tostringtag": "^1.0.2" }, "engines": { diff --git a/infra/dataform-service/package.json b/infra/dataform-service/package.json new file mode 100644 index 00000000..0ba530a6 --- /dev/null +++ b/infra/dataform-service/package.json @@ -0,0 +1,18 @@ +{ + "name": "dataform-service", + "version": "1.0.0", + "author": "@max-ostapenko", + "main": "index.js", + "type": "module", + "dependencies": { + "@google-cloud/bigquery": "8.1.0", + "@google-cloud/dataform": "2.1.0", + "@google-cloud/functions-framework": "4.0.0", + "@google-cloud/run": "2.1.0" + }, + "scripts": { + "start": "npx functions-framework --target=dataform-service --signature-type=http --port=${PORT:-8080}", + "start_dev": "npx functions-framework --target=dataform-service --signature-type=http --port=8080 --debug", + "build": "docker build -t dataform-service ." + } +} diff --git a/infra/dataform-trigger/package.json b/infra/dataform-trigger/package.json deleted file mode 100644 index e96ddce4..00000000 --- a/infra/dataform-trigger/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "dataform-trigger", - "version": "1.0.0", - "author": "@max-ostapenko", - "main": "index.js", - "dependencies": { - "@google-cloud/bigquery": "8.1.0", - "@google-cloud/dataform": "2.1.0", - "@google-cloud/functions-framework": "4.0.0" - }, - "scripts": { - "start": "npx functions-framework --target=dataform-trigger --signature-type=http --port=8080 --debug" - } -} diff --git a/infra/tf/.terraform.lock.hcl b/infra/tf/.terraform.lock.hcl index 2820bfce..17a9237e 100644 --- a/infra/tf/.terraform.lock.hcl +++ b/infra/tf/.terraform.lock.hcl @@ -1,26 +1,6 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. -provider "registry.terraform.io/hashicorp/archive" { - version = "2.7.1" - constraints = ">= 2.6.0" - hashes = [ - "h1:A7EnRBVm4h9ryO9LwxYnKr4fy7ExPMwD5a1DsY7m1Y0=", - "zh:19881bb356a4a656a865f48aee70c0b8a03c35951b7799b6113883f67f196e8e", - "zh:2fcfbf6318dd514863268b09bbe19bfc958339c636bcbcc3664b45f2b8bf5cc6", - "zh:3323ab9a504ce0a115c28e64d0739369fe85151291a2ce480d51ccbb0c381ac5", - "zh:362674746fb3da3ab9bd4e70c75a3cdd9801a6cf258991102e2c46669cf68e19", - "zh:7140a46d748fdd12212161445c46bbbf30a3f4586c6ac97dd497f0c2565fe949", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:875e6ce78b10f73b1efc849bfcc7af3a28c83a52f878f503bb22776f71d79521", - "zh:b872c6ed24e38428d817ebfb214da69ea7eefc2c38e5a774db2ccd58e54d3a22", - "zh:cd6a44f731c1633ae5d37662af86e7b01ae4c96eb8b04144255824c3f350392d", - "zh:e0600f5e8da12710b0c52d6df0ba147a5486427c1a2cc78f31eea37a47ee1b07", - "zh:f21b2e2563bbb1e44e73557bcd6cdbc1ceb369d471049c40eb56cb84b6317a60", - "zh:f752829eba1cc04a479cf7ae7271526b402e206d5bcf1fcce9f535de5ff9e4e6", - ] -} - provider "registry.terraform.io/hashicorp/google" { version = "6.42.0" constraints = ">= 6.13.0, >= 6.40.0" @@ -60,3 +40,24 @@ provider "registry.terraform.io/hashicorp/google-beta" { "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } + +provider "registry.terraform.io/kreuzwerker/docker" { + version = "3.6.2" + constraints = ">= 3.0.2" + hashes = [ + "h1:/Oe7tViXf/xyQ4Pg8cDifMlD3RthOYkslwQiRgx7BTE=", + "zh:22b51a8fb63481d290bdad9a221bc8c9e45d66d1a0cd45beed3f3627bf1debd8", + "zh:2b902eb80a1ae033af1135cc165d192668820a7f8ea15beb5472f811c18bea1f", + "zh:57815dcea28aedb86ed33924cd186aaee8bd31670bd78437a2a2daf2b00ce2ae", + "zh:583af9c6fe7e3bfc04f50aec046a9b4f98b7eddd6d1e143454e5d06a66afcf87", + "zh:80f8cba54f639a53c4d7714edb7246064b7f4f48ba93a70f18c914d656d799db", + "zh:894709f0c393c4ee91fdb849128e7f0bce688f293cd1643a6d4e39c842367278", + "zh:a91b41dbcb203d6dae2bb72b98c4c21c41255026b35df01895882784c4650071", + "zh:aec40a8157aae093412a1fb9a71ab2bea370db152e285c2d81e37ed378444b9c", + "zh:b87d7def2485dde6e57723c1265158f371440a8a84954c9fdb0580cf89de66bf", + "zh:b9dc243200ad9cd00250cb8c793ecea4ee3c57a121faf8efdb289f30008b5778", + "zh:dcb103831db6d3ef95468685cd104be3928793996542a1f675dc34a2ce67951d", + "zh:e59b4a0f2b5881016896d4417b1ab2fb87f34450663efeb01f3bcf7c3606fbbb", + "zh:fbd068c01114f0712578cf02f363b5521338ab1befedddf7090da532298b43d0", + ] +} diff --git a/infra/tf/bigquery_export/docker.tf b/infra/tf/bigquery_export/docker.tf new file mode 100644 index 00000000..7e104bc6 --- /dev/null +++ b/infra/tf/bigquery_export/docker.tf @@ -0,0 +1,41 @@ + +# Get current Google Cloud access token +data "google_client_config" "default" {} + +# Configure Docker provider with Artifact Registry authentication +provider "docker" { + registry_auth { + address = "${var.region}-docker.pkg.dev" + username = "oauth2accesstoken" + password = data.google_client_config.default.access_token + } +} + +# Calculate hash of source files to determine if rebuild is needed +locals { + source_files = fileset(path.root, "../${var.function_name}/*") + source_hash = sha1(join("", [for f in local.source_files : filesha1(f)])) +} + +# Build Docker image +resource "docker_image" "function_image" { + name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}:latest" + + build { + context = "../${var.function_name}/" + dockerfile = "Dockerfile" + platform = "linux/amd64" + } + + triggers = { + source_hash = local.source_hash + } +} + +resource "docker_registry_image" "registry_image" { + name = docker_image.function_image.name + + triggers = { + source_hash = local.source_hash + } +} diff --git a/infra/tf/bigquery_export/main.tf b/infra/tf/bigquery_export/main.tf index 7c753439..c0038518 100644 --- a/infra/tf/bigquery_export/main.tf +++ b/infra/tf/bigquery_export/main.tf @@ -2,9 +2,9 @@ terraform { required_version = ">= 1.9.7" required_providers { - archive = { - source = "hashicorp/archive" - version = ">= 2.6.0" + docker = { + source = "kreuzwerker/docker" + version = ">= 3.6.2" } google = { source = "hashicorp/google" @@ -29,11 +29,11 @@ resource "google_cloud_run_v2_job" "bigquery_export" { max_retries = 0 # No retries containers { - image = "${var.location}.gcr.io/${var.project}/cloud-run/${var.function_name}:latest" + image = docker_registry_image.registry_image.name resources { limits = { cpu = "4" - memory = "4Gi" + memory = "8Gi" } } env { diff --git a/infra/tf/dataform_export/main.tf b/infra/tf/dataform_export/main.tf deleted file mode 100644 index 79949554..00000000 --- a/infra/tf/dataform_export/main.tf +++ /dev/null @@ -1,82 +0,0 @@ -terraform { - required_version = ">= 1.9.7" - - required_providers { - archive = { - source = "hashicorp/archive" - version = ">= 2.6.0" - } - google = { - source = "hashicorp/google" - version = ">= 6.13.0" - } - } -} - -data "archive_file" "zip" { - type = "zip" - source_dir = "../${var.function_name}/" - output_path = "./tmp/${var.function_name}.zip" -} - -resource "google_storage_bucket_object" "source" { - bucket = "gcf-v2-uploads-${var.project_number}-${var.region}" - name = "${var.function_name}_${data.archive_file.zip.id}.zip" - source = data.archive_file.zip.output_path -} - -resource "google_cloudfunctions2_function" "dataform_export" { - name = var.function_name - location = var.region - build_config { - runtime = "nodejs20" - entry_point = var.function_name - source { - storage_source { - bucket = google_storage_bucket_object.source.bucket - object = google_storage_bucket_object.source.name - generation = google_storage_bucket_object.source.generation - } - } - } - service_config { - max_instance_count = 2 - available_cpu = 1 - available_memory = "256M" - service_account_email = var.function_identity - ingress_settings = "ALLOW_INTERNAL_ONLY" - } -} - -resource "google_bigquery_routine" "run_export_job" { - dataset_id = "reports" - routine_id = "run_export_job" - routine_type = "SCALAR_FUNCTION" - definition_body = "" - description = < Date: Sun, 6 Jul 2025 18:49:40 +0200 Subject: [PATCH 3/6] lint --- Makefile | 6 ------ infra/dataform-service/SUMMARY.md => SUMMARY.md | 9 +++++++-- infra/dataform-service/cloud_run.js | 3 +-- infra/dataform-service/index.js | 15 +++++++-------- infra/tf/bigquery_export/docker.tf | 4 ++-- infra/tf/bigquery_export/main.tf | 10 +++++----- infra/tf/dataform_service/docker.tf | 4 ++-- infra/tf/dataform_service/export_resources.tf | 2 +- infra/tf/dataform_service/main.tf | 2 +- infra/tf/dataform_service/trigger_resources.tf | 2 +- infra/tf/main.tf | 10 +++++----- 11 files changed, 32 insertions(+), 35 deletions(-) rename infra/dataform-service/SUMMARY.md => SUMMARY.md (98%) diff --git a/Makefile b/Makefile index 515af2cb..b9f45b89 100644 --- a/Makefile +++ b/Makefile @@ -11,9 +11,3 @@ tf_plan: tf_apply: terraform -chdir=infra/tf init && terraform -chdir=infra/tf apply -auto-approve - -bigquery_export_deploy: - cd infra/bigquery-export && npm run build - -#bigquery_export_spark_deploy: -# cd infra/bigquery_export_spark && gcloud builds submit --region=global --tag us-docker.pkg.dev/httparchive/bigquery-spark-procedures/firestore_export:latest diff --git a/infra/dataform-service/SUMMARY.md b/SUMMARY.md similarity index 98% rename from infra/dataform-service/SUMMARY.md rename to SUMMARY.md index 4bcf999b..0f1d8425 100644 --- a/infra/dataform-service/SUMMARY.md +++ b/SUMMARY.md @@ -6,7 +6,7 @@ I've successfully merged the `dataform-trigger` and `dataform-export` services i ### New Service Structure -``` +```files /infra/dataform-service/ ├── index.js # Main service with /trigger and /export endpoints ├── dataform.js # Dataform client utilities (converted to ES modules) @@ -20,7 +20,7 @@ I've successfully merged the `dataform-trigger` and `dataform-export` services i ### New Terraform Configuration -``` +```files /infra/tf/dataform_service/ ├── main.tf # Unified infrastructure configuration └── variables.tf # Variable definitions @@ -29,16 +29,19 @@ I've successfully merged the `dataform-trigger` and `dataform-export` services i ## Key Features ### Path-Based Routing + - **`/trigger`** - Handles Dataform workflow triggers (Pub/Sub events, scheduler) - **`/export`** - Handles BigQuery export jobs (remote functions) ### Unified Functionality + - Combined all trigger logic (crux_ready, crawl_complete events) - Combined all export logic (BigQuery to various destinations) - Shared dependencies and infrastructure - Single Docker image and deployment ### Infrastructure Updates + - Pub/Sub subscriptions point to `/trigger` endpoint - BigQuery remote functions point to `/export` endpoint - Cloud Scheduler jobs point to `/trigger` endpoint @@ -47,12 +50,14 @@ I've successfully merged the `dataform-trigger` and `dataform-export` services i ## Next Steps 1. **Deploy the new service:** + ```bash cd infra/dataform-service gcloud builds submit --config cloudbuild.yaml ``` 2. **Update Terraform to use the new module:** + ```terraform module "dataform_service" { source = "./dataform_service" diff --git a/infra/dataform-service/cloud_run.js b/infra/dataform-service/cloud_run.js index 8fb5ddf0..9700ef27 100644 --- a/infra/dataform-service/cloud_run.js +++ b/infra/dataform-service/cloud_run.js @@ -1,8 +1,7 @@ import run from '@google-cloud/run' - // Export functionality -export async function callRunJob(name, payload = {}) { +export async function callRunJob (name, payload = {}) { const client = new run.v2.JobsClient() const request = { diff --git a/infra/dataform-service/index.js b/infra/dataform-service/index.js index b9aaefac..1668c3ad 100644 --- a/infra/dataform-service/index.js +++ b/infra/dataform-service/index.js @@ -4,7 +4,6 @@ import { BigQuery } from '@google-cloud/bigquery' import { callRunJob } from './cloud_run.js' import { getCompilationResults, runWorkflow } from './dataform.js' - const projectId = 'httparchive' const location = 'us-central1' const jobId = 'bigquery-export' @@ -53,7 +52,7 @@ FROM crux, report; } } -function hasRequiredKeys(obj) { +function hasRequiredKeys (obj) { const requiredKeys = ['destination', 'config', 'query'] return requiredKeys.every(key => key in obj) } @@ -64,7 +63,7 @@ function hasRequiredKeys(obj) { * @param {object} req Cloud Function request context. * @param {object} res Cloud Function response context. */ -async function handleExport(req, res) { +async function handleExport (req, res) { console.log(JSON.stringify(req.body)) try { const payload = req.body.calls[0][0] @@ -106,7 +105,7 @@ async function handleExport(req, res) { * @param {object} req Cloud Function request context. * @param {object} res Cloud Function response context. */ -async function handleTrigger(req, res) { +async function handleTrigger (req, res) { try { const message = req.body.message if (!message) { @@ -167,7 +166,7 @@ async function handleTrigger(req, res) { * @param {string} query Polling query. * @returns {boolean} Query result. */ -async function runQuery(query) { +async function runQuery (query) { const bigquery = new BigQuery() const [job] = await bigquery.createQueryJob({ query }) @@ -183,7 +182,7 @@ async function runQuery(query) { * @param {string} actionName Action to execute. * @param {object} actionArgs Action arguments. */ -async function executeAction(actionName, actionArgs) { +async function executeAction (actionName, actionArgs) { if (actionName === 'runDataformRepo') { console.info(`Executing action: ${actionName}`) await runDataformRepo(actionArgs) @@ -195,7 +194,7 @@ async function executeAction(actionName, actionArgs) { * * @param {object} args Action arguments. */ -async function runDataformRepo(args) { +async function runDataformRepo (args) { const project = 'httparchive' const location = 'us-central1' const { repoName, tags } = args @@ -213,7 +212,7 @@ async function runDataformRepo(args) { * @param {object} req Cloud Function request context. * @param {object} res Cloud Function response context. */ -async function mainHandler(req, res) { +async function mainHandler (req, res) { const path = req.path || req.url console.info(`Received request for path: ${path}`) diff --git a/infra/tf/bigquery_export/docker.tf b/infra/tf/bigquery_export/docker.tf index 7e104bc6..6e01d41d 100644 --- a/infra/tf/bigquery_export/docker.tf +++ b/infra/tf/bigquery_export/docker.tf @@ -14,7 +14,7 @@ provider "docker" { # Calculate hash of source files to determine if rebuild is needed locals { source_files = fileset(path.root, "../${var.function_name}/*") - source_hash = sha1(join("", [for f in local.source_files : filesha1(f)])) + source_hash = sha1(join("", [for f in local.source_files : filesha1(f)])) } # Build Docker image @@ -33,7 +33,7 @@ resource "docker_image" "function_image" { } resource "docker_registry_image" "registry_image" { - name = docker_image.function_image.name + name = docker_image.function_image.name triggers = { source_hash = local.source_hash diff --git a/infra/tf/bigquery_export/main.tf b/infra/tf/bigquery_export/main.tf index c0038518..14c79b64 100644 --- a/infra/tf/bigquery_export/main.tf +++ b/infra/tf/bigquery_export/main.tf @@ -21,12 +21,12 @@ resource "google_cloud_run_v2_job" "bigquery_export" { template { parallelism = 1 - task_count = 1 # Ensure single task execution + task_count = 1 # Ensure single task execution template { - timeout = "10800s" # 3 hours + timeout = "10800s" # 3 hours service_account = var.function_identity - max_retries = 0 # No retries + max_retries = 0 # No retries containers { image = docker_registry_image.registry_image.name @@ -42,12 +42,12 @@ resource "google_cloud_run_v2_job" "bigquery_export" { } env { name = "NODE_OPTIONS" - value = "--expose-gc --max-old-space-size=6144" # 6GB heap limit with GC + value = "--expose-gc --max-old-space-size=6144" # 6GB heap limit with GC } env { name = "MEMORY_WARNING_THRESHOLD_MB" - value = "4915" # 80% of max heap size = 6144MB + value = "4915" # 80% of max heap size = 6144MB } env { diff --git a/infra/tf/dataform_service/docker.tf b/infra/tf/dataform_service/docker.tf index 7e104bc6..6e01d41d 100644 --- a/infra/tf/dataform_service/docker.tf +++ b/infra/tf/dataform_service/docker.tf @@ -14,7 +14,7 @@ provider "docker" { # Calculate hash of source files to determine if rebuild is needed locals { source_files = fileset(path.root, "../${var.function_name}/*") - source_hash = sha1(join("", [for f in local.source_files : filesha1(f)])) + source_hash = sha1(join("", [for f in local.source_files : filesha1(f)])) } # Build Docker image @@ -33,7 +33,7 @@ resource "docker_image" "function_image" { } resource "docker_registry_image" "registry_image" { - name = docker_image.function_image.name + name = docker_image.function_image.name triggers = { source_hash = local.source_hash diff --git a/infra/tf/dataform_service/export_resources.tf b/infra/tf/dataform_service/export_resources.tf index d66cf650..28c7b258 100644 --- a/infra/tf/dataform_service/export_resources.tf +++ b/infra/tf/dataform_service/export_resources.tf @@ -39,7 +39,7 @@ EOT return_type = "{\"typeKind\" : \"INT64\"}" remote_function_options { - endpoint = "${google_cloud_run_v2_service.dataform_service.uri}" + endpoint = google_cloud_run_v2_service.dataform_service.uri connection = google_bigquery_connection.cloud-resources.id max_batching_rows = "1" } diff --git a/infra/tf/dataform_service/main.tf b/infra/tf/dataform_service/main.tf index d10604d8..d02bd3fe 100644 --- a/infra/tf/dataform_service/main.tf +++ b/infra/tf/dataform_service/main.tf @@ -45,7 +45,7 @@ resource "google_cloud_run_v2_service" "dataform_service" { percent = 100 } - depends_on = [ docker_registry_image.registry_image ] + depends_on = [docker_registry_image.registry_image] } resource "google_cloud_run_service_iam_member" "member" { diff --git a/infra/tf/dataform_service/trigger_resources.tf b/infra/tf/dataform_service/trigger_resources.tf index 2571ace2..001de696 100644 --- a/infra/tf/dataform_service/trigger_resources.tf +++ b/infra/tf/dataform_service/trigger_resources.tf @@ -42,7 +42,7 @@ EOF resource "google_cloud_scheduler_job" "bq-poller-crux-ready" { attempt_deadline = "180s" - region = var.region + region = var.region description = null name = "bq-poller-crux-ready" paused = false diff --git a/infra/tf/main.tf b/infra/tf/main.tf index dc3bb50f..9ddad8a8 100644 --- a/infra/tf/main.tf +++ b/infra/tf/main.tf @@ -49,11 +49,11 @@ module "functions" { module "dataform_service" { source = "./dataform_service" - project = local.project - region = local.region - location = local.location - function_identity = local.function_identity - function_name = "dataform-service" + project = local.project + region = local.region + location = local.location + function_identity = local.function_identity + function_name = "dataform-service" } module "masthead_agent" { From be8b26b4e2943a9defa3746d64e3066b80c62605 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 6 Jul 2025 18:56:32 +0200 Subject: [PATCH 4/6] lint --- SUMMARY.md | 20 ++++++++++---------- infra/tf/bigquery_export/variables.tf | 4 ---- infra/tf/main.tf | 1 - 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/SUMMARY.md b/SUMMARY.md index 0f1d8425..e8ed3b6a 100644 --- a/SUMMARY.md +++ b/SUMMARY.md @@ -51,19 +51,19 @@ I've successfully merged the `dataform-trigger` and `dataform-export` services i 1. **Deploy the new service:** - ```bash - cd infra/dataform-service - gcloud builds submit --config cloudbuild.yaml - ``` + ```bash + cd infra/dataform-service + gcloud builds submit --config cloudbuild.yaml + ``` 2. **Update Terraform to use the new module:** - ```terraform - module "dataform_service" { - source = "./dataform_service" - # ... variables - } - ``` + ```terraform + module "dataform_service" { + source = "./dataform_service" + # ... variables + } + ``` 3. **Test both endpoints** using the examples in `examples.js` diff --git a/infra/tf/bigquery_export/variables.tf b/infra/tf/bigquery_export/variables.tf index da7dd2a5..91b79b54 100644 --- a/infra/tf/bigquery_export/variables.tf +++ b/infra/tf/bigquery_export/variables.tf @@ -13,7 +13,3 @@ variable "function_identity" { variable "function_name" { type = string } - -variable "location" { - type = string -} diff --git a/infra/tf/main.tf b/infra/tf/main.tf index 9ddad8a8..03e2dc18 100644 --- a/infra/tf/main.tf +++ b/infra/tf/main.tf @@ -34,7 +34,6 @@ module "bigquery_export" { project = local.project region = local.region - location = local.location function_identity = local.function_identity function_name = "bigquery-export" } From 7a955a672ff41b66b146dc54a1916fe4c458367c Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 6 Jul 2025 20:41:42 +0200 Subject: [PATCH 5/6] unique tag --- infra/tf/bigquery_export/docker.tf | 12 ++---------- infra/tf/dataform_service/docker.tf | 12 ++---------- infra/tf/dataform_service/main.tf | 4 +--- 3 files changed, 5 insertions(+), 23 deletions(-) diff --git a/infra/tf/bigquery_export/docker.tf b/infra/tf/bigquery_export/docker.tf index 6e01d41d..6890b635 100644 --- a/infra/tf/bigquery_export/docker.tf +++ b/infra/tf/bigquery_export/docker.tf @@ -14,28 +14,20 @@ provider "docker" { # Calculate hash of source files to determine if rebuild is needed locals { source_files = fileset(path.root, "../${var.function_name}/*") - source_hash = sha1(join("", [for f in local.source_files : filesha1(f)])) + source_hash = substr(sha1(join("", [for f in local.source_files : filesha1(f)])), 0, 8) } # Build Docker image resource "docker_image" "function_image" { - name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}:latest" + name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}:${local.source_hash}" build { context = "../${var.function_name}/" dockerfile = "Dockerfile" platform = "linux/amd64" } - - triggers = { - source_hash = local.source_hash - } } resource "docker_registry_image" "registry_image" { name = docker_image.function_image.name - - triggers = { - source_hash = local.source_hash - } } diff --git a/infra/tf/dataform_service/docker.tf b/infra/tf/dataform_service/docker.tf index 6e01d41d..6890b635 100644 --- a/infra/tf/dataform_service/docker.tf +++ b/infra/tf/dataform_service/docker.tf @@ -14,28 +14,20 @@ provider "docker" { # Calculate hash of source files to determine if rebuild is needed locals { source_files = fileset(path.root, "../${var.function_name}/*") - source_hash = sha1(join("", [for f in local.source_files : filesha1(f)])) + source_hash = substr(sha1(join("", [for f in local.source_files : filesha1(f)])), 0, 8) } # Build Docker image resource "docker_image" "function_image" { - name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}:latest" + name = "${var.region}-docker.pkg.dev/${var.project}/dataform/${var.function_name}:${local.source_hash}" build { context = "../${var.function_name}/" dockerfile = "Dockerfile" platform = "linux/amd64" } - - triggers = { - source_hash = local.source_hash - } } resource "docker_registry_image" "registry_image" { name = docker_image.function_image.name - - triggers = { - source_hash = local.source_hash - } } diff --git a/infra/tf/dataform_service/main.tf b/infra/tf/dataform_service/main.tf index d02bd3fe..bceffea7 100644 --- a/infra/tf/dataform_service/main.tf +++ b/infra/tf/dataform_service/main.tf @@ -36,7 +36,7 @@ resource "google_cloud_run_v2_service" "dataform_service" { } service_account = var.function_identity - timeout = "600s" + timeout = "60s" max_instance_request_concurrency = 1 } @@ -44,8 +44,6 @@ resource "google_cloud_run_v2_service" "dataform_service" { type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" percent = 100 } - - depends_on = [docker_registry_image.registry_image] } resource "google_cloud_run_service_iam_member" "member" { From 4c20c71867445664a66a20de38cc386b318b2997 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 6 Jul 2025 21:49:01 +0200 Subject: [PATCH 6/6] readme --- README.md | 114 ++++++++++++------- SUMMARY.md | 72 ------------ dataform.md | 53 +++++++++ docs/infrastructure.md | 131 --------------------- infra/README.md | 163 --------------------------- infra/dataform-service/.dockerignore | 1 + infra/dataform-service/README.md | 151 +++++++++++++------------ 7 files changed, 207 insertions(+), 478 deletions(-) delete mode 100644 SUMMARY.md create mode 100644 dataform.md delete mode 100644 docs/infrastructure.md delete mode 100644 infra/README.md diff --git a/README.md b/README.md index 6e579149..347a73f6 100644 --- a/README.md +++ b/README.md @@ -6,38 +6,35 @@ This repository handles the HTTP Archive data pipeline, which takes the results The pipelines are run in Dataform service in Google Cloud Platform (GCP) and are kicked off automatically on crawl completion and other events. The code in the `main` branch is used on each triggered pipeline run. -### Crawl results +### HTTP Archive Crawl Tag: `crawl_complete` -- httparchive.crawl.pages -- httparchive.crawl.parsed_css -- httparchive.crawl.requests +- Crawl dataset `httparchive.crawl.*` -### Core Web Vitals Technology Report + Consumers: -Tag: `crux_ready` + - public dataset and [BQ Sharing Listing](https://console.cloud.google.com/bigquery/analytics-hub/discovery/projects/httparchive/locations/us/dataExchanges/httparchive/listings/crawl) -- httparchive.core_web_vitals.technologies +- Blink Features Report `httparchive.blink_features.usage` -Consumers: + Consumers: -- [HTTP Archive Tech Report](https://httparchive.org/reports/techreport/landing) + - [chromestatus.com](https://chromestatus.com/metrics/feature/timeline/popularity/2089) -### Blink Features Report +### HTTP Archive Technology Report -Tag: `crawl_complete` +Tag: `crux_ready` -- httparchive.blink_features.features -- httparchive.blink_features.usage +- `httparchive.reports.cwv_tech_*` and `httparchive.reports.tech_*` -Consumers: + Consumers: -- chromestatus.com - [example](https://chromestatus.com/metrics/feature/timeline/popularity/2089) + - [HTTP Archive Tech Report](https://httparchive.org/reports/techreport/landing) ## Schedules -1. [crawl-complete](https://console.cloud.google.com/cloudpubsub/subscription/detail/dataformTrigger?authuser=7&project=httparchive) PubSub subscription +1. [crawl-complete](https://console.cloud.google.com/cloudpubsub/subscription/detail/dataform-service-crawl-complete?authuser=2&project=httparchive) PubSub subscription Tags: ["crawl_complete"] @@ -49,30 +46,66 @@ Consumers: In order to unify the workflow triggering mechanism, we use [a Cloud Run function](./infra/README.md) that can be invoked in a number of ways (e.g. listen to PubSub messages), do intermediate checks and trigger the particular Dataform workflow execution configuration. -## Contributing - -### Dataform development - -1. [Create new dev workspace](https://cloud.google.com/dataform/docs/quickstart-dev-environments) in Dataform. -2. Make adjustments to the dataform configuration files and manually run a workflow to verify. -3. Push all your changes to a dev branch & open a PR with the link to the BigQuery artifacts generated in the test workflow. - -#### Workspace hints - -1. In `workflow_settings.yaml` set `environment: dev` to process sampled data. -2. For development and testing, you can modify variables in `includes/constants.js`, but note that these are programmatically generated. - -## Repository Structure - -- `definitions/` - Contains the core Dataform SQL definitions and declarations - - `output/` - Contains the main pipeline transformation logic - - `declarations/` - Contains referenced tables/views declarations and other resources definitions -- `includes/` - Contains shared JavaScript utilities and constants -- `infra/` - Infrastructure code and deployment configurations - - `dataform-trigger/` - Cloud Run function for workflow automation - - `tf/` - Terraform configurations - - `bigquery-export/` - BigQuery export configurations -- `docs/` - Additional documentation +## Cloud resources overview + +```mermaid +graph TB; + subgraph Cloud Run + dataform-service[dataform-service service] + bigquery-export[bigquery-export job] + end + + subgraph PubSub + crawl-complete[crawl-complete topic] + dataform-service-crawl-complete[dataform-service-crawl-complete subscription] + crawl-complete --> dataform-service-crawl-complete + end + + dataform-service-crawl-complete --> dataform-service + + subgraph Cloud_Scheduler + bq-poller-crux-ready[bq-poller-crux-ready Poller Scheduler Job] + bq-poller-crux-ready --> dataform-service + end + + subgraph Dataform + dataform[Dataform Repository] + dataform_release_config[dataform Release Configuration] + dataform_workflow[dataform Workflow Execution] + end + + dataform-service --> dataform[Dataform Repository] + dataform --> dataform_release_config + dataform_release_config --> dataform_workflow + + subgraph BigQuery + bq_jobs[BigQuery jobs] + bq_datasets[BigQuery table updates] + bq_jobs --> bq_datasets + end + + dataform_workflow --> bq_jobs + + bq_jobs --> bigquery-export + + subgraph Monitoring + cloud_run_logs[Cloud Run logs] + dataform_logs[Dataform logs] + bq_logs[BigQuery logs] + alerting_policies[Alerting Policies] + slack_notifications[Slack notifications] + + cloud_run_logs --> alerting_policies + dataform_logs --> alerting_policies + bq_logs --> alerting_policies + alerting_policies --> slack_notifications + end + + dataform-service --> cloud_run_logs + dataform_workflow --> dataform_logs + bq_jobs --> bq_logs + bigquery-export --> cloud_run_logs +``` ## Development Setup @@ -86,6 +119,7 @@ In order to unify the workflow triggering mechanism, we use [a Cloud Run functio - `npm run format` - Format code using Standard.js, fix Markdown issues, and format Terraform files - `npm run lint` - Run linting checks on JavaScript, Markdown files, and compile Dataform configs + - `make tf_apply` - Apply Terraform configurations ## Code Quality diff --git a/SUMMARY.md b/SUMMARY.md deleted file mode 100644 index e8ed3b6a..00000000 --- a/SUMMARY.md +++ /dev/null @@ -1,72 +0,0 @@ -# Dataform Service Merge - Summary - -## What Was Created - -I've successfully merged the `dataform-trigger` and `dataform-export` services into a unified `dataform-service` with path-based routing. - -### New Service Structure - -```files -/infra/dataform-service/ -├── index.js # Main service with /trigger and /export endpoints -├── dataform.js # Dataform client utilities (converted to ES modules) -├── package.json # Combined dependencies from both services -├── Dockerfile # Docker configuration -├── cloudbuild.yaml # Cloud Build configuration -├── README.md # Service documentation -├── MIGRATION.md # Detailed migration guide -└── examples.js # Usage examples -``` - -### New Terraform Configuration - -```files -/infra/tf/dataform_service/ -├── main.tf # Unified infrastructure configuration -└── variables.tf # Variable definitions -``` - -## Key Features - -### Path-Based Routing - -- **`/trigger`** - Handles Dataform workflow triggers (Pub/Sub events, scheduler) -- **`/export`** - Handles BigQuery export jobs (remote functions) - -### Unified Functionality - -- Combined all trigger logic (crux_ready, crawl_complete events) -- Combined all export logic (BigQuery to various destinations) -- Shared dependencies and infrastructure -- Single Docker image and deployment - -### Infrastructure Updates - -- Pub/Sub subscriptions point to `/trigger` endpoint -- BigQuery remote functions point to `/export` endpoint -- Cloud Scheduler jobs point to `/trigger` endpoint -- Single Cloud Run service with appropriate memory allocation - -## Next Steps - -1. **Deploy the new service:** - - ```bash - cd infra/dataform-service - gcloud builds submit --config cloudbuild.yaml - ``` - -2. **Update Terraform to use the new module:** - - ```terraform - module "dataform_service" { - source = "./dataform_service" - # ... variables - } - ``` - -3. **Test both endpoints** using the examples in `examples.js` - -4. **Monitor the service** to ensure both functionalities work correctly - -The original services have been marked with migration notes but are preserved for rollback if needed. diff --git a/dataform.md b/dataform.md new file mode 100644 index 00000000..ce3e4f1b --- /dev/null +++ b/dataform.md @@ -0,0 +1,53 @@ +# Dataform + +Runs the batch processing workflows. There are two Dataform repositories for [development](https://console.cloud.google.com/bigquery/dataform/locations/us-central1/repositories/crawl-data-test/details/workspaces?authuser=7&project=httparchive) and [production](https://console.cloud.google.com/bigquery/dataform/locations/us-central1/repositories/crawl-data/details/workspaces?authuser=7&project=httparchive). + +The test repository is used [for development and testing purposes](https://cloud.google.com/dataform/docs/workspaces) and not connected to the rest of the pipeline infra. + +Pipeline can be [run manually](https://cloud.google.com/dataform/docs/code-lifecycle) from the Dataform UI. + +[Configuration](./tf/dataform.tf) + +## Dataform Development Workspace + +1. [Create new dev workspace](https://cloud.google.com/dataform/docs/quickstart-dev-environments) in test Dataform repository. +2. Make adjustments to the dataform configuration files and manually run a workflow to verify. +3. Push all your changes to a dev branch & open a PR with the link to the BigQuery artifacts generated in the test workflow. + +*Some useful hints:* + +1. In workflow settings vars set `dev_name: dev` to process sampled data in dev workspace. +2. Change `current_month` variable to a month in the past. May be helpful for testing pipelines based on `chrome-ux-report` data. +3. `definitions/extra/test_env.sqlx` script helps to setup the tables required to run pipelines when in dev workspace. It's disabled by default. + +## Workspace hints + +1. In `workflow_settings.yaml` set `environment: dev` to process sampled data. +2. For development and testing, you can modify variables in `includes/constants.js`, but note that these are programmatically generated. + +## Repository Structure + +- `definitions/` - Contains the core Dataform SQL definitions and declarations + - `output/` - Contains the main pipeline transformation logic + - `declarations/` - Contains referenced tables/views declarations and other resources definitions +- `includes/` - Contains shared JavaScript utilities and constants +- `infra/` - Infrastructure code and deployment configurations + - `bigquery-export/` - BigQuery export service + - `dataform-service/` - Cloud Run function for dataform workflows automation + - `tf/` - Terraform configurations +- `docs/` - Additional documentation + +## GiHub to Dataform connection + +GitHub PAT saved to a [Secret Manager secret](https://console.cloud.google.com/security/secret-manager/secret/GitHub_max-ostapenko_dataform_PAT/versions?authuser=7&project=httparchive). + +- repository: HTTPArchive/dataform +- permissions: + - Commit statuses: read + - Contents: read, write + +## Monitoring + +- [Production Dataform workflow execution logs](https://console.cloud.google.com/bigquery/dataform/locations/us-central1/repositories/crawl-data/details/workflows?authuser=7&project=httparchive) + +- [Dataform Workflow Invocation Failed](https://console.cloud.google.com/monitoring/alerting/policies/16526940745374967367?authuser=7&project=httparchive) policy diff --git a/docs/infrastructure.md b/docs/infrastructure.md deleted file mode 100644 index 0ef40ae6..00000000 --- a/docs/infrastructure.md +++ /dev/null @@ -1,131 +0,0 @@ -# Infrastucture - -```mermaid -graph LR; - subgraph Cloud_Run_Functions - dataformTrigger[Dataform Trigger Function] - end - - subgraph PubSub - crawl_complete_topic[Crawl Complete Topic] - dataformTrigger_subscription[Dataform Trigger Subscription] - crawl_complete_topic --> dataformTrigger_subscription - end - - dataformTrigger_subscription --> dataformTrigger - - subgraph Cloud_Scheduler - bq_poller_crux_ready[CrUX Readiness Poller Job] - bq_poller_crux_ready --> dataformTrigger - end - - subgraph Dataform - dataform_repo[Dataform Repository] - dataform_repo_release_config[Release Configuration] - dataform_repo_workflow[Workflow Execution] - end - - dataformTrigger --> dataform_repo[Dataform Repository] - dataform_repo --> dataform_repo_release_config[Release Configuration] - dataform_repo_release_config --> dataform_repo_workflow[Workflow Execution] - - subgraph BigQuery - bq_jobs[BigQuery Jobs] - bq_datasets[BigQuery Dataset Updates] - bq_jobs --> bq_datasets - end - dataform_repo_workflow --> bq_jobs - - subgraph Logs_and_Alerts - cloud_run_logs[Cloud Run Logs] - dataform_logs[Dataform Logs] - bq_logs[BigQuery Logs] - alerting_policies[Alerting Policies] - slack_notifications[Slack Notifications] - - cloud_run_logs --> alerting_policies - dataform_logs --> alerting_policies - bq_logs --> alerting_policies - alerting_policies --> slack_notifications - end - - dataformTrigger --> cloud_run_logs - dataform_repo_workflow --> dataform_logs - bq_jobs --> bq_logs -``` - -## Triggering pipelines - -[Configuration](./tf/functions.tf) - -### Cloud Run Function - -Triggers the Dataform workflow execution, based on events or cron schedules. - -- [dataformTrigger](https://console.cloud.google.com/functions/details/us-central1/dataformTrigger?env=gen2&project=httparchive) - -[Source](./src/README.md) - -### Cloud Scheduler - -- [bq-poller-crux-ready](https://console.cloud.google.com/cloudscheduler/jobs/edit/us-central1/bq-poller-crux-ready?authuser=7&project=httparchive) - -### Pub/Sub Subscription - -- [dataform-trigger-subscription](https://console.cloud.google.com/cloudpubsub/subscription/detail/dataformTrigger?authuser=7&project=httparchive) - -## Dataform - -Runs the batch processing workflows. There are two Dataform repositories for [development](https://console.cloud.google.com/bigquery/dataform/locations/us-central1/repositories/crawl-data-test/details/workspaces?authuser=7&project=httparchive) and [production](https://console.cloud.google.com/bigquery/dataform/locations/us-central1/repositories/crawl-data/details/workspaces?authuser=7&project=httparchive). - -The test reporsitory is used [for development and testing purposes](https://cloud.google.com/dataform/docs/workspaces) and not connected to the rest of the pipeline infra. - -Pipeline can be [run manually](https://cloud.google.com/dataform/docs/code-lifecycle) from the Dataform UI. - -[Configuration](./tf/dataform.tf) - -### Dataform Development Workspace - -1. [Create new dev workspace](https://cloud.google.com/dataform/docs/quickstart-dev-environments) in test Dataform repository. -2. Make adjustments to the dataform configuration files and manually run a workflow to verify. -3. Push all your changes to a dev branch & open a PR with the link to the BigQuery artifacts generated in the test workflow. - -*Some useful hints:* - -1. In workflow settings vars set `dev_name: dev` to process sampled data in dev workspace. -2. Change `current_month` variable to a month in the past. May be helpful for testing pipelines based on `chrome-ux-report` data. -3. `definitions/extra/test_env.sqlx` script helps to setup the tables required to run pipelines when in dev workspace. It's disabled by default. - -## Monitoring - -[Configuration](./tf/monitoring.tf) - -### Dataform repository - -- [Production Dataform Workflow Excution logs](https://console.cloud.google.com/bigquery/dataform/locations/us-central1/repositories/crawl-data/details/workflows?authuser=7&project=httparchive) -- [Logs Explorer](https://cloudlogging.app.goo.gl/k9qfqCh4RjFwTnQ56) - -### Cloud Run logs - -- [Trigger function logs](https://console.cloud.google.com/run/detail/us-central1/dataformtrigger/logs?authuser=7&project=httparchive) -- [Logs Explorer](https://cloudlogging.app.goo.gl/6Q879UjnTPDqtVBx5) - -### BigQuery logs - -- [Logs Explorer](https://cloudlogging.app.goo.gl/rFjRMcvejd1Tyi7KA) - -### Alerting policies - -- [Dataform Trigger Function Error](https://console.cloud.google.com/monitoring/alerting/policies/3950167380893746326?authuser=7&project=httparchive) -- [Dataform Workflow Invocation Failed](https://console.cloud.google.com/monitoring/alerting/policies/7137542315653007241?authuser=7&project=httparchive) - -## CI/CD pipeline - -### Dataform / GiHub connection - -GitHub PAT saved to a [Secret Manager secret](https://console.cloud.google.com/security/secret-manager/secret/GitHub_max-ostapenko_dataform_PAT/versions?authuser=7&project=httparchive). - -- repository: HTTPArchive/dataform -- permissions: - - Commit statuses: read - - Contents: read, write diff --git a/infra/README.md b/infra/README.md deleted file mode 100644 index d33b3ec2..00000000 --- a/infra/README.md +++ /dev/null @@ -1,163 +0,0 @@ -# Infrastructure for the HTTP Archive data pipeline - -## Cloud Function for triggering Dataform workflows - -[dataformTrigger](https://console.cloud.google.com/functions/details/us-central1/dataformTrigger?env=gen2&authuser=7&project=httparchive) Cloud Run Function - -This function may be triggered by a PubSub message or Cloud Scheduler and invokes a Dataform workflow based on the provided configuration. - -Trigger types: - -1. `event` - immediately triggers a Dataform workflow using tags provided in configuration. - -2. `poller` - first triggers a BigQuery polling query. If the query returns TRUE, the Dataform workflow is triggered using the tags provided in configuration. - -See [available trigger configurations](https://github.com/HTTPArchive/dataform/blob/main/src/index.js#L4). - -Request body example: - -```json -{ - "message": { - "name": "crux_ready" - } -} -``` - -Request example for local development: - -```bash -curl -X POST http://localhost:8080/ \ - -H "Content-Type: application/json" \ - -d '{ - "message": { - "name": "crux_ready" - } - }' -``` - -## Cloud Function for triggering data exports - -[exportReport](https://console.cloud.google.com/functions/details/us-central1/bqExport?env=gen2&authuser=7&project=httparchive) Cloud Run Function - -This function triggers a job to export data to GCS or Firestore. - -Request body example: - -```json -{ - "protoPayload": { - "serviceData": { - "jobCompletedEvent": { - "job": { - "jobConfiguration": { - "query": { - "query": "/* {\"dataform_trigger\": \"report_cwv_tech_complete\", \"date\": \"2024-11-01\", \"name\": \"technologies\", \"type\": \"dict\"} *\/" - } - } - } - } - } - } -} -``` - -Request example for local development: - -```bash -curl -X POST http://localhost:8080/ \ - -H "Content-Type: application/json" \ - -d '{ - "protoPayload": { - "serviceData": { - "jobCompletedEvent": { - "job": { - "jobConfiguration": { - "query": { - "query": "/* {\"dataform_trigger\": \"report_complete\", \"date\": \"2024-11-01\", \"name\": \"bytesTotal\", \"type\": \"timeseries\"} *\/" - } - } - } - } - } - } -}' -``` - -or - -```bash -curl -X POST http://localhost:8080/ \ - -H "Content-Type: application/json" \ - -d '{ - "protoPayload": { - "serviceData": { - "jobCompletedEvent": { - "job": { - "jobConfiguration": { - "query": { - "query": "/* {\"dataform_trigger\": \"report_cwv_tech_complete\", \"date\": \"2024-11-01\", \"name\": \"lighthouse\", \"type\": \"report\"} *\/" - } - } - } - } - } - } -}' -``` - -## Cloud Run Job for exporting data - -[exportData](https://console.cloud.google.com/run/detail/us-central1/export-data?authuser=7&project=httparchive) Cloud Run Job - -This job exports data to GCS or Firestore based on the provided configuration. - -Input parameters: - -- `EXPORT_CONFIG` - JSON string with the export configuration. - -Example values: - -```plaintext -{"dataform_trigger":"report_cwv_tech_complete","name":"technologies","type":"dict"} -{"dataform_trigger":"report_cwv_tech_complete","date":"2024-11-01","name":"page_weight","type":"report"} -{"dataform_trigger":"report_complete","name":"bytesTotal","type":"timeseries"} -{"dataform_trigger":"report_complete","date":"2024-11-01","name":"bytesTotal","type":"histogram"} -``` - -## Monitoring - -The issues within the pipeline are being tracked using the following alerts: - -1. the event trigger processing fails - [Dataform Trigger Function Error](https://console.cloud.google.com/monitoring/alerting/policies/570799173843203905?authuser=7&project=httparchive) -2. a job in the workflow fails - "[Dataform Workflow Invocation Failed](https://console.cloud.google.com/monitoring/alerting/policies/16526940745374967367?authuser=7&project=httparchive) -3. the export function fails - [Dataform Export Function Error](https://console.cloud.google.com/monitoring/alerting/policies/570799173843203905?authuser=7&project=httparchive) - -Error notifications are sent to [#10x-infra](https://httparchive.slack.com/archives/C030V4WAVL3) Slack channel. - -## Local development - -To test the function locally run from the function directory: - -```bash -npm run start -``` - -Then, in a separate terminal, run the command with the test trigger payload. - -## Build - -Building a container image for the `bigquery-export` Cloud Run Job: - -```bash -cd infra/bigquery-export -npm run buildpack -``` - -## Deployment - -From project root directory run: - -```bash -make tf_apply -``` diff --git a/infra/dataform-service/.dockerignore b/infra/dataform-service/.dockerignore index c3bf1e3e..c0fa7e85 100644 --- a/infra/dataform-service/.dockerignore +++ b/infra/dataform-service/.dockerignore @@ -3,3 +3,4 @@ node_modules .git .gitignore .DS_Store +*.md diff --git a/infra/dataform-service/README.md b/infra/dataform-service/README.md index 9c92441f..1a50d657 100644 --- a/infra/dataform-service/README.md +++ b/infra/dataform-service/README.md @@ -1,14 +1,23 @@ # Dataform Service -A unified Cloud Run service that combines the functionality of both dataform-trigger and dataform-export services. This service provides two main endpoints for different operations: +A unified [dataform-service](https://console.cloud.google.com/functions/details/us-central1/dataform-service?authuser=7&project=httparchive) Cloud Run service that provides two main endpoints for different operations. -## Endpoints +## `/trigger` Trigger Dataform workflows -### `/trigger` +This service may be triggered by a PubSub message or Cloud Scheduler and invokes a Dataform workflow based on the provided configuration. -Handles Dataform workflow triggers based on events or polling conditions. +Trigger types: -**Example Request:** +1. `event` - immediately triggers a Dataform workflow using tags provided in configuration. + +2. `poller` - first triggers a BigQuery polling query. If the query returns TRUE, the Dataform workflow is triggered using the tags provided in configuration. + +Supported Triggers: + +- `crux_ready` - polls for Chrome UX Report data availability and triggers processing when conditions are met +- `crawl_complete` - event-based trigger for when crawl data processing is complete + +Request body example: ```json { @@ -18,100 +27,98 @@ Handles Dataform workflow triggers based on events or polling conditions. } ``` -### `/export` +Request example for local development: + +```bash +curl -X POST http://localhost:8080/ \ + -H "Content-Type: application/json" \ + -d '{ + "message": { + "name": "crux_ready" + } + }' +``` + +## `/` Trigger data exports -Handles BigQuery export job initialization. +[exportReport](https://console.cloud.google.com/functions/details/us-central1/bqExport?env=gen2&authuser=7&project=httparchive) Cloud Run Function -**Example Request:** +This function triggers a job to export data to GCS or Firestore. + +Request body example: ```json { "calls": [[{ - "destination": "...", - "config": "...", - "query": "..." + "destination": "gs://httparchive-reports/tech-report-2024", + "config": { + "format": "PARQUET", + "compression": "SNAPPY" + }, + "query": "SELECT * FROM httparchive.reports.tech_report_categories WHERE _TABLE_SUFFIX = '2024_01_01'" }]] } ``` -## Supported Triggers +Request example for local development: -- `crux_ready`: Polls for Chrome UX Report data availability and triggers processing when conditions are met -- `crawl_complete`: Event-based trigger for when crawl data processing is complete +```bash +curl -X POST http://localhost:8080/ \ + -H "Content-Type: application/json" \ + -d '{ + "calls": [[{ + "destination": "gs://httparchive-reports/tech-report-2024", + "config": { + "format": "PARQUET", + "compression": "SNAPPY" + }, + "query": "SELECT * FROM httparchive.reports.tech_report_categories WHERE _TABLE_SUFFIX = '2024_01_01'" + }]] +}' +``` -## Environment Variables +## Cloud Run Job for exporting data -- `PORT`: Port number for the service (default: 8080) +[exportData](https://console.cloud.google.com/run/detail/us-central1/export-data?authuser=7&project=httparchive) Cloud Run Job -## Development +This job exports data to GCS or Firestore based on the provided configuration. -```bash -# Install dependencies -npm install +Input parameters: -# Start development server -npm run start_dev +- `EXPORT_CONFIG` - JSON string with the export configuration. + +Example values: -# Build Docker image -npm run build +```plaintext +{"dataform_trigger":"report_cwv_tech_complete","name":"technologies","type":"dict"} +{"dataform_trigger":"report_cwv_tech_complete","date":"2024-11-01","name":"page_weight","type":"report"} +{"dataform_trigger":"report_complete","name":"bytesTotal","type":"timeseries"} +{"dataform_trigger":"report_complete","date":"2024-11-01","name":"bytesTotal","type":"histogram"} ``` -## Deployment +## Monitoring -The service is deployed to Google Cloud Run using Cloud Build: +The issues within the pipeline are being tracked using the following alerts: -```bash -gcloud builds submit --config cloudbuild.yaml -``` +- [Dataform Trigger Function Error](https://console.cloud.google.com/monitoring/alerting/policies/570799173843203905?authuser=2&project=httparchive) policy +- [Dataform Export Function Error](https://console.cloud.google.com/monitoring/alerting/policies/2588749473925942477?authuser=2&project=httparchive) policy -## Migration from Separate Services +Error notifications are sent to [#10x-infra](https://httparchive.slack.com/archives/C030V4WAVL3) Slack channel. -This service replaces the following separate services: +## Local development -- `dataform-trigger`: Now accessible via `/trigger` endpoint -- `dataform-export`: Now accessible via `/export` endpoint +To test the function locally run from the function directory: -Update your service calls to use the new endpoint paths when migrating. +```bash +npm run start_dev +``` -```js -// Example usage of the merged dataform-service +Then, in a separate terminal, run the command with the test trigger payload. -// Trigger example -const triggerPayload = { - message: { - name: "crux_ready" - } -} +## Deployment -// Export example -const exportPayload = { - calls: [[{ - destination: "gs://httparchive-reports/tech-report-2024", - config: { - format: "PARQUET", - compression: "SNAPPY" - }, - query: "SELECT * FROM httparchive.reports.tech_report_categories WHERE _TABLE_SUFFIX = '2024_01_01'" - }]] -} +From project root directory run: -// Usage examples: - -// POST to /trigger endpoint -fetch('https://dataform-service-url/trigger', { - method: 'POST', - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify(triggerPayload) -}) - -// POST to /export endpoint -fetch('https://dataform-service-url/export', { - method: 'POST', - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify(exportPayload) -}) +```bash +make tf_apply ```