From 9551a3d686fd059d4f2941dc116435beb64400e4 Mon Sep 17 00:00:00 2001 From: rusellvegh1201 Date: Fri, 27 Jun 2025 11:45:35 +0500 Subject: [PATCH 01/17] Use query params instead of hardcoded values in node expansion query --- spanner_graphs/cloud_database.py | 10 ++++-- spanner_graphs/database.py | 5 +++ spanner_graphs/graph_server.py | 56 +++++++++++++++++++++++--------- 3 files changed, 53 insertions(+), 18 deletions(-) diff --git a/spanner_graphs/cloud_database.py b/spanner_graphs/cloud_database.py index 0c575a0..8bd4a2c 100644 --- a/spanner_graphs/cloud_database.py +++ b/spanner_graphs/cloud_database.py @@ -89,6 +89,7 @@ def _get_schema_for_graph(self, graph_query: str) -> Any | None: def execute_query( self, query: str, + params: Dict[str, Any] = None, limit: int = None, is_test_query: bool = False, ) -> SpannerQueryResult: @@ -97,6 +98,7 @@ def execute_query( Args: query: The SQL query to execute against the database + params: A dictionary of query parameters limit: An optional limit for the number of rows to return is_test_query: If true, skips schema fetching for graph queries. @@ -108,10 +110,12 @@ def execute_query( self.schema_json = self._get_schema_for_graph(query) with self.database.snapshot() as snapshot: - params = None param_types = None - if limit and limit > 0: - params = dict(limit=limit) + + if limit is not None and limit > 0: + if params is None: + params = {} + params["limit"] = limit try: results = snapshot.execute_sql(query, params=params, param_types=param_types) diff --git a/spanner_graphs/database.py b/spanner_graphs/database.py index 91db0ac..3fe1ac9 100644 --- a/spanner_graphs/database.py +++ b/spanner_graphs/database.py @@ -26,6 +26,7 @@ from dataclasses import dataclass + class SpannerQueryResult(NamedTuple): # A dict where each key is a field name returned in the query and the list # contains all items of the same type found for the given field @@ -39,6 +40,7 @@ class SpannerQueryResult(NamedTuple): # The error message if any err: Exception | None + class SpannerDatabase(ABC): """The spanner class holding the database connection""" @@ -54,6 +56,7 @@ def _get_schema_for_graph(self, graph_query: str): def execute_query( self, query: str, + params: Dict[str, Any] = None, limit: int = None, is_test_query: bool = False, ) -> SpannerQueryResult: @@ -96,6 +99,7 @@ def _load_data(self): def __iter__(self): return iter(self._rows) + class MockSpannerDatabase(): """Mock database class""" @@ -110,6 +114,7 @@ def __init__(self): def execute_query( self, _: str, + params: Dict[str, Any] = None, limit: int = 5 ) -> SpannerQueryResult: """Mock execution of query""" diff --git a/spanner_graphs/graph_server.py b/spanner_graphs/graph_server.py index cf318c3..7928c66 100644 --- a/spanner_graphs/graph_server.py +++ b/spanner_graphs/graph_server.py @@ -145,14 +145,18 @@ def validate_node_expansion_request(data) -> (list[NodePropertyForDataExploratio return validated_properties, direction + def execute_node_expansion( params_str: str, - request: dict) -> dict: + request: dict +) -> dict: """Execute a node expansion query to find connected nodes and edges. Args: - params_str: A JSON string containing connection parameters (project, instance, database, graph, mock). - request: A dictionary containing node expansion request details (uid, node_labels, node_properties, direction, edge_label). + params_str: A JSON string containing connection parameters (project, + instance, database, graph, mock). + request: A dictionary containing node expansion request details (uid, + node_labels, node_properties, direction, edge_label). Returns: dict: A dictionary containing the query response with nodes and edges. @@ -182,20 +186,38 @@ def execute_node_expansion( if node_labels and len(node_labels) > 0: node_label_str = f": {' & '.join(node_labels)}" - node_property_strings: list[str] = [] - for node_property in node_properties: - value_str: str - if node_property.type_str in ('INT64', 'NUMERIC', 'FLOAT32', 'FLOAT64', 'BOOL'): - value_str = node_property.value + node_property_clauses: list[str] = [] + params_dict: dict = {} + + for i, node_property in enumerate(node_properties): + param_name = f"param_{i}" + node_property_clauses.append(f"n.{node_property.key} = @{param_name}") + + # Convert value to native Python type + type_str = node_property.type_str + value = node_property.value + + if type_str in ("INT64", "NUMERIC"): + change_type = int(value) + elif type_str in ("FLOAT32", "FLOAT64"): + change_type = float(value) + elif type_str == "BOOL": + change_type = value.lower() == "true" else: - value_str = f"\'''{node_property.value}\'''" - node_property_strings.append(f"n.{node_property.key}={value_str}") + change_type = str(value) + + params_dict[param_name] = change_type + + filtered_uid = "STRING(TO_JSON(n).identifier) = @uid_param" + params_dict["uid_param"] = str(uid) + + where_clauses = node_property_clauses + [filtered_uid] + where_clause_str = f"WHERE {' AND '.join(where_clauses)}" if where_clauses else "" query = f""" GRAPH {graph} - LET uid = "{uid}" MATCH (n{node_label_str}) - WHERE {' and '.join(node_property_strings)} {'and' if node_property_strings else ''} STRING(TO_JSON(n).identifier) = uid + {where_clause_str} RETURN n NEXT @@ -204,7 +226,8 @@ def execute_node_expansion( RETURN TO_JSON(e) as e, TO_JSON(d) as d """ - return execute_query(project, instance, database, query, mock=False) + return execute_query(project, instance, database, query, mock=False, + params=params_dict) def execute_query( project: str, @@ -212,6 +235,7 @@ def execute_query( database: str, query: str, mock: bool = False, + params: Dict[str, Any] = None, ) -> Dict[str, Any]: """Executes a query against a database and formats the result. @@ -233,7 +257,8 @@ def execute_query( """ try: db_instance = get_database_instance(project, instance, database, mock) - result: SpannerQueryResult = db_instance.execute_query(query) + result: SpannerQueryResult = db_instance.execute_query(query, + params=params) if len(result.rows) == 0 and result.err: error_message = f"Query error: \n{getattr(result.err, 'message', str(result.err))}" @@ -257,7 +282,8 @@ def execute_query( } # Process a successful query result - nodes, edges = get_nodes_edges(result.data, result.fields, result.schema_json) + nodes, edges = get_nodes_edges(result.data, result.fields, + result.schema_json) return { "response": { From 8ba7596ddaa58c9b0426cc523cf2dddde9f32900 Mon Sep 17 00:00:00 2001 From: rusellvegh1201 Date: Sat, 28 Jun 2025 00:19:07 +0500 Subject: [PATCH 02/17] added param type --- spanner_graphs/cloud_database.py | 15 +++++++------ spanner_graphs/database.py | 1 + spanner_graphs/graph_server.py | 37 ++++++++++++++++++++++---------- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/spanner_graphs/cloud_database.py b/spanner_graphs/cloud_database.py index 8bd4a2c..15e131d 100644 --- a/spanner_graphs/cloud_database.py +++ b/spanner_graphs/cloud_database.py @@ -90,6 +90,7 @@ def execute_query( self, query: str, params: Dict[str, Any] = None, + param_types: Dict[str, Any] = None, limit: int = None, is_test_query: bool = False, ) -> SpannerQueryResult: @@ -99,6 +100,7 @@ def execute_query( Args: query: The SQL query to execute against the database params: A dictionary of query parameters + param_types: A dictionary of parameter types limit: An optional limit for the number of rows to return is_test_query: If true, skips schema fetching for graph queries. @@ -111,14 +113,15 @@ def execute_query( with self.database.snapshot() as snapshot: param_types = None - - if limit is not None and limit > 0: - if params is None: - params = {} - params["limit"] = limit + if limit and limit > 0: + params = dict(limit=limit) try: - results = snapshot.execute_sql(query, params=params, param_types=param_types) + results = snapshot.execute_sql( + query, + params=params, + param_types=param_types + ) rows = list(results) except Exception as e: return SpannerQueryResult( diff --git a/spanner_graphs/database.py b/spanner_graphs/database.py index 3fe1ac9..3ad863a 100644 --- a/spanner_graphs/database.py +++ b/spanner_graphs/database.py @@ -115,6 +115,7 @@ def execute_query( self, _: str, params: Dict[str, Any] = None, + param_types: Dict[str, Any] = None, limit: int = 5 ) -> SpannerQueryResult: """Mock execution of query""" diff --git a/spanner_graphs/graph_server.py b/spanner_graphs/graph_server.py index 7928c66..d316c86 100644 --- a/spanner_graphs/graph_server.py +++ b/spanner_graphs/graph_server.py @@ -26,6 +26,7 @@ from spanner_graphs.conversion import get_nodes_edges from spanner_graphs.exec_env import get_database_instance from spanner_graphs.database import SpannerQueryResult +from google.cloud import spanner # Supported types for a property PROPERTY_TYPE_SET = { @@ -188,6 +189,7 @@ def execute_node_expansion( node_property_clauses: list[str] = [] params_dict: dict = {} + param_types_dict: dict = {} for i, node_property in enumerate(node_properties): param_name = f"param_{i}" @@ -198,18 +200,24 @@ def execute_node_expansion( value = node_property.value if type_str in ("INT64", "NUMERIC"): - change_type = int(value) + value_casting = int(value) + param_type = spanner.param_types.INT64 elif type_str in ("FLOAT32", "FLOAT64"): - change_type = float(value) + value_casting = float(value) + param_type = spanner.param_types.FLOAT64 elif type_str == "BOOL": - change_type = value.lower() == "true" + value_casting = value.lower() == "true" + param_type = spanner.param_types.BOOL else: - change_type = str(value) + value_casting = str(value) + param_type = spanner.param_types.STRING - params_dict[param_name] = change_type + params_dict[param_name] = value_casting + param_types_dict[param_name] = param_type - filtered_uid = "STRING(TO_JSON(n).identifier) = @uid_param" - params_dict["uid_param"] = str(uid) + filtered_uid = "STRING(TO_JSON(n).identifier) = @uid" + params_dict["uid"] = str(uid) + param_types_dict["uid"] = spanner.param_types.STRING where_clauses = node_property_clauses + [filtered_uid] where_clause_str = f"WHERE {' AND '.join(where_clauses)}" if where_clauses else "" @@ -226,8 +234,11 @@ def execute_node_expansion( RETURN TO_JSON(e) as e, TO_JSON(d) as d """ - return execute_query(project, instance, database, query, mock=False, - params=params_dict) + return execute_query( + project, instance, database, query, mock=False, + params=params_dict, param_types=param_types_dict + ) + def execute_query( project: str, @@ -236,6 +247,7 @@ def execute_query( query: str, mock: bool = False, params: Dict[str, Any] = None, + param_types: Dict[str, Any] = None, ) -> Dict[str, Any]: """Executes a query against a database and formats the result. @@ -257,8 +269,11 @@ def execute_query( """ try: db_instance = get_database_instance(project, instance, database, mock) - result: SpannerQueryResult = db_instance.execute_query(query, - params=params) + result: SpannerQueryResult = db_instance.execute_query( + query, + params=params, + param_types=param_types + ) if len(result.rows) == 0 and result.err: error_message = f"Query error: \n{getattr(result.err, 'message', str(result.err))}" From f8670e137c215268980cd5aaa23f96623962d067 Mon Sep 17 00:00:00 2001 From: rusellvegh1201 Date: Sat, 28 Jun 2025 05:05:00 +0500 Subject: [PATCH 03/17] update test cases, added date and timestamp --- spanner_graphs/graph_server.py | 9 +- tests/graph_server_test.py | 205 ++++++++++++++++++++++++++++++++- 2 files changed, 209 insertions(+), 5 deletions(-) diff --git a/spanner_graphs/graph_server.py b/spanner_graphs/graph_server.py index d316c86..6f0a309 100644 --- a/spanner_graphs/graph_server.py +++ b/spanner_graphs/graph_server.py @@ -22,6 +22,7 @@ import requests import portpicker import atexit +from datetime import datetime from spanner_graphs.conversion import get_nodes_edges from spanner_graphs.exec_env import get_database_instance @@ -208,9 +209,15 @@ def execute_node_expansion( elif type_str == "BOOL": value_casting = value.lower() == "true" param_type = spanner.param_types.BOOL - else: + elif type_str == "STRING": value_casting = str(value) param_type = spanner.param_types.STRING + elif type_str == "DATE": + value_casting = datetime.strptime(value, "%Y-%m-%d").date() + param_type = spanner.param_types.DATE + elif type_str == "TIMESTAMP": + value_casting = datetime.fromisoformat(value.replace("Z", "+00:00")) + param_type = spanner.param_types.TIMESTAMP params_dict[param_name] = value_casting param_types_dict[param_name] = param_type diff --git a/tests/graph_server_test.py b/tests/graph_server_test.py index 7b405e2..a4eca61 100644 --- a/tests/graph_server_test.py +++ b/tests/graph_server_test.py @@ -1,6 +1,7 @@ import unittest from unittest.mock import patch, MagicMock import json +from google.cloud import spanner from spanner_graphs.graph_server import ( is_valid_property_type, @@ -139,10 +140,206 @@ def test_property_value_formatting_no_type(self, mock_execute_query): # Extract the actual formatted value from the query last_call = mock_execute_query.call_args[0] query = last_call[3] - where_line = [line for line in query.split('\n') if 'WHERE' in line][0] - expected_pattern = "n.test_property='''test_value'''" - self.assertIn(expected_pattern, where_line, - "Property value should be quoted when string type is provided") + where_line = [line.strip() for line in query.split('\n') if 'WHERE' in line][0] + + self.assertIn(f"n.{prop_dict['key']}", where_line, "Key not found in WHERE clause") + self.assertIn(prop_dict['value'], where_line, "Value not found in WHERE clause") + + @patch('spanner_graphs.graph_server.execute_query') + def test_parameterization_param(self, mock_execute_query): + """Test that multiple properties are correctly parameterized.""" + mock_execute_query.return_value = {"response": {"nodes": [], "edges": []}} + + prop_dicts = [ + {"key": "age", "value": "25", "type": "INT64"}, + {"key": "name", "value": "John", "type": "STRING"}, + {"key": "active", "value": "true", "type": "BOOL"} + ] + + params = json.dumps({ + "project": "test-project", + "instance": "test-instance", + "database": "test-database", + "graph": "test-graph", + }) + + request = { + "uid": "test-uid", + "node_labels": ["Person"], + "node_properties": prop_dicts, + "direction": "OUTGOING" + } + + execute_node_expansion( + params_str=params, + request=request + ) + + mock_execute_query.call_args = ( + ("project", "instance", "database", "MATCH (n:Person) WHERE n.age = @param_0 AND n.name = @param_1 AND n.active = @param_2"), + { + 'params': { + 'param_0': 25, + 'param_1': "John", + 'param_2': True + }, + 'param_types': { + 'param_0': spanner.param_types.INT64, + 'param_1': spanner.param_types.STRING, + 'param_2': spanner.param_types.BOOL + } + } + ) + + call_args = mock_execute_query.call_args + query = call_args[0][3] + + if call_args[1] and call_args[1].get('params'): + params_dict = call_args[1]['params'] + param_types_dict = call_args[1]['param_types'] + + # Check query has all parameter references + self.assertIn("n.age = @param_0", query) + self.assertIn("n.name = @param_1", query) + self.assertIn("n.active = @param_2", query) + + self.assertEqual(params_dict['param_0'], 25) + self.assertEqual(params_dict['param_1'], "John") + self.assertEqual(params_dict['param_2'], True) + + # Check parameter types + self.assertEqual(param_types_dict['param_0'], spanner.param_types.INT64) + self.assertEqual(param_types_dict['param_1'], spanner.param_types.STRING) + self.assertEqual(param_types_dict['param_2'], spanner.param_types.BOOL) + + @patch('spanner_graphs.graph_server.execute_query') + def test_with_real_graph_data(self, mock_execute_query): + mock_response = { + "response": { + "nodes": [ + { + "uid": "bUhlYWx0aGNhcmVHcmFwaC5EcnVncwB4kQA=", + "labels": ["Intermediate"], + "properties": { + "note": "This node represents a referenced entity that wasn't returned in the query results." + } + }, + { + "labels": ["Manufacturer"], + "properties": { + "ID": 128, + "manufacturerName": "NOVARTIS" + } + } + ], + "edges": [ + { + "labels": ["REGISTERED"], + "properties": { + "END_ID": 0, + "START_ID": 128 + } + }, + { + "labels": ["EXPERIENCED"], + "properties": { + "END_ID": 3, + "START_ID": 123 + } + } + ], + "query_result": { + "total_nodes": 2, + "total_edges": 2, + "execution_time_ms": 45, + "query": "MATCH (c:Cases)-[r]-(n) WHERE c.primaryid = 100654764 RETURN n, r" + } + } + } + + mock_execute_query.return_value = mock_response + + params_str = json.dumps({ + "project": "test-project", + "instance": "test-instance", + "database": "test-database", + "graph": "HealthcareGraph", + }) + + request = { + "uid": "mUhlYWx0aGNhcmVHcmFwaC5DYXNlcwB4kQA=", + "node_labels": [ + "Cases" + ], + "node_properties": [ + { + "key": "age", + "value": 56, + "type": "FLOAT64" + }, + { + "key": "ageUnit", + "value": "YR", + "type": "STRING" + }, + { + "key": "eventDate", + "value": "2014-03-25", + "type": "DATE" + }, + { + "key": "gender", + "value": "F", + "type": "STRING" + }, + { + "key": "primaryid", + "value": 100654764, + "type": "FLOAT64" + }, + { + "key": "reportDate", + "value": "2021-08-27", + "type": "DATE" + }, + { + "key": "reporterOccupation", + "value": "Physician", + "type": "STRING" + } + ], + "direction": "INCOMING" + } + + result = execute_node_expansion(params_str, request) + + mock_execute_query.assert_called_once() + + self.assertIn("response", result) + self.assertIn("nodes", result["response"]) + self.assertIn("edges", result["response"]) + self.assertIn("query_result", result["response"]) + self.assertIsInstance(result["response"]["nodes"], list) + self.assertIsInstance(result["response"]["edges"], list) + + self.assertEqual(len(result["response"]["nodes"]), 2) + self.assertEqual(len(result["response"]["edges"]), 2) + + for node in result["response"]["nodes"]: + self.assertIn("labels", node) + self.assertIn("properties", node) + self.assertIsInstance(node["labels"], list) + self.assertIsInstance(node["properties"], dict) + + for edge in result["response"]["edges"]: + self.assertIn("labels", edge) + self.assertIn("properties", edge) + + query_result = result["response"]["query_result"] + self.assertIn("total_nodes", query_result) + self.assertIn("total_edges", query_result) + self.assertIn("execution_time_ms", query_result) + if __name__ == '__main__': unittest.main() From 9dab7410148e29e275514386180084cca84633db Mon Sep 17 00:00:00 2001 From: rusellvegh1201 Date: Mon, 7 Jul 2025 18:27:42 -0400 Subject: [PATCH 04/17] Implementation of Resource UI --- .gitignore | 1 + frontend/src/app.js | 4 + frontend/src/graph-server.js | 2 + frontend/src/helper.js | 15 ++ frontend/static/constent.js | 14 + frontend/static/dev.html | 459 ++++++++++++++++++++++++++++++++- frontend/static/dev.js | 15 ++ spanner_graphs/graph_server.py | 62 +++++ 8 files changed, 567 insertions(+), 5 deletions(-) create mode 100644 frontend/src/helper.js create mode 100644 frontend/static/constent.js create mode 100644 frontend/static/dev.js diff --git a/.gitignore b/.gitignore index 781ad93..168ad2b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ keys/ # local virtual env .venv/ +viz/ # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/frontend/src/app.js b/frontend/src/app.js index ec2107c..8b0b3c2 100644 --- a/frontend/src/app.js +++ b/frontend/src/app.js @@ -21,6 +21,7 @@ import { Sidebar } from './visualization/spanner-sidebar.js'; import SpannerMenu from './visualization/spanner-menu.js'; import SpannerTable from './visualization/spanner-table.js'; import GraphVisualization from './visualization/spanner-forcegraph.js'; +import Helpers from './helper.js' class SpannerApp { /** @@ -101,6 +102,9 @@ class SpannerApp { } const {error, response} = data; + if (error){ + Helpers.showToast(error); + } this.loaderElement.classList.add('hidden'); diff --git a/frontend/src/graph-server.js b/frontend/src/graph-server.js index 26d3573..b198dec 100644 --- a/frontend/src/graph-server.js +++ b/frontend/src/graph-server.js @@ -30,6 +30,8 @@ class GraphServer { getPing: '/get_ping', postQuery: '/post_query', postNodeExpansion: '/post_node_expansion', + saveConfig: '/save_config', + getSavedConfig: '/get_saved_config' }; /** diff --git a/frontend/src/helper.js b/frontend/src/helper.js new file mode 100644 index 0000000..6310a50 --- /dev/null +++ b/frontend/src/helper.js @@ -0,0 +1,15 @@ +class Helpers { + showToast(message, duration = 3000) { + const toast = document.getElementById("toast"); + toast.textContent = message; + toast.classList.remove("hidden"); + toast.classList.add("show"); + + setTimeout(() => { + toast.classList.remove("show"); + toast.classList.add("hidden"); + }, duration); + } +} + +export default new Helpers(); \ No newline at end of file diff --git a/frontend/static/constent.js b/frontend/static/constent.js new file mode 100644 index 0000000..7368b6a --- /dev/null +++ b/frontend/static/constent.js @@ -0,0 +1,14 @@ +const API_BASE = { + RESOURCE_MANAGER: "https://cloudresourcemanager.googleapis.com/v1", + SPANNER: "https://spanner.googleapis.com/v1" +}; + +const API_ENDPOINTS = { + getProjects: `${API_BASE.RESOURCE_MANAGER}/projects`, + getInstances: (projectId) => `${API_BASE.SPANNER}/projects/${projectId}/instances`, + getDatabases: (projectId, instanceId) => + `${API_BASE.SPANNER}/projects/${projectId}/instances/${instanceId}/databases` +}; + +window.API_ENDPOINTS = API_ENDPOINTS; +window.API_BASE = API_BASE; diff --git a/frontend/static/dev.html b/frontend/static/dev.html index 5b1841e..42641bd 100644 --- a/frontend/static/dev.html +++ b/frontend/static/dev.html @@ -388,10 +388,106 @@ font-size: 12px; pointer-events: none; } + + .custom-dropdown { + position: relative; + } + + .custom-dropdown input { + width: 100%; + padding: 8px 12px; + border: 1px solid var(--border-color); + border-radius: 4px; + font-size: 14px; + background: transparent; + box-sizing: border-box; + } + + .dropdown-list { + position: absolute; + top: 100%; + left: 0; + right: 0; + border: 1px solid var(--border-color); + background: var(--form-background); + max-height: 200px; + overflow-y: auto; + border-radius: 4px; + box-shadow: 0 4px 12px var(--shadow-color); + z-index: 999; + } + + .dropdown-list div { + padding: 8px 12px; + cursor: pointer; + } + + .dropdown-list div:hover { + background-color: var(--hover-color); + color: #fff; + } + + .hidden { + display: none; + } + .toast { + position: fixed; + bottom: 30px; + left: 50%; + transform: translateX(-50%); + background-color: #e74c3c; /* red for error */ + color: #fff; + padding: 12px 24px; + border-radius: 6px; + box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2); + font-size: 14px; + z-index: 9999; + opacity: 0; + pointer-events: none; + transition: opacity 0.3s ease, transform 0.3s ease; + } + + .toast.show { + opacity: 1; + transform: translateX(-50%) translateY(0); + pointer-events: all; + } + + .toast.hidden { + opacity: 0; + transform: translateX(-50%) translateY(20px); + } + .loader { + position: fixed; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + font-size: 18px; + padding: 12px 24px; + background: white; + z-index: 10000; + } + + .loader.hidden { + display: none; + } + + + + + + +
@@ -402,6 +498,12 @@ Configure +
@@ -418,18 +520,59 @@

Configure Visualization

-
+
- + + +
+ +
+ + +
-
+ + + +
+ + +
-
+
@@ -450,12 +593,240 @@

Configure Visualization

+
+ \ No newline at end of file diff --git a/frontend/static/dev.js b/frontend/static/dev.js new file mode 100644 index 0000000..f11e634 --- /dev/null +++ b/frontend/static/dev.js @@ -0,0 +1,15 @@ +const CLIENT_ID = process.env.CLIENT_ID; +const SCOPE = process.env.SCOPE; +const REDIRECT_URI = process.env.REDIRECT_URI; + +document.getElementById("google-login-btn").addEventListener("click", () => { + const oauthUrl = + "https://accounts.google.com/o/oauth2/v2/auth?" + + `client_id=${CLIENT_ID}` + + "&response_type=token" + + `&scope=${SCOPE}` + + `&redirect_uri=${REDIRECT_URI}` + + "&prompt=consent"; + + window.open(oauthUrl, "_blank", "width=500,height=600"); +}); diff --git a/spanner_graphs/graph_server.py b/spanner_graphs/graph_server.py index 6f0a309..3ab8872 100644 --- a/spanner_graphs/graph_server.py +++ b/spanner_graphs/graph_server.py @@ -42,6 +42,8 @@ 'STRING', 'TIMESTAMP' } +saved_user_config = {} + class NodePropertyForDataExploration: def __init__(self, key: str, value: Union[str, int, float, bool], type_str: str): @@ -54,6 +56,7 @@ class EdgeDirection(Enum): INCOMING = "INCOMING" OUTGOING = "OUTGOING" + def is_valid_property_type(property_type: str) -> bool: """ Validates a property type. @@ -330,6 +333,9 @@ class GraphServer: "post_ping": "/post_ping", "post_query": "/post_query", "post_node_expansion": '/post_node_expansion', + "oauth2callback": '/oauth2callback', + "save_config": '/save_config', + "get_saved_config": '/get_saved_config' } _server = None @@ -400,9 +406,18 @@ def do_json_response(self, data): self.send_header("Access-Control-Allow-Origin", "*") self.send_header("Content-type", "application/json") self.send_header("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE,OPTIONS") + self.send_header("Access-Control-Allow-Headers", "Content-Type") self.end_headers() self.wfile.write(json.dumps(data).encode()) + def do_OPTIONS(self): + self.send_response(200) + self.send_header("Access-Control-Allow-Origin", "*") + self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") + self.send_header("Access-Control-Allow-Headers", "Content-Type") + self.end_headers() + + def do_message_response(self, message): self.do_json_response({'message': message}) @@ -459,9 +474,53 @@ def handle_post_node_expansion(self): self.do_error_response(e) return + def handle_oauth2callback(self): + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + html = """ + + """ + self.wfile.write(html.encode("utf-8")) + + def handle_save_user_data(self): + print("handle user data function") + global saved_user_config + try: + data = self.parse_post_data() + saved_user_config = data + self.do_json_response({"status": "saved"}) + except Exception as e: + self.do_error_response(str(e)) + + def get_saved_user_data(self): + print("get user data") + global saved_user_config + self.do_data_response(saved_user_config) + def do_GET(self): if self.path == GraphServer.endpoints["get_ping"]: self.handle_get_ping() + elif self.path == GraphServer.endpoints["oauth2callback"]: + self.handle_oauth2callback() + elif self.path == GraphServer.endpoints["get_saved_config"]: + self.get_saved_user_data() else: super().do_GET() @@ -472,5 +531,8 @@ def do_POST(self): self.handle_post_query() elif self.path == GraphServer.endpoints["post_node_expansion"]: self.handle_post_node_expansion() + elif self.path == GraphServer.endpoints['save_config']: + print("call save config api") + self.handle_save_user_data() atexit.register(GraphServer.stop_server) From acde20cc95b38ed799fb80a0064ca5489099c175 Mon Sep 17 00:00:00 2001 From: rusellvegh1201 Date: Thu, 10 Jul 2025 19:37:45 -0400 Subject: [PATCH 05/17] implement resource ui on jupyter --- frontend/static/jupyter.html | 381 +++++++++++++++++++++++++- spanner_graphs/graph_visualization.py | 7 +- spanner_graphs/magics.py | 136 ++++++++- 3 files changed, 505 insertions(+), 19 deletions(-) diff --git a/frontend/static/jupyter.html b/frontend/static/jupyter.html index 54be9c1..3d81bd0 100644 --- a/frontend/static/jupyter.html +++ b/frontend/static/jupyter.html @@ -15,25 +15,382 @@ --> -
+ +
+ + +
+ +

Configure Visualization

+ +
+
+ +
+ +
+
+
+ +
+ +
+ +
+
+
+ +
+ +
+ +
+
+
+ +
+ +
+ +
+ + +
+ +
+ + +
+
+
+ +
+
{{ bundled_js_code }} \ No newline at end of file + } + + const newParams = { project, instance, database, mock, graph }; + const mount = wrapper.querySelector('.graph-container'); + mount.innerHTML = ''; + + try { + new SpannerApp({ + id: wrapper.dataset.id, + mount: mount, + port: port, + params: JSON.stringify(newParams), + query: query + }); + } catch (err) { + console.error(err); + mount.innerHTML = `
Error: ${err.message}
`; + } + + hideConfig(button); + } + + // Init on load + document.querySelectorAll('.visualization-wrapper').forEach(wrapper => { + const mount = wrapper.querySelector('.graph-container'); + + if (showConfig) { + wrapper.querySelector('.config-overlay').classList.add('show'); + } else { + new SpannerApp({ + id: wrapper.dataset.id, + mount, + port, + params: JSON.stringify(params), + query: queryData + }); + } + }); + + // Dropdowns + function populateDropdown(input, field) { + const wrapper = getWrapper(input); + const project = wrapper.querySelector('.project').value; + const instance = wrapper.querySelector('.instance').value; + + let items = []; + if (field === 'project') { + items = Object.keys(gcpData); + } else if (field === 'instance') { + if (gcpData[project]) { + items = Object.keys(gcpData[project].instances || {}); + } + } else if (field === 'database') { + if (gcpData[project] && gcpData[project].instances[instance]) { + items = gcpData[project].instances[instance]; + } + } + + const listEl = wrapper.querySelector(`.${field}-list`); + listEl.innerHTML = ''; + items.forEach(item => { + const div = document.createElement('div'); + div.textContent = item; + div.onclick = () => { + wrapper.querySelector(`.${field}`).value = item; + listEl.style.display = 'none'; + if (field === 'project') populateDropdown(input, 'instance'); + if (field === 'instance') populateDropdown(input, 'database'); + }; + listEl.appendChild(div); + }); + + listEl.style.display = 'block'; + } + + function filterDropdown(input, field) { + populateDropdown(input, field); + const wrapper = getWrapper(input); + const filter = input.value.toLowerCase(); + const listEl = wrapper.querySelector(`.${field}-list`); + Array.from(listEl.children).forEach(child => { + child.style.display = child.textContent.toLowerCase().includes(filter) ? 'block' : 'none'; + }); + } + + // ESC key close + document.addEventListener('keydown', e => { + if (e.key === 'Escape') { + document.querySelectorAll('.config-overlay.show').forEach(el => el.classList.remove('show')); + } + }); + \ No newline at end of file diff --git a/spanner_graphs/graph_visualization.py b/spanner_graphs/graph_visualization.py index 8a9cd77..fd57841 100644 --- a/spanner_graphs/graph_visualization.py +++ b/spanner_graphs/graph_visualization.py @@ -17,6 +17,7 @@ import base64 import uuid import os +import json from jinja2 import Template @@ -44,7 +45,7 @@ def _load_image(path: list[str]) -> str: with open(file_path, 'rb') as file: return base64.b64decode(file.read()).decode('utf-8') -def generate_visualization_html(query: str, port: int, params: str): +def generate_visualization_html(query: str, port: int, params: str, show_config_on_load: bool = False, gcp_data: str = "{}"): # Get the directory of the current file (magics.py) current_dir = os.path.dirname(os.path.abspath(__file__)) @@ -57,7 +58,7 @@ def generate_visualization_html(query: str, port: int, params: str): search_dir = parent template_content = _load_file([search_dir, 'frontend', 'static', 'jupyter.html']) - + # Load the JavaScript bundle directly js_file_path = os.path.join(search_dir, 'third_party', 'index.js') try: @@ -80,6 +81,8 @@ def generate_visualization_html(query: str, port: int, params: str): query=query, params=params, port=port, + show_config_on_load=show_config_on_load, + gcp_data=gcp_data, id=uuid.uuid4().hex # Prevent html/js selector collisions between cells ) diff --git a/spanner_graphs/magics.py b/spanner_graphs/magics.py index b412006..eb6d633 100644 --- a/spanner_graphs/magics.py +++ b/spanner_graphs/magics.py @@ -25,7 +25,7 @@ from threading import Thread import re -from IPython.core.display import HTML, JSON +from IPython.core.display import HTML, JSON, Javascript from IPython.core.magic import Magics, magics_class, cell_magic from IPython.display import display, clear_output from networkx import DiGraph @@ -39,6 +39,11 @@ validate_node_expansion_request ) from spanner_graphs.graph_visualization import generate_visualization_html +from google.cloud import spanner_admin_instance_v1, spanner_admin_database_v1 +from googleapiclient.discovery import build +from google.api_core.client_options import ClientOptions +import pydata_google_auth + singleton_server_thread: Thread = None @@ -118,6 +123,61 @@ def receive_node_expansion_request(request: dict, params_str: str): return JSON(execute_node_expansion(params_str, request)) except BaseException as e: return JSON({"error": e}) + +def get_default_credentials_with_project(): + credentials, _ = pydata_google_auth.default( + scopes=["https://www.googleapis.com/auth/cloud-platform"], + use_local_webserver=False + ) + return credentials + +def fetch_all_gcp_resources(credentials): + result = {} + try: + crm_service = build("cloudresourcemanager", "v1", credentials=credentials) + projects_resp = crm_service.projects().list().execute() + projects = projects_resp.get("projects", []) + + for project in projects: + project_id = project["projectId"] + result[project_id] = {"instances": {}} + + client_options = ClientOptions(quota_project_id=project_id) + instance_client = spanner_admin_instance_v1.InstanceAdminClient( + credentials=credentials, + client_options=client_options + ) + + try: + instances = instance_client.list_instances(parent=f"projects/{project_id}") + except Exception as e: + print(f"[!] Skipping project {project_id} due to instance error: {e}") + continue + + for instance in instances: + instance_id = instance.name.split("/")[-1] + result[project_id]["instances"][instance_id] = [] + + db_client = spanner_admin_database_v1.DatabaseAdminClient( + credentials=credentials, + client_options=client_options + ) + + try: + dbs = db_client.list_databases( + parent=f"projects/{project_id}/instances/{instance_id}" + ) + for db in dbs: + db_id = db.name.split("/")[-1] + result[project_id]["instances"][instance_id].append(db_id) + except Exception as e: + print(f"[!] Skipping databases for {project_id}/{instance_id}: {e}") + continue + except Exception as e: + print(f"[!] Error fetching GCP resources: {e}") + # Return an empty result if there's a broader error during fetching + return {} + return result @magics_class class NetworkVisualizationMagics(Magics): @@ -140,7 +200,7 @@ def __init__(self, shell): if not alive: singleton_server_thread = GraphServer.init() - def visualize(self): + def visualize(self, show_config_popup=False): """Helper function to create and display the visualization""" # Extract the graph name from the query (if present) graph = "" @@ -159,7 +219,9 @@ def visualize(self): "database": self.args.database, "mock": self.args.mock, "graph": graph - })) + }), + show_config_on_load=show_config_popup + ) display(HTML(html_content)) @cell_magic @@ -179,6 +241,70 @@ def spanner_graph(self, line: str, cell: str): help="Use mock database") try: + if not line.strip(): + self.args = argparse.Namespace( + project="", + instance="", + database="", + mock=False + ) + self.cell = cell + display(HTML(""" +
+
+
+
Authenticating and fetching GCP resources...
+
+ +
+ """)) + try: + credentials = get_default_credentials_with_project() + gcp_data = fetch_all_gcp_resources(credentials) + except Exception as e: + gcp_data = {} + print(f"Error fetching GCP resources: {e}") + + display(Javascript(""" + const loader = document.getElementById('loader-container'); + if (loader) loader.remove(); + """)) + + html_content = generate_visualization_html( + query=cell, + port=GraphServer.port, + params=json.dumps({ + "project": "", + "instance": "", + "database": "", + "mock": False, + "graph": "" + }), + gcp_data=json.dumps(gcp_data), # pass to HTML + show_config_on_load=True + ) + display(HTML(html_content)) + return + args = parser.parse_args(line.split()) if not args.mock: if not (args.project and args.instance and args.database): @@ -189,7 +315,7 @@ def spanner_graph(self, line: str, cell: str): print("Error: Query is required.") return - self.args = parser.parse_args(line.split()) + self.args = args self.cell = cell self.database = get_database_instance( self.args.project, @@ -197,7 +323,7 @@ def spanner_graph(self, line: str, cell: str): self.args.database, mock=self.args.mock) clear_output(wait=True) - self.visualize() + self.visualize(show_config_popup=False) except BaseException as e: print(f"Error: {e}") print("Usage: %%spanner_graph --project PROJECT_ID " From 0ec81a4f8c49d21249cc49553d8b07b18e0d24ed Mon Sep 17 00:00:00 2001 From: rusellvegh1201 Date: Fri, 11 Jul 2025 18:08:00 -0400 Subject: [PATCH 06/17] implement id based execution to avoid override cell output --- frontend/static/jupyter.html | 280 +++++++++++++++++++---------------- 1 file changed, 151 insertions(+), 129 deletions(-) diff --git a/frontend/static/jupyter.html b/frontend/static/jupyter.html index 3d81bd0..d57675d 100644 --- a/frontend/static/jupyter.html +++ b/frontend/static/jupyter.html @@ -1,4 +1,4 @@ -
-
- - - +
+
+ +
-
- - - +
+
+ +
- - -
- - - +
+
+ +
-
@@ -593,239 +571,196 @@

Configure Visualization

-
- \ No newline at end of file diff --git a/frontend/static/dev.js b/frontend/static/dev.js index f11e634..e69de29 100644 --- a/frontend/static/dev.js +++ b/frontend/static/dev.js @@ -1,15 +0,0 @@ -const CLIENT_ID = process.env.CLIENT_ID; -const SCOPE = process.env.SCOPE; -const REDIRECT_URI = process.env.REDIRECT_URI; - -document.getElementById("google-login-btn").addEventListener("click", () => { - const oauthUrl = - "https://accounts.google.com/o/oauth2/v2/auth?" + - `client_id=${CLIENT_ID}` + - "&response_type=token" + - `&scope=${SCOPE}` + - `&redirect_uri=${REDIRECT_URI}` + - "&prompt=consent"; - - window.open(oauthUrl, "_blank", "width=500,height=600"); -}); diff --git a/spanner_graphs/gcp_helper.py b/spanner_graphs/gcp_helper.py new file mode 100644 index 0000000..3aaed76 --- /dev/null +++ b/spanner_graphs/gcp_helper.py @@ -0,0 +1,61 @@ +from google.cloud import spanner_admin_instance_v1, spanner_admin_database_v1 +from googleapiclient.discovery import build +from google.api_core.client_options import ClientOptions +import pydata_google_auth + +class GcpHelper: + @staticmethod + def get_default_credentials_with_project(): + credentials, _ = pydata_google_auth.default( + scopes=["https://www.googleapis.com/auth/cloud-platform"], + use_local_webserver=False + ) + return credentials + @staticmethod + def fetch_all_gcp_resources(credentials): + result = {} + try: + crm_service = build("cloudresourcemanager", "v1", credentials=credentials) + projects_resp = crm_service.projects().list().execute() + projects = projects_resp.get("projects", []) + + for project in projects: + project_id = project["projectId"] + result[project_id] = {"instances": {}} + + client_options = ClientOptions(quota_project_id=project_id) + instance_client = spanner_admin_instance_v1.InstanceAdminClient( + credentials=credentials, + client_options=client_options + ) + + try: + instances = instance_client.list_instances(parent=f"projects/{project_id}") + except Exception as e: + print(f"[!] Skipping project {project_id} due to instance error: {e}") + continue + + for instance in instances: + instance_id = instance.name.split("/")[-1] + result[project_id]["instances"][instance_id] = [] + + db_client = spanner_admin_database_v1.DatabaseAdminClient( + credentials=credentials, + client_options=client_options + ) + + try: + dbs = db_client.list_databases( + parent=f"projects/{project_id}/instances/{instance_id}" + ) + for db in dbs: + db_id = db.name.split("/")[-1] + result[project_id]["instances"][instance_id].append(db_id) + except Exception as e: + print(f"[!] Skipping databases for {project_id}/{instance_id}: {e}") + continue + except Exception as e: + print(f"[!] Error fetching GCP resources: {e}") + # Return an empty result if there's a broader error during fetching + return {} + return result \ No newline at end of file diff --git a/spanner_graphs/graph_server.py b/spanner_graphs/graph_server.py index 3ab8872..1cc83f9 100644 --- a/spanner_graphs/graph_server.py +++ b/spanner_graphs/graph_server.py @@ -28,6 +28,7 @@ from spanner_graphs.exec_env import get_database_instance from spanner_graphs.database import SpannerQueryResult from google.cloud import spanner +from spanner_graphs.gcp_helper import GcpHelper # Supported types for a property PROPERTY_TYPE_SET = { @@ -42,8 +43,6 @@ 'STRING', 'TIMESTAMP' } -saved_user_config = {} - class NodePropertyForDataExploration: def __init__(self, key: str, value: Union[str, int, float, bool], type_str: str): @@ -333,9 +332,7 @@ class GraphServer: "post_ping": "/post_ping", "post_query": "/post_query", "post_node_expansion": '/post_node_expansion', - "oauth2callback": '/oauth2callback', - "save_config": '/save_config', - "get_saved_config": '/get_saved_config' + "gcp_resources": '/gcp_resources' } _server = None @@ -474,53 +471,21 @@ def handle_post_node_expansion(self): self.do_error_response(e) return - def handle_oauth2callback(self): - self.send_response(200) - self.send_header("Content-type", "text/html") - self.end_headers() - html = """ - - """ - self.wfile.write(html.encode("utf-8")) - - def handle_save_user_data(self): - print("handle user data function") - global saved_user_config + def get_gcp_resources(self): try: - data = self.parse_post_data() - saved_user_config = data - self.do_json_response({"status": "saved"}) + # Always fetch fresh data + credentials = GcpHelper.get_default_credentials_with_project() + gcp_data = GcpHelper.fetch_all_gcp_resources(credentials) + self.do_json_response(gcp_data) except Exception as e: self.do_error_response(str(e)) - - def get_saved_user_data(self): - print("get user data") - global saved_user_config - self.do_data_response(saved_user_config) + def do_GET(self): if self.path == GraphServer.endpoints["get_ping"]: self.handle_get_ping() - elif self.path == GraphServer.endpoints["oauth2callback"]: - self.handle_oauth2callback() - elif self.path == GraphServer.endpoints["get_saved_config"]: - self.get_saved_user_data() + elif self.path == GraphServer.endpoints["gcp_resources"]: + self.get_gcp_resources() else: super().do_GET() @@ -531,8 +496,5 @@ def do_POST(self): self.handle_post_query() elif self.path == GraphServer.endpoints["post_node_expansion"]: self.handle_post_node_expansion() - elif self.path == GraphServer.endpoints['save_config']: - print("call save config api") - self.handle_save_user_data() atexit.register(GraphServer.stop_server) diff --git a/spanner_graphs/magics.py b/spanner_graphs/magics.py index eb6d633..441758a 100644 --- a/spanner_graphs/magics.py +++ b/spanner_graphs/magics.py @@ -43,6 +43,7 @@ from googleapiclient.discovery import build from google.api_core.client_options import ClientOptions import pydata_google_auth +from spanner_graphs.gcp_helper import GcpHelper singleton_server_thread: Thread = None @@ -123,61 +124,6 @@ def receive_node_expansion_request(request: dict, params_str: str): return JSON(execute_node_expansion(params_str, request)) except BaseException as e: return JSON({"error": e}) - -def get_default_credentials_with_project(): - credentials, _ = pydata_google_auth.default( - scopes=["https://www.googleapis.com/auth/cloud-platform"], - use_local_webserver=False - ) - return credentials - -def fetch_all_gcp_resources(credentials): - result = {} - try: - crm_service = build("cloudresourcemanager", "v1", credentials=credentials) - projects_resp = crm_service.projects().list().execute() - projects = projects_resp.get("projects", []) - - for project in projects: - project_id = project["projectId"] - result[project_id] = {"instances": {}} - - client_options = ClientOptions(quota_project_id=project_id) - instance_client = spanner_admin_instance_v1.InstanceAdminClient( - credentials=credentials, - client_options=client_options - ) - - try: - instances = instance_client.list_instances(parent=f"projects/{project_id}") - except Exception as e: - print(f"[!] Skipping project {project_id} due to instance error: {e}") - continue - - for instance in instances: - instance_id = instance.name.split("/")[-1] - result[project_id]["instances"][instance_id] = [] - - db_client = spanner_admin_database_v1.DatabaseAdminClient( - credentials=credentials, - client_options=client_options - ) - - try: - dbs = db_client.list_databases( - parent=f"projects/{project_id}/instances/{instance_id}" - ) - for db in dbs: - db_id = db.name.split("/")[-1] - result[project_id]["instances"][instance_id].append(db_id) - except Exception as e: - print(f"[!] Skipping databases for {project_id}/{instance_id}: {e}") - continue - except Exception as e: - print(f"[!] Error fetching GCP resources: {e}") - # Return an empty result if there's a broader error during fetching - return {} - return result @magics_class class NetworkVisualizationMagics(Magics): @@ -278,8 +224,8 @@ def spanner_graph(self, line: str, cell: str):
""")) try: - credentials = get_default_credentials_with_project() - gcp_data = fetch_all_gcp_resources(credentials) + credentials = GcpHelper.get_default_credentials_with_project() + gcp_data = GcpHelper.fetch_all_gcp_resources(credentials) except Exception as e: gcp_data = {} print(f"Error fetching GCP resources: {e}") From 6f68cbd2c4ed47836c4b6bb184f693c0579a4a06 Mon Sep 17 00:00:00 2001 From: rusellvegh1201 Date: Tue, 15 Jul 2025 12:27:49 -0400 Subject: [PATCH 08/17] added validation --- frontend/static/dev.html | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/frontend/static/dev.html b/frontend/static/dev.html index e804462..b4f6a03 100644 --- a/frontend/static/dev.html +++ b/frontend/static/dev.html @@ -833,6 +833,20 @@

Configure Visualization

function handleSubmit(e) { e.preventDefault(); + const mock = document.getElementById('mock').checked; + + if (!mock) { + const project = document.getElementById('project').value; + const instance = document.getElementById('instance').value; + const database = document.getElementById('database').value; + const query = document.getElementById('query').value; + + if (!project || !instance || !database || !query) { + Helpers.showToast("input must not be empty"); + return; + } + } + toggleCommandPalette(); initializeApp(); } From e7550b11a7a9cabd1792d9628cf9736acb943f38 Mon Sep 17 00:00:00 2001 From: rusellvegh1201 Date: Tue, 15 Jul 2025 13:07:04 -0400 Subject: [PATCH 09/17] added required packages --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3a9a1c4..2e81f50 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ def package_files(directory): install_requires=[ "networkx", "numpy", "google-cloud-spanner", "ipython", "ipywidgets", "notebook", "requests", "portpicker", - "pydata-google-auth" + "pydata-google-auth", "spanner", "googleapiclient" ], include_package_data=True, description='Visually query Spanner Graph data in notebooks.', From 95a8a5a0eb4e12cc43222bd69ce7eaf292204527 Mon Sep 17 00:00:00 2001 From: Russell Vegh <96844463+rusellvegh1201@users.noreply.github.com> Date: Tue, 15 Jul 2025 16:22:26 -0400 Subject: [PATCH 10/17] Update setup.py google-api-python-client --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2e81f50..ea5deee 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ def package_files(directory): install_requires=[ "networkx", "numpy", "google-cloud-spanner", "ipython", "ipywidgets", "notebook", "requests", "portpicker", - "pydata-google-auth", "spanner", "googleapiclient" + "pydata-google-auth", "spanner", "google-api-python-client" ], include_package_data=True, description='Visually query Spanner Graph data in notebooks.', From 0b9099357457fd6182be1e93aa515107b8a549cd Mon Sep 17 00:00:00 2001 From: rusellvegh1201 Date: Tue, 15 Jul 2025 18:43:50 -0400 Subject: [PATCH 11/17] clean magics file , update setup file --- sample.ipynb | 3263 +++++++++++++++++++++++++++++++++++++- setup.py | 2 +- spanner_graphs/magics.py | 83 +- 3 files changed, 3296 insertions(+), 52 deletions(-) diff --git a/sample.ipynb b/sample.ipynb index 72b82a9..4779334 100644 --- a/sample.ipynb +++ b/sample.ipynb @@ -32,10 +32,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "e2b05db4-01c6-4e6f-96f3-f964d7f05786", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Spanner Graph Notebook loaded\n" + ] + } + ], "source": [ "%load_ext spanner_graphs" ] @@ -54,10 +62,1596 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "43e00d3c-add4-451c-999d-44807a7e80da", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "
\n", + " \n", + "

Configure Visualization

\n", + "\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "
\n", + "\n", + "
\n", + " \n", + " \n", + "
\n", + "\n", + "
\n", + " \n", + " \n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "%%spanner_graph --mock\n", "\n" @@ -71,6 +1665,1665 @@ "## Query and visualize Spanner Graph data\n" ] }, + { + "cell_type": "markdown", + "id": "032be8df-a4e4-4116-b4ea-6cc118aaa68e", + "metadata": {}, + "source": [ + "run the following, that will show config pop where you can select your resources and visualize the graph" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "e2051ee5-85c0-41a2-9b9b-f045f3ef62ae", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + "
\n", + "
\n", + "
Authenticating and fetching GCP resources...
\n", + "
\n", + " \n", + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "\n", + " const loader = document.getElementById('loader-container');\n", + " if (loader) loader.remove();\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "
\n", + " \n", + "

Configure Visualization

\n", + "\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "
\n", + "\n", + "
\n", + " \n", + " \n", + "
\n", + "\n", + "
\n", + " \n", + " \n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "%%spanner_graph\n", + " " + ] + }, { "cell_type": "markdown", "id": "671d1d2e-ec9a-4ead-bb99-2e4cd4d9e099", @@ -113,7 +3366,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.8" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/setup.py b/setup.py index ea5deee..4de568b 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ def package_files(directory): install_requires=[ "networkx", "numpy", "google-cloud-spanner", "ipython", "ipywidgets", "notebook", "requests", "portpicker", - "pydata-google-auth", "spanner", "google-api-python-client" + "pydata-google-auth", "google-api-python-client" ], include_package_data=True, description='Visually query Spanner Graph data in notebooks.', diff --git a/spanner_graphs/magics.py b/spanner_graphs/magics.py index 441758a..68e531b 100644 --- a/spanner_graphs/magics.py +++ b/spanner_graphs/magics.py @@ -16,38 +16,59 @@ import argparse import base64 -import random -import uuid -from enum import Enum, auto import json import os -import sys from threading import Thread import re from IPython.core.display import HTML, JSON, Javascript from IPython.core.magic import Magics, magics_class, cell_magic from IPython.display import display, clear_output -from networkx import DiGraph -import ipywidgets as widgets -from ipywidgets import interact -from jinja2 import Template from spanner_graphs.exec_env import get_database_instance from spanner_graphs.graph_server import ( GraphServer, execute_query, execute_node_expansion, - validate_node_expansion_request ) from spanner_graphs.graph_visualization import generate_visualization_html -from google.cloud import spanner_admin_instance_v1, spanner_admin_database_v1 -from googleapiclient.discovery import build -from google.api_core.client_options import ClientOptions -import pydata_google_auth from spanner_graphs.gcp_helper import GcpHelper singleton_server_thread: Thread = None +SHOW_LOADER = """ +
+
+
+
Authenticating and fetching GCP resources...
+
+ +
+""" + +REMOVE_LOADER = """ +const loader = document.getElementById('loader-container'); +if (loader) loader.remove(); +""" + def _load_file(path: list[str]) -> str: file_path = os.path.sep.join(path) if not os.path.exists(file_path): @@ -195,45 +216,15 @@ def spanner_graph(self, line: str, cell: str): mock=False ) self.cell = cell - display(HTML(""" -
-
-
-
Authenticating and fetching GCP resources...
-
- -
- """)) + display(HTML(SHOW_LOADER)) try: credentials = GcpHelper.get_default_credentials_with_project() gcp_data = GcpHelper.fetch_all_gcp_resources(credentials) except Exception as e: gcp_data = {} print(f"Error fetching GCP resources: {e}") - - display(Javascript(""" - const loader = document.getElementById('loader-container'); - if (loader) loader.remove(); - """)) + + display(Javascript(REMOVE_LOADER)) html_content = generate_visualization_html( query=cell, From d7f5732c9d332112e910d505dd285bf35f1e8b40 Mon Sep 17 00:00:00 2001 From: rusellvegh1201 Date: Tue, 15 Jul 2025 19:10:58 -0400 Subject: [PATCH 12/17] update readme file --- README.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/README.md b/README.md index 77fbe86..a2865cb 100644 --- a/README.md +++ b/README.md @@ -163,3 +163,52 @@ npm install npm run test:unit npm run test:visual ``` + + + + +# Project setup + +```Clone the Project Repository`` +Run the following command to clone the repository: + +# git clone + +```Then navigate into the project directory``` + +# cd spanner-graph-notebook + +```Create and Activate a Virtual Environment``` + +# python -m venv venv + +```Activate the virtual environment``` + +# source venv/bin/activate + + +```Install Backend Python Packages``` +From the root project directory (spanner-graph-notebook), install the required Python packages: + +# pip install -e . + +```Install Frontend Dependencies``` + +```avigate to the frontend directory``` + +cd frontend + +```Install the Node.js dependencies``` + +npm install + +```Start the Backend Server``` +Go to the backend development utility directory: + +cd ../spanner_graphs/dev_util + +```Start the development server``` + +# python serve_dev.py + + From 5b8f8ebe4ed15cfed6912357b922de38a83b2c7f Mon Sep 17 00:00:00 2001 From: rusellvegh1201 Date: Sat, 19 Jul 2025 14:13:06 -0400 Subject: [PATCH 13/17] reconfigure code, single percent sing %, make a dynamic loader --- frontend/src/authLoader.js | 141 ++++++++++++++++++++++++++ frontend/src/graph-server.js | 4 +- frontend/src/loader.js | 19 ++++ frontend/static/constent.js | 0 frontend/static/dev.js | 0 frontend/static/jupyter.html | 137 ++++++++++++++++++++----- frontend/static/loader.html | 66 ++++++++++++ spanner_graphs/cloud_database.py | 6 +- spanner_graphs/database.py | 2 +- spanner_graphs/gcp_helper.py | 42 ++++++++ spanner_graphs/graph_entities.py | 1 - spanner_graphs/graph_server.py | 59 ++++++++++- spanner_graphs/graph_visualization.py | 13 ++- spanner_graphs/magics.py | 95 +++-------------- spanner_graphs/utils.py | 44 ++++++++ 15 files changed, 507 insertions(+), 122 deletions(-) create mode 100644 frontend/src/authLoader.js create mode 100644 frontend/src/loader.js delete mode 100644 frontend/static/constent.js delete mode 100644 frontend/static/dev.js create mode 100644 frontend/static/loader.html create mode 100644 spanner_graphs/utils.py diff --git a/frontend/src/authLoader.js b/frontend/src/authLoader.js new file mode 100644 index 0000000..665a8ff --- /dev/null +++ b/frontend/src/authLoader.js @@ -0,0 +1,141 @@ +class LoaderManager { + constructor() { + this.templateCache = null; + } + + createLoaderElement(message) { + const loaderContainer = document.createElement('div'); + + Object.assign(loaderContainer.style, { + position: 'absolute', + top: '0', + left: '0', + right: '0', + bottom: '0', + width: '100%', + height: '100%', + background: 'rgba(255, 255, 255, 0.9)', + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + zIndex: '99999', + pointerEvents: 'all', + borderRadius: '8px' + }); + + const wrapper = document.createElement('div'); + Object.assign(wrapper.style, { + textAlign: 'center', + padding: '20px' + }); + + const spinner = document.createElement('div'); + Object.assign(spinner.style, { + border: '6px solid #f3f3f3', + borderTop: '6px solid #4285F4', + borderRadius: '50%', + width: '40px', + height: '40px', + margin: 'auto', + animation: 'spin 1s linear infinite' + }); + + const messageEl = document.createElement('div'); + Object.assign(messageEl.style, { + marginTop: '10px', + fontSize: '16px', + color: '#333', + fontFamily: 'Arial, sans-serif' + }); + messageEl.textContent = message; + + if (!document.getElementById('loader-keyframes')) { + const style = document.createElement('style'); + style.id = 'loader-keyframes'; + style.textContent = ` + @keyframes spin { + 0% { transform: rotate(0deg); } + 100% { transform: rotate(360deg); } + } + `; + document.head.appendChild(style); + } + + // Assemble elements + wrapper.appendChild(spinner); + wrapper.appendChild(messageEl); + loaderContainer.appendChild(wrapper); + + return loaderContainer; + } + + async showLoader(message = "Loading...", container = null) { + const loaderId = container ? `auth-loader-${container.dataset.id || 'default'}` : 'auth-loader'; + + // Remove existing loader first + const existingLoader = document.getElementById(loaderId); + if (existingLoader) { + existingLoader.remove(); + } + + const loaderContainer = this.createLoaderElement(message); + loaderContainer.id = loaderId; + + if (container) { + const originalPosition = window.getComputedStyle(container).position; + + if (originalPosition === 'static' || originalPosition === '') { + container.style.position = 'relative'; + loaderContainer.dataset.originalPosition = ''; + } else { + loaderContainer.dataset.originalPosition = container.style.position; + } + + if (container.offsetHeight === 0) { + container.style.minHeight = '200px'; + } + + container.appendChild(loaderContainer); + } else { + Object.assign(loaderContainer.style, { + position: 'fixed', + top: '0', + left: '0', + width: '100vw', + height: '100vh' + }); + document.body.appendChild(loaderContainer); + } + loaderContainer.offsetHeight; + + setTimeout(() => { + const computedStyle = window.getComputedStyle(loaderContainer); + }, 100); + + return loaderContainer; + } + + removeLoader(container = null) { + const loaderId = container ? `auth-loader-${container.dataset.id || 'default'}` : 'auth-loader'; + const loader = document.getElementById(loaderId); + + if (loader) { + if (container) { + const originalPosition = loader.dataset.originalPosition; + if (originalPosition !== undefined) { + if (originalPosition === '') { + container.style.position = 'static'; + } else { + container.style.position = originalPosition; + } + } + } + + loader.remove(); + } + } +} + +const loader = new LoaderManager(); +window.showLoader = loader.showLoader.bind(loader); +window.removeLoader = loader.removeLoader.bind(loader); \ No newline at end of file diff --git a/frontend/src/graph-server.js b/frontend/src/graph-server.js index 8e989d1..2e25c41 100644 --- a/frontend/src/graph-server.js +++ b/frontend/src/graph-server.js @@ -30,9 +30,7 @@ class GraphServer { getPing: '/get_ping', postQuery: '/post_query', postNodeExpansion: '/post_node_expansion', - // saveConfig: '/save_config', - // getSavedConfig: '/get_saved_config', - gcpResources:'/gcp_resources' + gcpResources:'/gcp_resources', }; /** diff --git a/frontend/src/loader.js b/frontend/src/loader.js new file mode 100644 index 0000000..0677809 --- /dev/null +++ b/frontend/src/loader.js @@ -0,0 +1,19 @@ +/** + * Copyright 2025 Google LLC + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +function removeLoader() { + const loader = document.getElementById('auth-loader-container'); + if (loader) loader.remove(); +} diff --git a/frontend/static/constent.js b/frontend/static/constent.js deleted file mode 100644 index e69de29..0000000 diff --git a/frontend/static/dev.js b/frontend/static/dev.js deleted file mode 100644 index e69de29..0000000 diff --git a/frontend/static/jupyter.html b/frontend/static/jupyter.html index d57675d..0493a5c 100644 --- a/frontend/static/jupyter.html +++ b/frontend/static/jupyter.html @@ -198,7 +198,7 @@ } -
+
@@ -259,7 +259,8 @@

Configure Visualization

@@ -508,10 +508,10 @@ -
+
-
Authenticating and Faching Resources
+
Fetching Resources ...
@@ -519,7 +519,7 @@
-