diff --git a/pkg/pip_requirements.txt b/pkg/pip_requirements.txt index 64474df..4aab298 100644 --- a/pkg/pip_requirements.txt +++ b/pkg/pip_requirements.txt @@ -2,3 +2,4 @@ spaceone-api typing-inspect python-multipart PyJWT +openpyxl diff --git a/src/cloudforet/console_api_v2/conf/global_conf.py b/src/cloudforet/console_api_v2/conf/global_conf.py index b6d83e0..7b796cb 100644 --- a/src/cloudforet/console_api_v2/conf/global_conf.py +++ b/src/cloudforet/console_api_v2/conf/global_conf.py @@ -70,7 +70,7 @@ "default": {}, "local": { "backend": "spaceone.core.cache.local_cache.LocalCache", - "max_size": 128, + "max_size": 1024, "ttl": 300, }, } diff --git a/src/cloudforet/console_api_v2/conf/router_conf.py b/src/cloudforet/console_api_v2/conf/router_conf.py index df85910..1ef3260 100644 --- a/src/cloudforet/console_api_v2/conf/router_conf.py +++ b/src/cloudforet/console_api_v2/conf/router_conf.py @@ -20,6 +20,13 @@ "tags": ["console-api > extension > agent"], }, }, + { + "router_path": "cloudforet.console_api_v2.interface.rest.extension.excel:router", + "router_options": { + "prefix": "/console-api/extension/excel", + "tags": ["console-api > extension > excel"], + }, + }, { "router_path": "cloudforet.console_api_v2.interface.rest.swagger:router", }, diff --git a/src/cloudforet/console_api_v2/interface/rest/extension/excel.py b/src/cloudforet/console_api_v2/interface/rest/extension/excel.py new file mode 100644 index 0000000..46c2b7b --- /dev/null +++ b/src/cloudforet/console_api_v2/interface/rest/extension/excel.py @@ -0,0 +1,68 @@ +import json +import logging +import inspect +from fastapi import APIRouter, Depends, Request, Query +from fastapi.concurrency import run_in_threadpool +from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials +from starlette.responses import StreamingResponse + +from spaceone.core import cache +from spaceone.core.error import ERROR_REQUIRED_PARAMETER, ERROR_CACHE_CONFIGURATION +from spaceone.core.fastapi.api import BaseAPI, exception_handler + +from cloudforet.console_api_v2.service.excel_service import ExcelService + +_LOGGER = logging.getLogger(__name__) +_AUTH_SCHEME = HTTPBearer(auto_error=False) + +router = APIRouter(include_in_schema=True) + +SERVICE = "console-api" +RESOURCE = "Excel" + + +@router.post("/export") +@exception_handler +async def export( + request: Request, token: HTTPAuthorizationCredentials = Depends(_AUTH_SCHEME) +) -> dict: + base_api = BaseAPI() + base_api.service = SERVICE + verb = inspect.currentframe().f_code.co_name + + if token: + params, metadata = await base_api.parse_request( + request, token=token.credentials, resource=RESOURCE, verb=verb + ) + else: + params, metadata = await base_api.parse_request( + request, None, resource=RESOURCE, verb=verb + ) + + excel_service = ExcelService(metadata) + + response = await run_in_threadpool(excel_service.export, params) + return response + + +@router.get("/download") +@exception_handler +async def download(key: str = Query(default=None)) -> StreamingResponse: + base_api = BaseAPI() + base_api.service = SERVICE + + if not key: + raise ERROR_REQUIRED_PARAMETER(key="key") + + if not cache.is_set(alias="local"): + raise ERROR_CACHE_CONFIGURATION(alias="local") + + json_str = cache.get(key=f"console-api:excel:{key}", alias="local") + json_obj = json.loads(json_str) + + params = json_obj["request_body"] + metadata = json_obj["auth_info"] + + excel_service = ExcelService(metadata) + response = await run_in_threadpool(excel_service.download, params) + return response diff --git a/src/cloudforet/console_api_v2/manager/cloudforet_manager.py b/src/cloudforet/console_api_v2/manager/cloudforet_manager.py index f6a5776..8c09be6 100644 --- a/src/cloudforet/console_api_v2/manager/cloudforet_manager.py +++ b/src/cloudforet/console_api_v2/manager/cloudforet_manager.py @@ -1,4 +1,5 @@ import logging +from typing import Tuple, Generator from spaceone.core.connector.space_connector import SpaceConnector from spaceone.core.manager import BaseManager @@ -10,16 +11,58 @@ class CloudforetManager(BaseManager): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self._page_size = 1000 def dispatch_api(self, grpc_method: str, params: dict, token: str = None): service, resource, verb = self._parse_grpc_method(grpc_method) space_connector = SpaceConnector(service=service, token=token) return space_connector.dispatch(f"{resource}.{verb}", params) + def paginate_api( + self, grpc_method: str, params: dict, token: str = None, limit: int = 1000 + ) -> Generator[dict, None, None]: + start = 1 + paged_params = params.copy() + while True: + query = paged_params.get("query", {}) + page_info = query.get("page", {}) + page_info["start"] = start + page_info["limit"] = max(page_info.get("limit", self._page_size), limit) + + paged_params["query"].update({"page": page_info}) + + response = self.dispatch_api(grpc_method, paged_params, token) + results = response.get("results", []) + yield response + + if len(results) < page_info["limit"]: + break + + start += page_info["limit"] + @staticmethod - def _parse_grpc_method(grpc_method): + def _parse_grpc_method(grpc_method: str) -> Tuple[str, str, str]: try: service, resource, verb = grpc_method.split(".") return service, resource, verb except Exception as e: raise ERROR_PARSE_GRPC_METHOD(grpc_method=grpc_method, reason=e) + + @classmethod + def convert_grpc_method_from_url(cls, url: str) -> str: + try: + parts = url.strip("/").split("/") + print(parts) + + if len(parts) != 3: + raise ValueError("Path must have at least two segments") + + service = parts[0].replace("-", "_") # snake_case + r_source = "".join( + word.capitalize() for word in parts[1].split("-") + ) # PascalCase + verb = parts[2].replace("-", "_") + + return ".".join([service, r_source, verb]) + except Exception as e: + raise ERROR_PARSE_GRPC_METHOD(grpc_method=url, reason=e) diff --git a/src/cloudforet/console_api_v2/model/auth/request.py b/src/cloudforet/console_api_v2/model/auth/request.py deleted file mode 100644 index 133f02a..0000000 --- a/src/cloudforet/console_api_v2/model/auth/request.py +++ /dev/null @@ -1,6 +0,0 @@ -from cloudforet.console_api_v2.model import BaseAPIModel -from pydantic import Field - - -class AuthBasicAuthRequest(BaseAPIModel): - http_authorization: str = Field(description='Token for authentication', default="Basic encoded_token") diff --git a/src/cloudforet/console_api_v2/model/auth/__init__.py b/src/cloudforet/console_api_v2/model/excel/__init__.py similarity index 100% rename from src/cloudforet/console_api_v2/model/auth/__init__.py rename to src/cloudforet/console_api_v2/model/excel/__init__.py diff --git a/src/cloudforet/console_api_v2/model/excel/request.py b/src/cloudforet/console_api_v2/model/excel/request.py new file mode 100644 index 0000000..ccb064b --- /dev/null +++ b/src/cloudforet/console_api_v2/model/excel/request.py @@ -0,0 +1,24 @@ +from typing import Union +from pydantic import BaseModel + +__all__ = ["ExcelExportRequest"] + + +class ExcelSource(BaseModel): + url: Union[str, None] = None + param: Union[dict, None] = None + data: Union[dict, list, None] = None + + +class ExcelTemplate(BaseModel): + options: Union[dict, None] = None + fields: Union[list, None] = None + + +class ExcelExportRequest(BaseModel): + source: ExcelSource + template: ExcelTemplate + + +class ExcelDownloadRequest(BaseModel): + key: str diff --git a/src/cloudforet/console_api_v2/model/resource.py b/src/cloudforet/console_api_v2/model/resource.py deleted file mode 100644 index 9d60b6a..0000000 --- a/src/cloudforet/console_api_v2/model/resource.py +++ /dev/null @@ -1,47 +0,0 @@ -from cloudforet.console_api_v2.model import BaseAPIModel -from pydantic import Field -from typing import Union, List - - -class ListFieldsRequest(BaseAPIModel): - service: str = Field(..., description='Service name (identity, inventory, etc.)', examples=['inventory','identity']) - resource: str = Field(..., description='Resource name') - options: Union[dict, None] = Field(default=None, description='Additional options for each resource') - limit: Union[int, None] = Field(None, description='Set the number of returns') - - class Config: - description = 'List fields of API Resource.' - - -class FieldInfo(BaseAPIModel): - key: str = Field(None) - name: str = Field(None) - nested: bool = Field(None) - - -class FieldsInfo(BaseAPIModel): - results: List[FieldInfo] = Field(None, description='List of fields') - more: bool = Field(None) - - -class ListFieldValuesRequest(BaseAPIModel): - service: str = Field(..., description='Service name (identity, inventory, etc.)') - resource: str = Field(..., description='Resource name') - field: str = Field(..., description='field name') - options: Union[dict, None] = Field(None, description='Additional options for each resource') - search: Union[str, None] = Field(None, description='search keywords for value') - limit: Union[int, None] = Field(None, description='Set the number of returns') - - class Config: - description = 'List values of API resource field' - - -class ValueInfo(BaseAPIModel): - key: str = Field(None) - name: str = Field(None) - - -class ValuesInfo(BaseAPIModel): - results: List[ValueInfo] = Field(None) - more: bool = Field(None) - diff --git a/src/cloudforet/console_api_v2/service/excel_service.py b/src/cloudforet/console_api_v2/service/excel_service.py new file mode 100644 index 0000000..14bfea1 --- /dev/null +++ b/src/cloudforet/console_api_v2/service/excel_service.py @@ -0,0 +1,465 @@ +import logging +import json +import uuid +import pytz +from datetime import datetime +from io import BytesIO +from openpyxl import Workbook +from typing import Optional, Any, Union, List, Dict +from openpyxl.worksheet.worksheet import Worksheet +from openpyxl.styles import Border, Side, PatternFill, Font, Alignment +from starlette.responses import StreamingResponse + +from spaceone.core import cache +from spaceone.core import utils +from spaceone.core.service import * +from spaceone.core.error import ERROR_CACHE_CONFIGURATION + +from cloudforet.console_api_v2.manager.cloudforet_manager import CloudforetManager +from cloudforet.console_api_v2.model.excel.request import ExcelExportRequest + +_LOGGER = logging.getLogger(__name__) + + +@authentication_handler +@event_handler +class ExcelService(BaseService): + resource = "Excel" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + @transaction() + def export(self, params: dict) -> dict: + if cache.is_set(alias="local"): + self._check_excel_export_params(params) + + redis_key = uuid.uuid4() + key_dict = { + "request_body": params, + "auth_info": { + "token": self.metadata.get("token"), + "domain_id": self.transaction.get_meta("authorization.domain_id"), + "audience": self.transaction.get_meta("authorization.audience"), + "owner_type": self.transaction.get_meta("authorization.owner_type"), + }, + } + + redis_value = json.dumps(key_dict) + cache.set( + key=f"console-api:excel:{redis_key}", value=redis_value, alias="local" + ) + else: + raise ERROR_CACHE_CONFIGURATION() + + return {"file_link": f"/console-api/extension/excel/download?key={redis_key}"} + + @transaction() + @convert_model + def download(self, params: ExcelExportRequest) -> StreamingResponse: + + params = params.dict() + wb = self._create_excel(params) + + stream = BytesIO() + wb.save(stream) + stream.seek(0) + + file_name = self._get_file_name(params) + + headers = { + "Content-Disposition": f"attachment; filename={file_name}", + "Content-Type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + } + + return StreamingResponse( + stream, + media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + headers=headers, + ) + + @staticmethod + def _check_excel_export_params(options: dict, array_idx: int = None): + source = options.get("source") + template = options.get("template") + + def idx_msg(): + return f" in {array_idx}th index" if array_idx is not None else "" + + if not source: + raise Exception(f"Required Parameter. (key = source{idx_msg()})") + if not template: + raise Exception(f"Required Parameter. (key = template{idx_msg()})") + + if "data" not in source: + if not source.get("url") or not source.get("param"): + raise Exception( + f"Invalid Parameter. (source{idx_msg()} = must have url and param keys)" + ) + if not isinstance(source["url"], str): + raise Exception( + f"Parameter type is invalid. (source.url{idx_msg()} = string)" + ) + if "query" not in source["param"]: + raise Exception( + f"Invalid Parameter. (source.param{idx_msg()} = must have query)" + ) + + def _create_excel(self, excel_options: Union[dict, List[dict]]) -> Workbook: + wb = Workbook() + wb.remove(wb.active) # Remove default sheet + + if isinstance(excel_options, list): + for opt in excel_options: + self._create_worksheet(wb, opt) + else: + self._create_worksheet(wb, excel_options) + + return wb + + def _create_worksheet(self, workbook: Workbook, excel_option: dict) -> Worksheet: + template = excel_option.get("template", {}) + options = template.get("options", {}) + sheet_name = options.get("sheet_name", "Sheet1") + fields = template.get("fields", []) + header_message = options.get("header_message", {}) + + source = excel_option.get("source", {}) + + # New worksheet + ws = workbook.create_sheet(title=sheet_name) + + # set worksheet headers + columns = self._get_excel_columns(fields) + self._set_excel_header(ws, columns, header_message) + + # add rows at worksheet + if source: + ws = self._set_excel_cell_rows(ws, source, fields, options) + + # set worksheet style + self._set_worksheet_row_style(ws, header_message) + self._set_worksheet_column_style(ws, header_message) + + return ws + + def _set_excel_cell_rows( + self, worksheet: Worksheet, source: dict, fields: list, options: dict + ) -> Worksheet: + cf_mgr = CloudforetManager() + token = self.transaction.get_meta("authorization.token") + reference_resource_map = self._get_reference_resource_map(cf_mgr, fields, token) + + timezone = options.get("timezone", "UTC") + + if data := source.get("data"): + raw_data = data + else: + raw_data = self._get_raw_data(cf_mgr, source, token) + + # add rows + for row in raw_data: + excel_row_data = {} + + for field in fields: + key = field["key"] + name = field.get("name") or key + field_type = field.get("type") + reference = field.get("reference") + options = field.get("options", {}) + enum_items = field.get("enum_items", {}) + + row_value = self.get_value_by_path(row, key) + + if reference: + resource_type = reference["resource_type"] + reference_key = reference["reference_key"] + reference_resources = reference_resource_map.get(resource_type, []) + row_value = self._convert_value_with_reference_data( + row_value, reference_key, reference_resources + ) + + if row_value is None: + value = "" + elif field_type == "datetime": + dt = datetime.fromisoformat(row_value) + tz = pytz.timezone(timezone) + value = dt.astimezone(tz).strftime("%Y-%m-%d %H:%M:%S") + + elif field_type == "currency": + value = row_value + elif field_type == "enum": + value = enum_items.get(row_value, row_value) + elif field_type == "list": + value = ", ".join(row_value) + + elif isinstance(row_value, list): + value = "\n".join([str(v) for v in row_value]) + elif isinstance(row_value, int) or isinstance(row_value, float): + value = row_value + else: + value = str(row_value) + + excel_row_key = field.get("name", key) + excel_row_data[excel_row_key] = value + + excel_data_values = [] + for value in excel_row_data.values(): + excel_data_values.append(value) + + worksheet.append(excel_data_values) + return worksheet + + def _set_excel_header( + self, worksheet: Worksheet, columns: list, header_message: dict + ): + header_title = header_message.get("title") + header_row_number = 2 if header_title else 1 + + headers = [column["header"] for column in columns] + if header_title: + self._set_excel_header_message(worksheet, header_title) + for col_index, header in enumerate(headers, start=1): + worksheet.cell(row=header_row_number, column=col_index, value=header) + else: + worksheet.append(headers) + + # set header style + self._set_worksheet_header_style(worksheet, header_row_number, len(columns)) + + # set column width + ExcelService._set_worksheet_column_style(worksheet, header_message) + + def _set_excel_header_message(self, worksheet: Worksheet, title: str): + worksheet.insert_rows(1) + worksheet["A1"].value = title + self._set_header_message_style(worksheet) + + @staticmethod + def _set_header_message_style(worksheet: Worksheet): + cell = worksheet["A1"] + cell.font = Font(bold=True, size=22, color="003566") + cell.alignment = Alignment(vertical="bottom", horizontal="left") + + @staticmethod + def _get_excel_columns(fields: List[dict]) -> List[Dict]: + return [ + { + "header": field["name"], + "key": field["key"], + "height": 24, + "style": { + "font": {"size": 12}, + "alignment": { + "vertical": "top", + "horizontal": "left", + "wrapText": True, + }, + }, + } + for field in fields + ] + + def _set_worksheet_header_style( + self, worksheet: Worksheet, header_row_number: int, column_length: int + ): + for num in range(0, column_length): + column_letter = self._get_column_letter(num) + cell = worksheet[f"{column_letter}{header_row_number}"] + + cell.fill = PatternFill(fill_type="solid", fgColor="003566") + cell.font = Font(bold=True, size=12, color="FFFFFF") + cell.alignment = Alignment(horizontal="left") + + @staticmethod + def _get_column_letter(number: int) -> str: + letters = "" + while number >= 0: + letters = chr(number % 26 + ord("A")) + letters + number = number // 26 - 1 + return letters + + @staticmethod + def _set_worksheet_column_style(worksheet: Worksheet, header_message: dict = None): + header_row_number = 2 if header_message and header_message.get("title") else 1 + min_width = 10 + + for column_cells in worksheet.columns: + max_column_length = 0 + for cell in column_cells: + if cell.row >= header_row_number: + cell_value = str(cell.value) if cell.value is not None else "" + max_column_length = max( + max_column_length, min_width, len(cell_value) + ) + column_letter = column_cells[0].column_letter + worksheet.column_dimensions[column_letter].width = max_column_length + 2 + + def _get_reference_resource_map( + self, cloudforet_mgr: CloudforetManager, fields: list, token: str + ) -> dict: + reference_resource_map = {} + + references_info = [] + for field in fields: + reference = field.get("reference", {}) or {} + resource_type = reference.get("resource_type") + + if ( + reference + and resource_type + and resource_type not in reference_resource_map + ): + references_info.append(reference) + + # get reference resources + for reference_info in references_info: + resource_type = reference_info.get("resource_type") + if resource_type in reference_resource_map: + continue + try: + res = self._get_reference_resources( + cloudforet_mgr, reference_info, token + ) + reference_resource_map[resource_type] = res + except Exception as e: + _LOGGER.error(f"Failed to get reference resources: {e}", exc_info=True) + return reference_resource_map + + @staticmethod + def _get_reference_resources( + cloudforet_mgr: CloudforetManager, reference_info: dict, token: str + ) -> list: + grpc_method = f"{reference_info.get('resource_type')}.list" + reference_key = reference_info.get("reference_key") + query_params = {"query": {"only": [reference_key, "name"]}} + results = [] + + responses = list(cloudforet_mgr.paginate_api(grpc_method, query_params, token)) + for res in responses: + results.extend(res.get("results", [])) + + return results + + @staticmethod + def _get_raw_data( + cloudforet_mgr: CloudforetManager, source: dict, token: str + ) -> list: + + url = source.get("url") + params = source.get("param", {}) + results = [] + + grpc_method: str = cloudforet_mgr.convert_grpc_method_from_url(url) + + responses = list(cloudforet_mgr.paginate_api(grpc_method, params, token)) + for res in responses: + results.extend(res.get("results", [])) + + return results + + @staticmethod + def _convert_value_with_reference_data( + row_value: Any, reference_key: str, reference_resources: list + ) -> Union[str, list]: + + if isinstance(row_value, list): + for idx, value in enumerate(row_value): + for reference_resource in reference_resources: + if reference_resource.get(reference_key) == value: + row_value[idx] = reference_resource.get("name") + break + else: + for reference_resource in reference_resources: + if reference_resource.get(reference_key) == row_value: + row_value = reference_resource.get("name") + break + + return row_value + + @staticmethod + def _get_file_name(excel_options: Union[dict, List[dict]]) -> str: + if isinstance(excel_options, list): + options = excel_options[0] + else: + options = excel_options + + timezone = options.get("template", {}).get("options", {}).get("timezone", "UTC") + prefix = ( + options.get("template", {}) + .get("options", {}) + .get("file_name_prefix", "export") + ) + + now = datetime.now(pytz.timezone(timezone)).strftime("%Y%m%d") + return f"{prefix}_export_{now}.xlsx" + + @staticmethod + def _get_row_value(row_value: Union[str, list, dict], key: str) -> Any: + if isinstance(row_value, dict): + print("row_value is dict") + row_value = utils.get_dict_value(data=row_value, dotted_key=key) + elif isinstance(row_value, list): + row_value = utils.get_list_values(row_value, key) + + return row_value + + def get_value_by_path( + self, data: Any, path: Optional[str], depth: Optional[int] = None + ) -> Any: + if not isinstance(path, str): + return data + + target = data + path_arr = path.split(".") + + last_depth_key = None + if depth is not None: + last_depth_key = ".".join(path_arr[depth:]) + path_arr = path_arr[:depth] + + path_count = depth if depth is not None else len(path_arr) + + for i in range(path_count): + if target is None: + return None + + current_path = path_arr[i] + + # 숫자인 경우 list index로 처리 + if isinstance(target, list): + try: + index = int(current_path) + target = target[index] + except (ValueError, IndexError): + return None + elif isinstance(target, dict): + target = target.get(current_path) + else: + return None + + if depth and last_depth_key: + return self.get_value_by_path(target, last_depth_key) + + return target + + @staticmethod + def _set_worksheet_row_style(worksheet: Worksheet, header_message: dict = None): + header_row_number = 2 if header_message and header_message.get("title") else 1 + + thin_border = Border( + top=Side(style="thin", color="E5E5E8"), + left=Side(style="thin", color="E5E5E8"), + bottom=Side(style="thin", color="E5E5E8"), + right=Side(style="thin", color="E5E5E8"), + ) + + striped_fill = PatternFill(fill_type="solid", fgColor="F7F7F7") + + for row in worksheet.iter_rows(): + row_number = row[0].row + for cell in row: + cell.border = thin_border + if row_number > header_row_number and row_number % 2 == 0: + for cell in row: + cell.fill = striped_fill