From 3b671f2dadb0bd539089fdd6b00a55689b48e8f8 Mon Sep 17 00:00:00 2001 From: teh_coderer Date: Mon, 17 Apr 2023 10:13:11 -0400 Subject: [PATCH 1/8] init --- openbb_terminal/base_helpers.py | 6 ++ openbb_terminal/core/models/profile_model.py | 2 + openbb_terminal/loggers.py | 105 ++++++++++++++++++- openbb_terminal/parent_classes.py | 7 +- 4 files changed, 113 insertions(+), 7 deletions(-) diff --git a/openbb_terminal/base_helpers.py b/openbb_terminal/base_helpers.py index eb8317a2c69d..0b8c886a9e4e 100644 --- a/openbb_terminal/base_helpers.py +++ b/openbb_terminal/base_helpers.py @@ -4,6 +4,7 @@ from typing import Any, List, Literal, Optional from dotenv import load_dotenv +from posthog import Posthog from rich.console import Console from openbb_terminal.core.config.paths import ( @@ -16,6 +17,11 @@ menus = Literal["", "featflags", "settings"] +openbb_posthog = Posthog( + os.environ.get("POSTHOG_API_KEY", ""), + host="https://app.posthog.com", +) + def handle_error(name: str, default: Any, menu: menus = ""): """Handles the error by returning the default value and printing an diff --git a/openbb_terminal/core/models/profile_model.py b/openbb_terminal/core/models/profile_model.py index 4fb75f1b5883..b1611f6e05dc 100644 --- a/openbb_terminal/core/models/profile_model.py +++ b/openbb_terminal/core/models/profile_model.py @@ -2,6 +2,7 @@ from pydantic.dataclasses import dataclass +from openbb_terminal.base_helpers import openbb_posthog from openbb_terminal.core.models import BaseModel @@ -30,6 +31,7 @@ def load_user_info(self, session: dict, email: str): self.uuid = session.get("uuid", "") self.email = email self.username = self.email[: self.email.find("@")] + openbb_posthog.identify(self.uuid, {"email": self.email}) def get_uuid(self) -> str: """Get uuid. diff --git a/openbb_terminal/loggers.py b/openbb_terminal/loggers.py index c2cac6534a4a..969641f422ea 100644 --- a/openbb_terminal/loggers.py +++ b/openbb_terminal/loggers.py @@ -2,12 +2,15 @@ __docformat__ = "numpy" # IMPORTATION STANDARD +import json import logging +import re import sys import time import uuid from pathlib import Path -from typing import Optional +from platform import platform, python_version +from typing import Any, Dict, Optional # IMPORTATION THIRDPARTY try: @@ -18,6 +21,7 @@ WITH_GIT = True # IMPORTATION INTERNAL +from openbb_terminal.base_helpers import openbb_posthog from openbb_terminal.core.log.generation.directories import get_log_dir from openbb_terminal.core.log.generation.formatter_with_exceptions import ( FormatterWithExceptions, @@ -31,8 +35,13 @@ LogSettings, Settings, ) -from openbb_terminal.core.log.generation.user_logger import get_user_uuid -from openbb_terminal.core.session.current_system import get_current_system +from openbb_terminal.core.log.generation.user_logger import ( + NO_USER_PLACEHOLDER, + get_user_uuid, +) +from openbb_terminal.core.session.current_system import ( + get_current_system, +) logger = logging.getLogger(__name__) current_system = get_current_system() @@ -85,6 +94,95 @@ def get_commit_hash(use_env=True) -> str: return commit_hash +class PosthogHandler(logging.Handler): + """Posthog Handler""" + + def __init__(self, settings: Settings): + super().__init__() + self.settings = settings + self.app_settings = settings.app_settings + self.logged_in = False + + def emit(self, record: logging.LogRecord): + try: + self.send(record=record) + except Exception: + self.handleError(record) + + def log_to_dict(self, log_info: str) -> dict: + """Log to dict""" + log_regex = r"(KEYS|PREFERENCES|SYSTEM|CMD|QUEUE): (.*)" + log_dict = {} + + for log in re.findall(log_regex, log_info): + log_dict[log[0]] = json.loads(log[1]) + + return log_dict + + def send(self, record: logging.LogRecord): + """Send log record to Posthog""" + + app_settings = self.app_settings + + level_name = logging.getLevelName(record.levelno) + log_line = FormatterWithExceptions.filter_log_line(text=record.getMessage()) + + log_extra = self.extract_log_extra(record=record) + log_extra.update(dict(level=level_name, message=log_line)) + event_name = f"log_{level_name.lower()}" + + log_dict = self.log_to_dict(log_info=log_line) + if log_dict: + key = list(log_dict.keys())[0] + event_name = f"log_{key.lower()}" + log_extra = {**log_extra, **log_dict[key]} + log_extra.pop("message", None) + + ignore_regex = r"^(START|END|INPUT:)" + if re.match(ignore_regex, log_line): + return + + if not self.logged_in and get_user_uuid() != NO_USER_PLACEHOLDER: + self.logged_in = True + openbb_posthog.alias(app_settings.identifier, get_user_uuid()) + + log_extra.update({"$geoip_disable": True}) + openbb_posthog.capture( + app_settings.identifier, + event_name, + properties=log_extra, + ) + + def extract_log_extra(self, record: logging.LogRecord) -> Dict[str, Any]: + """Extract log extra from record""" + + log_extra = { + "appName": self.app_settings.name, + "appId": self.app_settings.identifier, + "sessionId": self.app_settings.session_id, + "commitHash": self.app_settings.commit_hash, + "platform": platform(), + "pythonVersion": python_version(), + "terminalVersion": current_system.VERSION, + } + + if get_user_uuid() != NO_USER_PLACEHOLDER: + log_extra["userId"] = get_user_uuid() + + if hasattr(record, "extra"): + log_extra = {**log_extra, **record.extra} + + return log_extra + + +def add_posthog_handler(settings: Settings): + app_settings = settings.app_settings + handler = PosthogHandler(settings=settings) + formatter = FormatterWithExceptions(app_settings=app_settings) + handler.setFormatter(formatter) + logging.getLogger().addHandler(handler) + + def add_stdout_handler(settings: Settings): app_settings = settings.app_settings handler = logging.StreamHandler(sys.stdout) @@ -149,6 +247,7 @@ def setup_handlers(settings: Settings): FormatterWithExceptions.LOGPREFIXFORMAT.replace("|", "-"), FormatterWithExceptions.LOGFORMAT.replace("|", "-"), ) + add_posthog_handler(settings=settings) def setup_logging( diff --git a/openbb_terminal/parent_classes.py b/openbb_terminal/parent_classes.py index 26c833791dc9..fdeea426412c 100644 --- a/openbb_terminal/parent_classes.py +++ b/openbb_terminal/parent_classes.py @@ -299,10 +299,9 @@ def log_queue(self) -> None: if self.queue: joined_queue = self.COMMAND_SEPARATOR.join(self.queue) if not self.contains_keys(joined_queue): + queue = {"path": self.PATH, "queue": joined_queue} logger.info( - "QUEUE: {'path': '%s', 'queue': '%s'}", - self.PATH, - joined_queue, + "QUEUE: %s ", json.dumps(queue, default=str, ensure_ascii=False) ) def log_cmd_and_queue( @@ -326,7 +325,7 @@ def log_cmd_and_queue( "other_args": other_args_str, "input": the_input, } - logger.info("CMD: %s", json.dumps(cmd)) + logger.info("CMD: %s ", json.dumps(cmd)) if the_input not in self.KEYS_MENU: self.log_queue() From 6bc9a5a8fee7fac2568ebee5edf07da10cc589c8 Mon Sep 17 00:00:00 2001 From: teh_coderer Date: Mon, 17 Apr 2023 12:34:12 -0400 Subject: [PATCH 2/8] updates --- openbb_terminal/base_helpers.py | 2 +- openbb_terminal/core/models/system_model.py | 1 + openbb_terminal/core/plots/backend.py | 5 ++++- openbb_terminal/core/plots/plotly.html | 15 +++++++++++++++ openbb_terminal/loggers.py | 5 ++++- 5 files changed, 25 insertions(+), 3 deletions(-) diff --git a/openbb_terminal/base_helpers.py b/openbb_terminal/base_helpers.py index 0b8c886a9e4e..e7d18a61b16d 100644 --- a/openbb_terminal/base_helpers.py +++ b/openbb_terminal/base_helpers.py @@ -18,7 +18,7 @@ menus = Literal["", "featflags", "settings"] openbb_posthog = Posthog( - os.environ.get("POSTHOG_API_KEY", ""), + "phc_8fP8xXi0ptWTlGAXOcMQnSFsul4lmLoxnwp9EiXQstO", host="https://app.posthog.com", ) diff --git a/openbb_terminal/core/models/system_model.py b/openbb_terminal/core/models/system_model.py index d8cd6220773c..829229f16396 100644 --- a/openbb_terminal/core/models/system_model.py +++ b/openbb_terminal/core/models/system_model.py @@ -29,6 +29,7 @@ class SystemModel(BaseModel): VERSION = "3.0.0rc2" # Logging section + LOGGING_APP_ID: str = "REPLACE_ME" LOGGING_APP_NAME: str = "gst" LOGGING_AWS_ACCESS_KEY_ID: str = "REPLACE_ME" LOGGING_AWS_SECRET_ACCESS_KEY: str = "REPLACE_ME" diff --git a/openbb_terminal/core/plots/backend.py b/openbb_terminal/core/plots/backend.py index 9ba38ef914dd..e56d710b4b2c 100644 --- a/openbb_terminal/core/plots/backend.py +++ b/openbb_terminal/core/plots/backend.py @@ -168,11 +168,14 @@ def send_figure( if export_image and isinstance(export_image, str): export_image = Path(export_image).resolve() + data = json.loads(fig.to_json()) + data.update({"user_id": get_current_system().LOGGING_APP_ID}) + self.outgoing.append( json.dumps( { "html_path": self.get_plotly_html(), - "json_data": json.loads(fig.to_json()), + "json_data": data, "export_image": str(export_image), **self.get_kwargs(command_location), } diff --git a/openbb_terminal/core/plots/plotly.html b/openbb_terminal/core/plots/plotly.html index d4623858add0..e42649bfb0cf 100644 --- a/openbb_terminal/core/plots/plotly.html +++ b/openbb_terminal/core/plots/plotly.html @@ -18,6 +18,11 @@ crossorigin="anonymous" referrerpolicy="no-referrer" > + @@ -202,6 +207,8 @@ globals.cmd_src = ""; if (typeof OpenBBMain != "undefined" && window.plotly_figure) { clearInterval(interval); + let user_id = window.plotly_figure.user_id; + posthog.identify(user_id); const date = new Date(); const formattedDate = new Intl.DateTimeFormat("en-GB", { dateStyle: "full", @@ -245,6 +252,14 @@ }); } + if (globals.cmd_src != "") { + posthog.capture("chart", { + command: globals.cmd_src, + title: window.plotly_figure.layout.title.text, + date: formattedDate, + }); + } + document .querySelector('meta[name="color-scheme"]') .setAttribute( diff --git a/openbb_terminal/loggers.py b/openbb_terminal/loggers.py index 969641f422ea..49b5a121543e 100644 --- a/openbb_terminal/loggers.py +++ b/openbb_terminal/loggers.py @@ -41,6 +41,7 @@ ) from openbb_terminal.core.session.current_system import ( get_current_system, + set_current_system, ) logger = logging.getLogger(__name__) @@ -102,6 +103,8 @@ def __init__(self, settings: Settings): self.settings = settings self.app_settings = settings.app_settings self.logged_in = False + current_system.LOGGING_APP_ID = get_app_id() + set_current_system(current_system) def emit(self, record: logging.LogRecord): try: @@ -144,7 +147,7 @@ def send(self, record: logging.LogRecord): if not self.logged_in and get_user_uuid() != NO_USER_PLACEHOLDER: self.logged_in = True - openbb_posthog.alias(app_settings.identifier, get_user_uuid()) + openbb_posthog.alias(get_user_uuid(), app_settings.identifier) log_extra.update({"$geoip_disable": True}) openbb_posthog.capture( From e86c583753309991daa64c2d51118045df0b0c71 Mon Sep 17 00:00:00 2001 From: teh_coderer Date: Tue, 18 Apr 2023 09:34:34 -0400 Subject: [PATCH 3/8] Update loggers.py --- openbb_terminal/loggers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openbb_terminal/loggers.py b/openbb_terminal/loggers.py index 49b5a121543e..97e03cb43456 100644 --- a/openbb_terminal/loggers.py +++ b/openbb_terminal/loggers.py @@ -2,6 +2,7 @@ __docformat__ = "numpy" # IMPORTATION STANDARD +import atexit import json import logging import re @@ -105,6 +106,7 @@ def __init__(self, settings: Settings): self.logged_in = False current_system.LOGGING_APP_ID = get_app_id() set_current_system(current_system) + atexit.register(openbb_posthog.shutdown) def emit(self, record: logging.LogRecord): try: From 66e3e4b1fb46d6f3eaa17131e1a49c3534d26ca8 Mon Sep 17 00:00:00 2001 From: teh_coderer Date: Tue, 18 Apr 2023 09:58:30 -0400 Subject: [PATCH 4/8] Update loggers.py --- openbb_terminal/loggers.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/openbb_terminal/loggers.py b/openbb_terminal/loggers.py index 97e03cb43456..656f61d572b3 100644 --- a/openbb_terminal/loggers.py +++ b/openbb_terminal/loggers.py @@ -104,8 +104,6 @@ def __init__(self, settings: Settings): self.settings = settings self.app_settings = settings.app_settings self.logged_in = False - current_system.LOGGING_APP_ID = get_app_id() - set_current_system(current_system) atexit.register(openbb_posthog.shutdown) def emit(self, record: logging.LogRecord): @@ -252,7 +250,9 @@ def setup_handlers(settings: Settings): FormatterWithExceptions.LOGPREFIXFORMAT.replace("|", "-"), FormatterWithExceptions.LOGFORMAT.replace("|", "-"), ) - add_posthog_handler(settings=settings) + + if not current_system.TEST_MODE and not current_system.LOGGING_SUPPRESS: + add_posthog_handler(settings=settings) def setup_logging( @@ -269,6 +269,9 @@ def setup_logging( session_id = get_session_id() user_id = get_user_uuid() + current_system.LOGGING_APP_ID = identifier + set_current_system(current_system) + # AWSSettings aws_access_key_id = current_system.LOGGING_AWS_ACCESS_KEY_ID aws_secret_access_key = current_system.LOGGING_AWS_SECRET_ACCESS_KEY From 98857074082316ddb1add070d137fbae78c7541f Mon Sep 17 00:00:00 2001 From: teh_coderer Date: Wed, 19 Apr 2023 13:41:54 -0400 Subject: [PATCH 5/8] sdk handling --- openbb_terminal/core/plots/web/css/style.css | 4 ++-- openbb_terminal/core/plots/web/popups.js | 5 ----- openbb_terminal/loggers.py | 8 ++++++++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/openbb_terminal/core/plots/web/css/style.css b/openbb_terminal/core/plots/web/css/style.css index 948fa5a7b74d..ad5e6454b368 100644 --- a/openbb_terminal/core/plots/web/css/style.css +++ b/openbb_terminal/core/plots/web/css/style.css @@ -79,8 +79,8 @@ body { position: absolute; z-index: 9; text-align: center; - top: 0; - right: 50; + top: 50%; + left: 50%; width: 100px; height: 60px; background-color: transparent; diff --git a/openbb_terminal/core/plots/web/popups.js b/openbb_terminal/core/plots/web/popups.js index 5e1ab8ac0b56..d075168f7175 100644 --- a/openbb_terminal/core/plots/web/popups.js +++ b/openbb_terminal/core/plots/web/popups.js @@ -459,11 +459,6 @@ function on_submit(popup_id, on_annotation = null) { // We get main plotly trace to get the x/y axis let main_trace = gd.data[0]; - gd.data.forEach((trace) => { - if (globals.added_traces.indexOf(trace.name) == -1) { - trace.showlegend = false; - } - }); // We check how many yaxis have ticklabels on the left let left_yaxis_ticks = Object.keys(gd.layout) diff --git a/openbb_terminal/loggers.py b/openbb_terminal/loggers.py index 656f61d572b3..b4767fc92394 100644 --- a/openbb_terminal/loggers.py +++ b/openbb_terminal/loggers.py @@ -120,6 +120,11 @@ def log_to_dict(self, log_info: str) -> dict: for log in re.findall(log_regex, log_info): log_dict[log[0]] = json.loads(log[1]) + sdk_regex = r"({\"INPUT\":.*})" + sdk_dict = re.findall(sdk_regex, log_info) + if sdk_dict: + log_dict["SDK"] = json.loads(sdk_dict[0]) + return log_dict def send(self, record: logging.LogRecord): @@ -138,6 +143,9 @@ def send(self, record: logging.LogRecord): if log_dict: key = list(log_dict.keys())[0] event_name = f"log_{key.lower()}" + if key == "SDK": + log_extra.update(log_dict[key].pop("INPUT", {})) + log_extra = {**log_extra, **log_dict[key]} log_extra.pop("message", None) From dabf0f04521dd72764dacd64e9de89d6d382a686 Mon Sep 17 00:00:00 2001 From: teh_coderer Date: Thu, 20 Apr 2023 11:30:35 -0400 Subject: [PATCH 6/8] feature flag --- openbb_terminal/core/plots/plotly.html | 5 +++++ openbb_terminal/loggers.py | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/openbb_terminal/core/plots/plotly.html b/openbb_terminal/core/plots/plotly.html index e42649bfb0cf..8929d91267f5 100644 --- a/openbb_terminal/core/plots/plotly.html +++ b/openbb_terminal/core/plots/plotly.html @@ -21,6 +21,11 @@ diff --git a/openbb_terminal/loggers.py b/openbb_terminal/loggers.py index b4767fc92394..ed83f875b405 100644 --- a/openbb_terminal/loggers.py +++ b/openbb_terminal/loggers.py @@ -259,7 +259,10 @@ def setup_handlers(settings: Settings): FormatterWithExceptions.LOGFORMAT.replace("|", "-"), ) - if not current_system.TEST_MODE and not current_system.LOGGING_SUPPRESS: + if ( + not any([current_system.TEST_MODE, current_system.LOGGING_SUPPRESS]) + # and current_system.LOG_COLLECT + ): add_posthog_handler(settings=settings) From 42902fcf0b1e897a61e42b67bba94666704db0f3 Mon Sep 17 00:00:00 2001 From: teh_coderer Date: Thu, 20 Apr 2023 15:43:27 -0400 Subject: [PATCH 7/8] fix MOCKING_USER_PATH for windows --- .../generation/formatter_with_exceptions.py | 6 +++-- openbb_terminal/loggers.py | 23 +++++++++++-------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/openbb_terminal/core/log/generation/formatter_with_exceptions.py b/openbb_terminal/core/log/generation/formatter_with_exceptions.py index 1b569af582f4..a6a61a0b0773 100644 --- a/openbb_terminal/core/log/generation/formatter_with_exceptions.py +++ b/openbb_terminal/core/log/generation/formatter_with_exceptions.py @@ -67,8 +67,10 @@ def mock_password(text: str) -> str: @staticmethod def mock_home_directory(text: str) -> str: - user_home_directory = str(HOME_DIRECTORY) - text_mocked = text.replace(user_home_directory, "MOCKING_USER_PATH") + user_home_directory = str(HOME_DIRECTORY.as_posix()) + text_mocked = text.replace("\\\\", "/").replace( + user_home_directory, "MOCKING_USER_PATH" + ) return text_mocked diff --git a/openbb_terminal/loggers.py b/openbb_terminal/loggers.py index ed83f875b405..7639fd835ecf 100644 --- a/openbb_terminal/loggers.py +++ b/openbb_terminal/loggers.py @@ -139,18 +139,13 @@ def send(self, record: logging.LogRecord): log_extra.update(dict(level=level_name, message=log_line)) event_name = f"log_{level_name.lower()}" - log_dict = self.log_to_dict(log_info=log_line) - if log_dict: - key = list(log_dict.keys())[0] - event_name = f"log_{key.lower()}" - if key == "SDK": - log_extra.update(log_dict[key].pop("INPUT", {})) - - log_extra = {**log_extra, **log_dict[key]} + if log_dict := self.log_to_dict(log_info=log_line): + event_name = f"log_{list(log_dict.keys())[0].lower()}" + + log_extra = {**log_extra, **log_dict} log_extra.pop("message", None) - ignore_regex = r"^(START|END|INPUT:)" - if re.match(ignore_regex, log_line): + if re.match(r"^(START|END|INPUT:)", log_line): return if not self.logged_in and get_user_uuid() != NO_USER_PLACEHOLDER: @@ -158,6 +153,7 @@ def send(self, record: logging.LogRecord): openbb_posthog.alias(get_user_uuid(), app_settings.identifier) log_extra.update({"$geoip_disable": True}) + openbb_posthog.capture( app_settings.identifier, event_name, @@ -183,6 +179,13 @@ def extract_log_extra(self, record: logging.LogRecord) -> Dict[str, Any]: if hasattr(record, "extra"): log_extra = {**log_extra, **record.extra} + if record.exc_info: + log_extra["exception"] = { + "type": str(record.exc_info[0]), + "value": str(record.exc_info[1]), + "traceback": self.format(record), + } + return log_extra From 87bc199971fc4794caa3bd5b6f4b96630394c13a Mon Sep 17 00:00:00 2001 From: teh_coderer Date: Thu, 20 Apr 2023 15:52:51 -0400 Subject: [PATCH 8/8] Update loggers.py --- openbb_terminal/loggers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_terminal/loggers.py b/openbb_terminal/loggers.py index 7639fd835ecf..ad12a4658964 100644 --- a/openbb_terminal/loggers.py +++ b/openbb_terminal/loggers.py @@ -264,7 +264,7 @@ def setup_handlers(settings: Settings): if ( not any([current_system.TEST_MODE, current_system.LOGGING_SUPPRESS]) - # and current_system.LOG_COLLECT + and current_system.LOG_COLLECT ): add_posthog_handler(settings=settings)