Merge pull request #956 from skalenetwork/update-settings-structure

New settings structure, migrate to toml configs

Author: dmytrotkk

.github/copilot-instructions.md

@@ -11,7 +11,10 @@
 - no redundant code - move repeated logic into helper functions
 - use type hints to specify the expected types of function arguments and return values
 
-- check `ruff.toml` for formatting rules
-- always lint changes using `ruff check`
+- check `pyproject.toml` for formatting rules
+- always lint changes using `uv run ruff check`
 - tests should be placed in `tests/` directory, follow the existing structure and code style
-- to run a test always use `bash scripts/run_tests.sh tests/path_to_test.py -k [TEST_NAME]` command
+- always use `uv` to run all commands in the repo (e.g., `uv run ruff`, `uv run pytest`, etc.)
+- for running tests, export environment variables in the terminal before running the tests: `. ./scripts/export_env.sh`
+
+- additional external context is located in context directory

.gitignore

@@ -125,3 +125,4 @@ tests/.skale/node_data/node_options.json
 tests/.skale/config/nginx.conf.j2
 
 .zed
+uv.lock

node_cli/cli/fair_boot.py

@@ -88,6 +88,6 @@ def signature_boot(validator_id):
 @streamed_cmd
 def update_node(env_file, pull_config_for_schain):
     update(
-        env_filepath=env_file,
+        config_file=env_file,
         pull_config_for_schain=pull_config_for_schain,
     )

node_cli/cli/fair_node.py

@@ -57,10 +57,10 @@ def fair_node_info(format):
 
 
 @node.command('init', help='Initialize regular Fair node')
-@click.argument('env_filepath')
+@click.argument('config_file')
 @streamed_cmd
-def init_node(env_filepath: str):
-    init_fair(node_mode=NodeMode.ACTIVE, env_filepath=env_filepath)
+def init_node(config_file: str):
+    init_fair(node_mode=NodeMode.ACTIVE, config_file=config_file)
 
 
 @node.command('register', help=TEXTS['fair']['node']['register']['help'])
@@ -70,7 +70,7 @@ def register(ip: str) -> None:
 
 
 @node.command('update', help='Update Fair node')
-@click.argument('env_filepath')
+@click.argument('config_file')
 @click.option(
     '--yes',
     is_flag=True,
@@ -88,10 +88,10 @@ def register(ip: str) -> None:
     is_flag=True,
 )
 @streamed_cmd
-def update_node(env_filepath: str, pull_config_for_schain, force_skaled_start: bool):
+def update_node(config_file: str, pull_config_for_schain, force_skaled_start: bool):
     update_fair(
         node_mode=NodeMode.ACTIVE,
-        env_filepath=env_filepath,
+        config_file=config_file,
         pull_config_for_schain=pull_config_for_schain,
         force_skaled_start=force_skaled_start,
     )
@@ -106,20 +106,20 @@ def backup_node(backup_folder_path):
 
 @node.command('restore', help='Restore Fair node from a backup file.')
 @click.argument('backup_path')
-@click.argument('env_file')
+@click.argument('config_file')
 @click.option(
     '--config-only',
     help='Only restore configuration files in .skale and artifacts',
     is_flag=True,
     hidden=True,
 )
 @streamed_cmd
-def restore_node(backup_path, env_file, config_only):
-    restore_fair(backup_path, env_file, config_only)
+def restore_node(backup_path, config_file, config_only):
+    restore_fair(backup_path, config_file, config_only)
 
 
 @node.command('migrate', help='Switch from boot to regular Fair node.')
-@click.argument('env_filepath')
+@click.argument('config_file')
 @click.option(
     '--yes',
     is_flag=True,
@@ -128,8 +128,8 @@ def restore_node(backup_path, env_file, config_only):
     prompt='Are you sure you want to migrate to regular Fair node? The action cannot be undone',
 )
 @streamed_cmd
-def migrate_node(env_filepath: str) -> None:
-    migrate_from_boot(env_filepath=env_filepath)
+def migrate_node(config_file: str) -> None:
+    migrate_from_boot(config_file=config_file)
 
 
 @node.command('repair', help='Toggle fair chain repair mode')
@@ -221,7 +221,7 @@ def turn_off_node() -> None:
     expose_value=False,
     prompt='Are you sure you want to turn on the node?',
 )
-@click.argument('env_filepath')
+@click.argument('config_file')
 @streamed_cmd
-def turn_on_node(env_filepath: str) -> None:
-    turn_on_fair(env_file=env_filepath, node_type=TYPE)
+def turn_on_node(config_file: str) -> None:
+    turn_on_fair(env_file=config_file, node_type=TYPE)

node_cli/cli/node.py

@@ -17,8 +17,11 @@
 #   You should have received a copy of the GNU Affero General Public License
 #   along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+from typing import get_args
+
 import click
 
+from skale_core.types import EnvType
 from node_cli.cli.info import TYPE
 from node_cli.core.node import (
     cleanup as cleanup_skale,
@@ -38,7 +41,6 @@
     run_checks,
 )
 from node_cli.configs import DEFAULT_NODE_BASE_PORT
-from node_cli.configs.user import ALLOWED_ENV_TYPES
 from node_cli.core.node_options import upsert_node_mode
 from node_cli.utils.decorators import check_inited
 from node_cli.utils.helper import abort_if_false, streamed_cmd, IP_TYPE
@@ -85,10 +87,10 @@ def register_node(name, ip, port, domain):
 
 
 @node.command('init', help='Initialize SKALE node')
-@click.argument('env_file')
+@click.argument('config_file')
 @streamed_cmd
-def init_node(env_file):
-    init(env_filepath=env_file, node_type=TYPE)
+def init_node(config_file):
+    init(config_file=config_file, node_type=TYPE)
 
 
 @node.command('update', help='Update node from .env file')
@@ -101,12 +103,12 @@ def init_node(env_file):
 )
 @click.option('--pull-config', 'pull_config_for_schain', hidden=True, type=str)
 @click.option('--unsafe', 'unsafe_ok', help='Allow unsafe update', hidden=True, is_flag=True)
-@click.argument('env_file')
+@click.argument('config_file')
 @streamed_cmd
-def update_node(env_file, pull_config_for_schain, unsafe_ok):
+def update_node(config_file, pull_config_for_schain, unsafe_ok):
     update(
         node_mode=NodeMode.ACTIVE,
-        env_filepath=env_file,
+        config_file=config_file,
         pull_config_for_schain=pull_config_for_schain,
         node_type=TYPE,
         unsafe_ok=unsafe_ok,
@@ -143,7 +145,7 @@ def backup_node(backup_folder_path):
 def restore_node(backup_path, env_file, no_snapshot, config_only):
     restore(
         backup_path=backup_path,
-        env_filepath=env_file,
+        config_file=env_file,
         no_snapshot=no_snapshot,
         config_only=config_only,
         node_type=TYPE,
@@ -227,7 +229,7 @@ def _set_domain_name(domain):
 @click.option(
     '--network',
     '-n',
-    type=click.Choice(ALLOWED_ENV_TYPES),
+    type=click.Choice(get_args(EnvType)),
     default='mainnet',
     help='Network to check',
 )

node_cli/cli/passive_fair_node.py

@@ -49,7 +49,7 @@ def passive_node():
 
 
 @passive_node.command('init', help='Initialize a passive Fair node')
-@click.argument('env_filepath')
+@click.argument('config_file')
 @click.option('--id', required=True, type=int, help=TEXTS['fair']['node']['setup']['id'])
 @click.option('--indexer', help=TEXTS['passive_node']['init']['indexer'], is_flag=True)
 @click.option('--archive', help=TEXTS['passive_node']['init']['archive'], is_flag=True)
@@ -61,15 +61,15 @@ def passive_node():
 )
 @streamed_cmd
 def init_passive_node(
-    env_filepath: str, id: int, indexer: bool, archive: bool, snapshot: str | None
+    config_file: str, id: int, indexer: bool, archive: bool, snapshot: str | None
 ):
     if indexer and archive:
         error_exit('Cannot use both --indexer and --archive options')
     if (indexer or archive) and snapshot == 'any':
         error_exit('Cannot use any for indexer/archive node')
     init_fair(
         node_mode=NodeMode.PASSIVE,
-        env_filepath=env_filepath,
+        config_file=config_file,
         node_id=id,
         indexer=indexer,
         archive=archive,
@@ -78,7 +78,7 @@ def init_passive_node(
 
 
 @passive_node.command('update', help='Update Fair node')
-@click.argument('env_filepath')
+@click.argument('config_file')
 @click.option(
     '--yes',
     is_flag=True,
@@ -96,10 +96,10 @@ def init_passive_node(
     is_flag=True,
 )
 @streamed_cmd
-def update_node(env_filepath: str, pull_config_for_schain, force_skaled_start: bool):
+def update_node(config_file: str, pull_config_for_schain, force_skaled_start: bool):
     update_fair(
         node_mode=NodeMode.PASSIVE,
-        env_filepath=env_filepath,
+        config_file=config_file,
         pull_config_for_schain=pull_config_for_schain,
         force_skaled_start=force_skaled_start,
     )
@@ -146,7 +146,7 @@ def turn_off_node() -> None:
     expose_value=False,
     prompt='Are you sure you want to turn on the node?',
 )
-@click.argument('env_filepath')
+@click.argument('config_file')
 @streamed_cmd
-def turn_on_node(env_filepath: str) -> None:
-    turn_on_fair(env_file=env_filepath, node_type=TYPE)
+def turn_on_node(config_file: str) -> None:
+    turn_on_fair(env_file=config_file, node_type=TYPE)

node_cli/cli/schains.py

@@ -21,6 +21,8 @@
 
 import click
 
+from skale_core.settings import get_settings
+
 from node_cli.utils.helper import abort_if_false, URL_TYPE
 from node_cli.core.schains import (
     describe,
@@ -104,8 +106,12 @@ def info_(schain_name: str, json_format: bool) -> None:
 @click.argument('schain_name')
 @click.argument('snapshot_path')
 @click.option('--schain-type', default='medium')
-@click.option('--env-type', default=None)
-def restore(
-    schain_name: str, snapshot_path: str, schain_type: str, env_type: Optional[str]
-) -> None:
-    restore_schain_from_snapshot(schain_name, snapshot_path, node_type=TYPE)
+def restore(schain_name: str, snapshot_path: str, schain_type: str) -> None:
+    settings = get_settings()
+    restore_schain_from_snapshot(
+        schain_name,
+        snapshot_path,
+        node_type=TYPE,
+        env_type=settings.env_type,
+        schain_type=schain_type,
+    )

node_cli/configs/__init__.py

@@ -19,6 +19,7 @@
 
 import os
 import sys
+from pathlib import Path
 
 from node_cli.utils.global_config import read_g_config
 
@@ -43,17 +44,19 @@
 NODE_DATA_PATH = os.path.join(SKALE_DIR, 'node_data')
 SCHAIN_NODE_DATA_PATH = os.path.join(NODE_DATA_PATH, 'schains')
 NODE_CLI_STATUS_FILENAME = 'node_cli.status'
+
+SETTINGS_DIR = Path(NODE_DATA_PATH) / 'settings'
+NODE_SETTINGS_PATH = SETTINGS_DIR / 'node.toml'
+INTERNAL_SETTINGS_PATH = SETTINGS_DIR / 'internal.toml'
+
 NODE_CONFIG_PATH = os.path.join(NODE_DATA_PATH, 'node_config.json')
 CONTAINER_CONFIG_PATH = os.path.join(SKALE_DIR, 'config')
 CONTAINER_CONFIG_TMP_PATH = os.path.join(SKALE_TMP_DIR, 'config')
 CONTRACTS_PATH = os.path.join(SKALE_DIR, 'contracts_info')
 REPORTS_PATH = os.path.join(SKALE_DIR, 'reports')
 BACKUP_CONTRACTS_PATH = os.path.join(SKALE_DIR, '.old_contracts_info')
-INIT_ENV_FILEPATH = os.path.join(SKALE_DIR, '.env')
 SKALE_RUN_DIR = '/var/run/skale'
 
-SGX_CERTIFICATES_DIR_NAME = 'sgx_certs'
-
 COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose.yml')
 FAIR_COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose-fair.yml')
 STATIC_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'static_params.yaml')
@@ -73,9 +76,6 @@
 SGX_CERTS_PATH = os.path.join(NODE_DATA_PATH, 'sgx_certs')
 SCHAINS_DATA_PATH = os.path.join(NODE_DATA_PATH, 'schains')
 
-CURRENT_FILE_LOCATION = os.path.dirname(os.path.realpath(__file__))
-DOTENV_FILEPATH = os.path.join(os.path.dirname(CURRENT_FILE_LOCATION), '.env')
-
 SRC_FILEBEAT_CONFIG_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'filebeat.yml')
 FILEBEAT_CONFIG_PATH = os.path.join(NODE_DATA_PATH, 'filebeat.yml')
 
@@ -95,9 +95,6 @@
 IPTABLES_RULES_STATE_FILEPATH = os.path.join(IPTABLES_DIR, 'rules.v4')
 DEFAULT_SSH_PORT = 22
 
-FLASK_SECRET_KEY_FILENAME = 'flask_db_key.txt'
-FLASK_SECRET_KEY_FILE = os.path.join(NODE_DATA_PATH, FLASK_SECRET_KEY_FILENAME)
-
 DOCKER_CONFIG_FILEPATH = '/etc/docker/daemon.json'
 HIDE_STREAM_LOG = os.getenv('HIDE_STREAM_LOG')