diff --git a/devservices/commands/up.py b/devservices/commands/up.py index 8143ea9..7807941 100644 --- a/devservices/commands/up.py +++ b/devservices/commands/up.py @@ -162,7 +162,10 @@ def up(args: Namespace, existing_status: Status | None = None) -> None: }, ) - remote_dependencies = _install_service_dependencies(service, mode, status) + offline = getattr(args, "offline", False) + remote_dependencies = _install_service_dependencies( + service, mode, status, offline=offline + ) _create_devservices_network() # Add the service to the starting services table state.update_service_entry(service.name, mode, StateTables.STARTING_SERVICES) @@ -204,6 +207,7 @@ def up(args: Namespace, existing_status: Status | None = None) -> None: service_name=local_runtime_dependency_name, mode="default", # We intentionally don't use the mode from the parent command here exclude_local=True, # TODO: This should be False (or maybe whatever the parent command is set to) + offline=offline, ), status, ) @@ -216,7 +220,12 @@ def up(args: Namespace, existing_status: Status | None = None) -> None: span.set_data("exclude_local", exclude_local) try: bring_up_docker_compose_services( - service, [mode], remote_dependencies, mode_dependencies, status + service, + [mode], + remote_dependencies, + mode_dependencies, + status, + skip_pull=offline, ) except DockerComposeError as dce: capture_exception(dce, level="info") @@ -239,25 +248,32 @@ def up(args: Namespace, existing_status: Status | None = None) -> None: def _install_service_dependencies( - service: Service, mode: str, status: Status + service: Service, mode: str, status: Status, offline: bool = False ) -> set[InstalledRemoteDependency]: with start_span( op="service.dependencies.install", name="Install dependencies" ) as span: - status.info("Retrieving dependencies") + status.info( + "Using cached dependencies" if offline else "Retrieving dependencies" + ) span.set_data("service_name", service.name) span.set_data("mode", mode) + span.set_data("offline", offline) try: remote_dependencies = install_and_verify_dependencies( - service, force_update_dependencies=True, modes=[mode] + service, + force_update_dependencies=not offline, + modes=[mode], + offline=offline, ) span.set_data("remote_dependency_count", len(remote_dependencies)) return remote_dependencies except DependencyError as de: capture_exception(de) - status.failure( - f"{str(de)}. If this error persists, try running `devservices purge`" - ) + msg = de.stderr if de.stderr else str(de) + if not offline: + msg = f"{msg}. If this error persists, try running `devservices purge`" + status.failure(msg) exit(1) except ModeDoesNotExistError as mde: status.failure(str(mde)) @@ -310,6 +326,7 @@ def bring_up_docker_compose_services( remote_dependencies: set[InstalledRemoteDependency], mode_dependencies: list[str], status: Status, + skip_pull: bool = False, ) -> None: relative_local_dependency_directory = os.path.relpath( os.path.join(DEVSERVICES_DEPENDENCIES_CACHE_DIR, DEPENDENCY_CONFIG_VERSION), @@ -334,27 +351,29 @@ def bring_up_docker_compose_services( ), ) - # Pull all images in parallel - status.info("Pulling images") - pull_commands = get_docker_compose_commands_to_run( - service=service, - remote_dependencies=sorted_remote_dependencies, - current_env=current_env, - command="pull", - options=[], - service_config_file_path=service_config_file_path, - mode_dependencies=mode_dependencies, - ) + if not skip_pull: + status.info("Pulling images") + pull_commands = get_docker_compose_commands_to_run( + service=service, + remote_dependencies=sorted_remote_dependencies, + current_env=current_env, + command="pull", + options=[], + service_config_file_path=service_config_file_path, + mode_dependencies=mode_dependencies, + ) - with concurrent.futures.ThreadPoolExecutor() as pull_dependency_executor: - futures = [ - pull_dependency_executor.submit( - _pull_dependency_images, cmd, current_env, status - ) - for cmd in pull_commands - ] - for future in concurrent.futures.as_completed(futures): - _ = future.result() + with concurrent.futures.ThreadPoolExecutor() as pull_dependency_executor: + futures = [ + pull_dependency_executor.submit( + _pull_dependency_images, cmd, current_env, status + ) + for cmd in pull_commands + ] + for future in concurrent.futures.as_completed(futures): + _ = future.result() + else: + status.info("Skipping image pull (using cached images)") # Bring up all necessary containers up_commands = get_docker_compose_commands_to_run( diff --git a/devservices/main.py b/devservices/main.py index 7d5ceac..26ca9f1 100644 --- a/devservices/main.py +++ b/devservices/main.py @@ -148,6 +148,12 @@ def main() -> None: help="Path to a custom devservices config file", default=None, ) + parser.add_argument( + "--offline", + help="Use cached dependencies and images without network access", + action="store_true", + default=False, + ) subparsers = parser.add_subparsers(dest="command", title="commands", metavar="") diff --git a/devservices/utils/dependencies.py b/devservices/utils/dependencies.py index cc1bec8..fc1be68 100644 --- a/devservices/utils/dependencies.py +++ b/devservices/utils/dependencies.py @@ -214,6 +214,7 @@ def install_and_verify_dependencies( service: Service, force_update_dependencies: bool = False, modes: list[str] | None = None, + offline: bool = False, ) -> set[InstalledRemoteDependency]: """ Install and verify dependencies for a service @@ -235,6 +236,17 @@ def install_and_verify_dependencies( if dependency_key in mode_dependencies ] + if offline: + are_dependencies_valid = verify_local_dependencies(matching_dependencies) + if not are_dependencies_valid: + raise DependencyError( + repo_name="", + repo_link="", + branch="", + stderr="Cannot use --offline: some dependencies are not cached locally. Run 'devservices up' without --offline first to cache dependencies", + ) + return get_installed_remote_dependencies(matching_dependencies) + if force_update_dependencies: remote_dependencies = install_dependencies(matching_dependencies) else: diff --git a/devservices/utils/state.py b/devservices/utils/state.py index 63e19ce..09b3e55 100644 --- a/devservices/utils/state.py +++ b/devservices/utils/state.py @@ -30,7 +30,7 @@ def __new__(cls) -> State: if cls._instance is None: cls._instance = super(State, cls).__new__(cls) if not os.path.exists(DEVSERVICES_LOCAL_DIR): - os.makedirs(DEVSERVICES_LOCAL_DIR) + os.makedirs(DEVSERVICES_LOCAL_DIR, exist_ok=True) cls._instance.state_db_file = STATE_DB_FILE cls._instance.conn = sqlite3.connect(cls._instance.state_db_file) cls._instance.initialize_database() diff --git a/tests/commands/test_serve.py b/tests/commands/test_serve.py index 3bcb5a6..0790281 100644 --- a/tests/commands/test_serve.py +++ b/tests/commands/test_serve.py @@ -48,7 +48,11 @@ def test_serve_success( args = Namespace(extra=[]) - serve(args) + with patch( + "devservices.utils.supervisor.DEVSERVICES_SUPERVISOR_CONFIG_DIR", + str(tmp_path / "supervisor"), + ): + serve(args) mock_pty_spawn.assert_called_once_with(["run", "devserver"]) diff --git a/tests/utils/test_dependencies.py b/tests/utils/test_dependencies.py index 2e6ed24..ebd677b 100644 --- a/tests/utils/test_dependencies.py +++ b/tests/utils/test_dependencies.py @@ -3448,3 +3448,75 @@ def test_get_active_service_names_removes_stale_entries( remaining = state.get_service_entries(StateTables.STARTED_SERVICES) assert "stale-service" not in remaining assert "service-1" not in remaining + + +@mock.patch( + "devservices.utils.dependencies.verify_local_dependencies", return_value=True +) +@mock.patch( + "devservices.utils.dependencies.get_installed_remote_dependencies", + return_value=set(), +) +@mock.patch("devservices.utils.dependencies.install_dependencies", return_value=set()) +def test_install_and_verify_dependencies_offline_cached( + mock_install_dependencies: mock.Mock, + mock_get_installed: mock.Mock, + mock_verify: mock.Mock, +) -> None: + service = Service( + name="test-service", + repo_path="/path/to/test-service", + config=ServiceConfig( + version=0.1, + service_name="test-service", + dependencies={ + "dependency-1": Dependency( + description="dependency-1", + remote=RemoteConfig( + repo_name="dependency-1", + repo_link="file://path/to/dependency-1", + branch="main", + ), + dependency_type=DependencyType.SERVICE, + ), + }, + modes={"default": ["dependency-1"]}, + ), + ) + install_and_verify_dependencies(service, offline=True) + + # Should NOT call install_dependencies (no network) + mock_install_dependencies.assert_not_called() + # Should verify local dependencies and use cached versions + mock_verify.assert_called_once() + mock_get_installed.assert_called_once() + + +@mock.patch( + "devservices.utils.dependencies.verify_local_dependencies", return_value=False +) +def test_install_and_verify_dependencies_offline_not_cached( + mock_verify: mock.Mock, +) -> None: + service = Service( + name="test-service", + repo_path="/path/to/test-service", + config=ServiceConfig( + version=0.1, + service_name="test-service", + dependencies={ + "dependency-1": Dependency( + description="dependency-1", + remote=RemoteConfig( + repo_name="dependency-1", + repo_link="file://path/to/dependency-1", + branch="main", + ), + dependency_type=DependencyType.SERVICE, + ), + }, + modes={"default": ["dependency-1"]}, + ), + ) + with pytest.raises(DependencyError): + install_and_verify_dependencies(service, offline=True)