diff --git a/dvc_ssh/__init__.py b/dvc_ssh/__init__.py
index 8053def..3cf646e 100644
--- a/dvc_ssh/__init__.py
+++ b/dvc_ssh/__init__.py
@@ -65,9 +65,23 @@ def _prepare_credentials(self, **config):
         if port := config.get("port"):
             login_info["port"] = port
 
-        for option in ("password", "passphrase"):
+        # we can constrain which authentication methods are used, to avoid trying ones
+        # the user doesn't have configured, which can lead to input prompts and timeouts
+        # asyncssh: empty list is falsy, so methods are not constrained and all tried
+        login_info["preferred_auth"] = []
+
+        # non exhaustive, maps which dvc config option uses which ssh auth method
+        _option_to_auth_methods = {
+            "password": "password",
+            "passphrase": "publickey",
+        }
+
+        for option, auth_method in _option_to_auth_methods.items():
             login_info[option] = config.get(option)
 
+            if login_info[option] or config.get(f"ask_{option}"):
+                login_info["preferred_auth"].append(auth_method)
+
             if config.get(f"ask_{option}") and login_info[option] is None:
                 login_info[option] = ask_password(
                     login_info["host"],
diff --git a/dvc_ssh/tests/docker-compose.yml b/dvc_ssh/tests/docker-compose.yml
index ec5bcb0..e7ff9e0 100644
--- a/dvc_ssh/tests/docker-compose.yml
+++ b/dvc_ssh/tests/docker-compose.yml
@@ -5,7 +5,9 @@ services:
     image: ghcr.io/linuxserver/openssh-server
     environment:
       - USER_NAME=user
+      - USER_PASSWORD=password
       - PUBLIC_KEY_FILE=/tmp/key
+      - PASSWORD_ACCESS=true
     ports:
       - 2222
     volumes:
diff --git a/dvc_ssh/tests/test_conn.py b/dvc_ssh/tests/test_conn.py
new file mode 100644
index 0000000..7deaea0
--- /dev/null
+++ b/dvc_ssh/tests/test_conn.py
@@ -0,0 +1,45 @@
+from dvc_ssh import SSHFileSystem
+
+from .cloud import TEST_SSH_KEY_PATH
+
+
+def test_keyfile_set(ssh_server, mocker):
+    f = SSHFileSystem(
+        host=ssh_server["host"],
+        port=ssh_server["port"],
+        user="user",
+        keyfile=TEST_SSH_KEY_PATH,
+    )
+
+    assert f.fs
+
+
+def test_password_set(ssh_server, mocker):
+    # make sure that we don't open a password prompt if password is set
+    # this would let the test fail with a timeout
+    f = SSHFileSystem(
+        host=ssh_server["host"],
+        port=ssh_server["port"],
+        # see config in tests/docker-compose.yml, can't use environment variable here
+        # because it is not passed to the test process
+        user="user",
+        password="password",
+    )
+
+    assert f.fs
+
+
+def test_password_prompt(ssh_server, mocker):
+    f = SSHFileSystem(
+        host=ssh_server["host"],
+        port=ssh_server["port"],
+        user="user",
+        ask_password=True,
+    )
+
+    mock_getpass = mocker.patch(
+        "dvc_ssh.ask_password",
+    )
+    mock_getpass.return_value = "password"
+
+    assert f.fs