Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 40 additions & 40 deletions noq-proto/src/connection/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,8 @@ pub(crate) use packet_crypto::EncryptionLevel;

mod paths;
pub use paths::{
ClosedPath, PathAbandonReason, PathEvent, PathId, PathStatus, RttEstimator, SetPathStatusError,
ClosedPath, OpenPathOpts, PathAbandonReason, PathEvent, PathId, PathStatus, RttEstimator,
SetPathStatusError,
};
use paths::{PathData, PathState};

Expand Down Expand Up @@ -518,47 +519,19 @@ impl Connection {
}
}

/// Opens a new path only if no path on the same network path currently exists.
///
/// This comparison will use [`FourTuple::is_probably_same_path`] on the given `network_path`
/// and pass it existing path's network paths.
///
/// This means that you can pass `local_ip: None` to make the comparison only compare
/// remote addresses.
///
/// This avoids having to guess which local interface will be used to communicate with the
/// remote, should it not be known yet. We assume that if we already have a path to the remote,
/// the OS is likely to use the same interface to talk to said remote.
///
/// See also [`open_path`]. Returns `(path_id, true)` if the path already existed. `(path_id,
/// false)` if was opened.
///
/// [`open_path`]: Connection::open_path
pub fn open_path_ensure(
&mut self,
network_path: FourTuple,
initial_status: PathStatus,
now: Instant,
) -> Result<(PathId, bool), PathError> {
let existing_open_path = self.paths.iter().find(|(id, path)| {
network_path.is_probably_same_path(&path.data.network_path)
&& !self.abandoned_paths.contains(*id)
});
match existing_open_path {
Some((path_id, _state)) => Ok((*path_id, true)),
None => Ok((self.open_path(network_path, initial_status, now)?, false)),
}
}

/// Opens a new path.
///
/// By default, if a path with `network_path` already exists,
/// [`PathError::PathExistsForNetworkPath`] is returned. Set
/// [`OpenPathOpts::allow_duplicate`] to `true` to open a new path regardless.
///
/// Further errors might occur and they will be emitted in [`PathEvent::Abandoned`]
/// events with this path id. Once the path is opened and can carry application data it
/// will be reported using a [`PathEvent::Established`] event.
pub fn open_path(
&mut self,
network_path: FourTuple,
initial_status: PathStatus,
opts: OpenPathOpts,
now: Instant,
) -> Result<PathId, PathError> {
if !self.is_multipath_negotiated() {
Expand All @@ -568,6 +541,17 @@ impl Connection {
return Err(PathError::ServerSideNotAllowed);
}

if !opts.allow_duplicate {
let existing_open_path = self.paths.iter().find(|(id, path)| {
network_path.is_probably_same_path(&path.data.network_path)
&& !self.abandoned_paths.contains(*id)
});

if let Some((path_id, _state)) = existing_open_path {
return Err(PathError::PathExistsForNetworkPath(*path_id));
}
}

let max_abandoned = self.abandoned_paths.iter().max().copied();
let max_used = self.paths.keys().last().copied();
let path_id = max_abandoned
Expand Down Expand Up @@ -596,7 +580,7 @@ impl Connection {
}

let path = self.ensure_path(path_id, network_path, now, None);
path.status.local_update(initial_status);
path.status.local_update(opts.initial_status);

Ok(path_id)
}
Expand Down Expand Up @@ -5640,16 +5624,23 @@ impl Connection {
.ok()
.and_then(|s| s.pop_pending_path_open())
{
match self.open_path_ensure(network_path, PathStatus::Backup, now) {
Ok((path_id, already_existed)) => {
let opts = OpenPathOpts::default().initial_status(PathStatus::Backup);
match self.open_path(network_path, opts, now) {
Ok(path_id) => {
debug!(
%path_id,
?network_path,
new_path = !already_existed,
"Opened NAT traversal path",
);
}
Err(err) => match err {
PathError::PathExistsForNetworkPath(path_id) => {
debug!(
%path_id,
?network_path,
"NAT traversal path already exists",
);
}
PathError::MultipathNotNegotiated
| PathError::ServerSideNotAllowed
| PathError::ValidationFailed
Expand Down Expand Up @@ -5857,7 +5848,13 @@ impl Connection {
local_ip: None, /* allow the local ip to be discovered */
};

if open_first && let Err(e) = self.open_path(network_path, status, now) {
// We set `allow_duplicate` to true: the loop above cleared `local_ip` on every open
// path, so any other path with the same remote would otherwise fail to open.
let opts = OpenPathOpts::default()
.initial_status(status)
.allow_duplicate(true);

if open_first && let Err(e) = self.open_path(network_path, opts, now) {
if self.side().is_client() {
debug!(%e, "Failed to open new path for network change");
}
Expand All @@ -5874,7 +5871,7 @@ impl Connection {
continue;
}

if !open_first && let Err(e) = self.open_path(network_path, status, now) {
if !open_first && let Err(e) = self.open_path(network_path, opts, now) {
// Path has already been closed if we got here. Since the path was not recoverable,
// this might be desirable in any case, because other paths exist (!open_first) and
// this was is considered non recoverable
Expand Down Expand Up @@ -7447,6 +7444,9 @@ pub enum PathError {
/// The remote address for the path is not supported by the endpoint
#[error("invalid remote address")]
InvalidRemoteAddress(SocketAddr),
/// A path with the same network 4-tuple already exists.
#[error("a path with this network 4-tuple already exists (path id {_0})")]
PathExistsForNetworkPath(PathId),
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this new error needed? Why can the open_path function not simply return the PathId of the open path?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I can remove it. The information whether open_path opened a new path or returned the PathId of an existing path would be lost. We currently only use it to log differently in one place, no functionality depends on this.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, actually we can't with how I impl'd it, because in noq::Connection::open_path we now rely on the contract that noq_proto::Connection::open_path now either returns a new path id or an error, but never an existing path id. This made the impl simpler (no more watch channels), but could be changed again.

}

/// Errors triggered when abandoning a path
Expand Down
35 changes: 35 additions & 0 deletions noq-proto/src/connection/paths.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1039,6 +1039,41 @@ pub enum PathStatus {
Backup,
}

/// Options for opening a new path via [`Connection::open_path`].
///
/// Defaults to [`PathStatus::Available`] and `allow_duplicate: false`.
///
/// [`Connection::open_path`]: crate::Connection::open_path
#[derive(Debug, Copy, Clone, Default)]
pub struct OpenPathOpts {
pub(crate) initial_status: PathStatus,
pub(crate) allow_duplicate: bool,
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

did you have a motivation to have to keep this option?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  • in handle_network_change, when opening replacement paths for non-recoverable paths, I currently set allow_duplicates(true). Reason: The replacement path wouldn't be opened otherwise, because we cleared the local_ip beforehand on all paths.
  • Not a justification, but a reason in the initial PR: We need it in the noq test suite, because otherwise we couldn't open multiple paths over localhost reliably, which we do in the noq multipath tests.

Do you think it should be removed?

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In handle_network_change don't you first abandon the old path after which opening a new one should work? Can you point me to the code because I don't quite follow.

I recognise we use this functionality a lot in the tests, that's awkward and I need to think about this a bit more. My original thinking was to do the opposite: open_path would always open a path, and simply remove the open_path_ensure version. What this PR does now seems mostly replacing one API that's so-so with another API that's so-so. What was the reason to not simply remove open_path_ensure?

I'm not really enthusiastic about this OpenPathOpts I have to admit, so I'm trying to find reasons why we might not need it. I liked the old signature of open_path and would like to preserve it.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In handle_network_change don't you first abandon the old path after which opening a new one should work? Can you point me to the code because I don't quite follow.

if open_first && let Err(e) = self.open_path(network_path, opts, now) {
- if open_first is true we open before closing. The current logic is described here:
// Decide if we need to close first or open first in the multipath case.
- it is set to true when all paths are non-recoverable. The reasoning seems to be that otherwise we'd close the last open path before opening new paths.

open_path would always open a path

Why? Isn't it rarely the desired outcome to have multiple paths for the same FourTuple? Especially for public usage, I would guess you usually want to not create multiple paths for the same FourTuple. If open_path always opened a new path, we would have no public API to do that, and users would likely open redundant paths. So we'd have to add a new API for this, which is prone to TOCTOU because you'd need to lock-check-lock-set.

Also, internally, when opening paths for successful NAT traversal probes, we only want to open paths when no path exists yet for the FourTuple.

So I think it's either the OpenPathOpts or open_path and open_path_ensure. I can revert to the two methods if you prefer? I don't have a strong preference between the two (only that I always found the open_path_ensure name a little weird).

Or do you have other ideas how to have a way to "only open if no path exists yet for this network path" with a single function?

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think right now this is throwing too many questions that I can't resolve in the time we have for the RCs. I had reasons to want to remove the open_path_ensure variant but digging through the corner cases takes some work. So yeah, I'm leaning towards keeping the API as it was with open_path and open_path_ensure, they worked. So maybe that is the best call and we reduce this PR to being only about changing the SocketAddr with the FourTuple.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed. Opened #661 and closing this for now. If I come up with a better idea I'll do a new PR.

}

impl OpenPathOpts {
/// Sets the initial [`PathStatus`] for the new path.
#[must_use]
pub fn initial_status(mut self, initial_status: PathStatus) -> Self {
self.initial_status = initial_status;
self
}

/// Whether to open a new path even if another open path with the same network 4-tuple
/// already exists.
///
/// When `false` (the default), [`Connection::open_path`] returns
/// [`PathError::PathExistsForNetworkPath`] if an open path with the same network 4-tuple
/// already exists. When `true`, a new path is opened regardless.
///
/// [`Connection::open_path`]: crate::Connection::open_path
/// [`PathError::PathExistsForNetworkPath`]: crate::PathError::PathExistsForNetworkPath
#[must_use]
pub fn allow_duplicate(mut self, allow_duplicate: bool) -> Self {
self.allow_duplicate = allow_duplicate;
self
}
}

/// Application events about paths
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PathEvent {
Expand Down
4 changes: 2 additions & 2 deletions noq-proto/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@ pub(crate) mod connection;
pub use crate::connection::{
Chunk, Chunks, ClosePathError, ClosedPath, ClosedStream, Connection, ConnectionError,
ConnectionStats, Datagrams, Event, FinishError, FrameStats, MultipathNotNegotiated,
NetworkChangeHint, PathAbandonReason, PathError, PathEvent, PathId, PathStats, PathStatus,
ReadError, ReadableError, RecvStream, RttEstimator, SendDatagramError, SendStream,
NetworkChangeHint, OpenPathOpts, PathAbandonReason, PathError, PathEvent, PathId, PathStats,
PathStatus, ReadError, ReadableError, RecvStream, RttEstimator, SendDatagramError, SendStream,
SetPathStatusError, ShouldTransmit, StreamEvent, Streams, UdpStats, WriteError,
};
#[cfg(test)]
Expand Down
4 changes: 1 addition & 3 deletions noq-proto/src/tests/multipath.rs
Original file line number Diff line number Diff line change
Expand Up @@ -605,11 +605,9 @@ fn open_path_ensure_after_abandon() -> TestResult {
);

info!("opening path 2");
let (path_id, existed) = pair.open_path_ensure(Client, second_path, PathStatus::Available)?;
let path_id = pair.open_path(Client, second_path, PathStatus::Available)?;
pair.drive();

assert!(!existed);

// The path should have been opened:
assert_matches!(
pair.poll(Client),
Expand Down
6 changes: 4 additions & 2 deletions noq-proto/src/tests/random_interaction.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ use test_strategy::Arbitrary;
use tracing::{debug, error, info, trace};

use crate::{
ClientConfig, Connection, ConnectionHandle, Dir, FourTuple, PathId, PathStatus, Side, StreamId,
ClientConfig, Connection, ConnectionHandle, Dir, FourTuple, OpenPathOpts, PathId, PathStatus,
Side, StreamId,
tests::{Pair, TestEndpoint},
};

Expand Down Expand Up @@ -166,7 +167,8 @@ impl TestOp {
remote,
local_ip: None,
};
conn.open_path(network_path, status, now)
let opts = OpenPathOpts::default().initial_status(status);
conn.open_path(network_path, opts, now)
.inspect_err(|err| error!(?err, "OpenPath failed"))
.ok();
}
Expand Down
17 changes: 4 additions & 13 deletions noq-proto/src/tests/util.rs
Original file line number Diff line number Diff line change
Expand Up @@ -500,26 +500,17 @@ impl ConnPair {
self.conn_mut(side).send_stream(id)
}

pub(super) fn open_path_ensure(
&mut self,
side: Side,
network_path: FourTuple,
initial_status: PathStatus,
) -> Result<(PathId, bool), PathError> {
let now = self.pair.time;
self.conn_mut(side)
.open_path_ensure(network_path, initial_status, now)
}

pub(super) fn open_path(
&mut self,
side: Side,
network_path: FourTuple,
initial_status: PathStatus,
) -> Result<PathId, PathError> {
let now = self.pair.time;
self.conn_mut(side)
.open_path(network_path, initial_status, now)
let opts = OpenPathOpts::default()
.initial_status(initial_status)
.allow_duplicate(true);
self.conn_mut(side).open_path(network_path, opts, now)
}

pub(super) fn close_path(
Expand Down
Loading
Loading