- * BASIC -> validates UTF-8 in log records (throws a LogError if invalid),
- * throws a LogError on abandoned log records (missing .$() at the end of log statement),
- * detects closed stdout in LogConsoleWriter.
- * This introduces a low overhead to logging.
- * AGGRESSIVE -> BASIC + holds recent history of log lines to help diagnose closed stdout,
- * holds the stack trace of abandoned log record.
- * This introduces a significant overhead to logging.
- *
- * When running inside JUnit/Surefire, BASIC log paranoia mode gets activated automatically.
- * You can manually edit the code in the static { } block below to activate AGGRESSIVE instead.
- *
- * Logs may go silent when Maven Surefire plugin closes stdout due to broken text encoding.
- * In BASIC mode, the log writer will detect this and print errors through System.out, which
- * under Surefire uses an alternate channel and not stdout.
- * In AGGRESSIVE mode, it will additionally remember the most recent log lines and print them.
- * This will help you find the offending log line with broken encoding.
- *
- * The logging framework detects a common coding error where you forget to end a log statement
- * with .$(), causing the statement not to be logged. This problem can only be detected after
- * the fact, when you start a new log record and the previous one wasn't completed.
- *
- * With Log Paranoia off (LOG_PARANOIA_MODE_NONE), we only detect this problem and print an
- * error message.
- * In BASIC mode, we throw a LogError without a stack trace.
- * In AGGRESSIVE mode, we capture the stack trace at every start of a log statement, so when
- * we throw the LogError, it points to the code that created and then abandoned the log record.
- *
- */
- public static final int LOG_PARANOIA_MODE;
- public static final int LOG_PARANOIA_MODE_AGGRESSIVE = 2;
- public static final int LOG_PARANOIA_MODE_BASIC = 1;
- public static final int LOG_PARANOIA_MODE_NONE = 0;
- // Set to true to enable Thread Local path instances created/closed stack trace logs.
- public static final boolean THREAD_LOCAL_PATH_PARANOIA_MODE = false;
- // Set to true to enable stricter boundary checks on Vm memories implementations.
- public static final boolean VM_PARANOIA_MODE = false;
- // Set to true to enable stricter File Descriptor double close checks, trace closed usages.
- public static boolean FD_PARANOIA_MODE = false;
-
- public static boolean isInsideJUnitTest() {
- StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
- for (StackTraceElement element : stackTrace) {
- String className = element.getClassName();
- if (className.startsWith("org.apache.maven.surefire") || className.startsWith("org.junit.")) {
- return true;
- }
- }
- return false;
- }
-
- static {
- LOG_PARANOIA_MODE = isInsideJUnitTest() ? LOG_PARANOIA_MODE_BASIC : LOG_PARANOIA_MODE_NONE;
- }
-}
\ No newline at end of file
diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java
index c63609f..c815bf7 100644
--- a/core/src/main/java/io/questdb/client/Sender.java
+++ b/core/src/main/java/io/questdb/client/Sender.java
@@ -34,6 +34,7 @@
import io.questdb.client.cutlass.line.http.AbstractLineHttpSender;
import io.questdb.client.cutlass.line.tcp.DelegatingTlsChannel;
import io.questdb.client.cutlass.line.tcp.PlainTcpLineChannel;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
import io.questdb.client.impl.ConfStringParser;
import io.questdb.client.network.NetworkFacade;
import io.questdb.client.network.NetworkFacadeImpl;
@@ -95,7 +96,7 @@
* 2. Call {@link #reset()} to clear the internal buffers and start building a new row
* @@ -144,7 +145,12 @@ static LineSenderBuilder builder(CharSequence configurationString) { * @return Builder object to create a new Sender instance. */ static LineSenderBuilder builder(Transport transport) { - return new LineSenderBuilder(transport == Transport.HTTP ? LineSenderBuilder.PROTOCOL_HTTP : LineSenderBuilder.PROTOCOL_TCP); + int protocol = switch (transport) { + case HTTP -> LineSenderBuilder.PROTOCOL_HTTP; + case TCP -> LineSenderBuilder.PROTOCOL_TCP; + case WEBSOCKET -> LineSenderBuilder.PROTOCOL_WEBSOCKET; + }; + return new LineSenderBuilder(protocol); } /** @@ -461,7 +467,15 @@ enum Transport { * and for use-cases where HTTP transport is not suitable, when communicating with a QuestDB server over a high-latency * network */ - TCP + TCP, + + /** + * Use WebSocket transport to communicate with a QuestDB server. + *
+ * WebSocket transport uses the ILP v4 binary protocol for efficient data ingestion.
+ * It supports both synchronous and asynchronous modes with flow control.
+ */
+ WEBSOCKET
}
/**
@@ -508,12 +522,17 @@ final class LineSenderBuilder {
private static final int DEFAULT_BUFFER_CAPACITY = 64 * 1024;
private static final int DEFAULT_HTTP_PORT = 9000;
private static final int DEFAULT_HTTP_TIMEOUT = 30_000;
+ private static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = 8;
private static final int DEFAULT_MAXIMUM_BUFFER_CAPACITY = 100 * 1024 * 1024;
private static final int DEFAULT_MAX_BACKOFF_MILLIS = 1_000;
private static final int DEFAULT_MAX_NAME_LEN = 127;
private static final long DEFAULT_MAX_RETRY_NANOS = TimeUnit.SECONDS.toNanos(10); // keep sync with the contract of the configuration method
private static final long DEFAULT_MIN_REQUEST_THROUGHPUT = 100 * 1024; // 100KB/s, keep in sync with the contract of the configuration method
private static final int DEFAULT_TCP_PORT = 9009;
+ private static final int DEFAULT_WEBSOCKET_PORT = 9000;
+ private static final int DEFAULT_WS_AUTO_FLUSH_BYTES = 1024 * 1024; // 1MB
+ private static final long DEFAULT_WS_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms
+ private static final int DEFAULT_WS_AUTO_FLUSH_ROWS = 500;
private static final int MIN_BUFFER_SIZE = AuthUtils.CHALLENGE_LEN + 1; // challenge size + 1;
// The PARAMETER_NOT_SET_EXPLICITLY constant is used to detect if a parameter was set explicitly in configuration parameters
// where it matters. This is needed to detect invalid combinations of parameters. Why?
@@ -522,8 +541,11 @@ final class LineSenderBuilder {
private static final int PARAMETER_NOT_SET_EXPLICITLY = -1;
private static final int PROTOCOL_HTTP = 1;
private static final int PROTOCOL_TCP = 0;
+ private static final int PROTOCOL_WEBSOCKET = 2;
private final ObjList
- * The number of dimensions of this type is undefined, so the decoded number on
- * dimensions for the returned column type will be -1.
- */
- public static int encodeArrayTypeWithWeakDims(short elemType, boolean checkSupportedElementTypes) {
- return encodeArrayType(elemType, 1, checkSupportedElementTypes) | TYPE_FLAG_ARRAY_WEAK_DIMS;
- }
-
/**
* Generate a decimal type from a given precision and scale.
* It will choose the proper subtype (DECIMAL8, DECIMAL16, etc.) from the precision, depending on the amount
diff --git a/core/src/main/java/io/questdb/client/cairo/GeoHashes.java b/core/src/main/java/io/questdb/client/cairo/GeoHashes.java
deleted file mode 100644
index e785f91..0000000
--- a/core/src/main/java/io/questdb/client/cairo/GeoHashes.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*******************************************************************************
- * ___ _ ____ ____
- * / _ \ _ _ ___ ___| |_| _ \| __ )
- * | | | | | | |/ _ \/ __| __| | | | _ \
- * | |_| | |_| | __/\__ \ |_| |_| | |_) |
- * \__\_\\__,_|\___||___/\__|____/|____/
- *
- * Copyright (c) 2014-2019 Appsicle
- * Copyright (c) 2019-2026 QuestDB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- ******************************************************************************/
-
-package io.questdb.client.cairo;
-
-import io.questdb.client.std.Numbers;
-import io.questdb.client.std.NumericException;
-import io.questdb.client.std.str.CharSink;
-
-public class GeoHashes {
-
- // geohash null value: -1
- // we use the highest bit of every storage size (byte, short, int, long)
- // to indicate null value. When a null value is cast down, nullity is
- // preserved, i.e. highest bit remains set:
- // long nullLong = -1L;
- // short nullShort = (short) nullLong;
- // nullShort == nullLong;
- // in addition, -1 is the first negative non geohash value.
- public static final int MAX_STRING_LENGTH = 12;
- public static final long NULL = -1L;
-
- private static final char[] base32 = {
- '0', '1', '2', '3', '4', '5', '6', '7',
- '8', '9', 'b', 'c', 'd', 'e', 'f', 'g',
- 'h', 'j', 'k', 'm', 'n', 'p', 'q', 'r',
- 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'
- };
-
- public static void append(long hash, int bits, CharSink sink) {
- if (hash == GeoHashes.NULL) {
- sink.putAscii("null");
- } else {
- sink.putAscii('\"');
- if (bits < 0) {
- GeoHashes.appendCharsUnsafe(hash, -bits, sink);
- } else {
- GeoHashes.appendBinaryStringUnsafe(hash, bits, sink);
- }
- sink.putAscii('\"');
- }
- }
-
- public static void appendBinaryStringUnsafe(long hash, int bits, CharSink sink) {
- // Below assertion can happen if there is corrupt metadata
- // which should not happen in production code since reader and writer check table metadata
- assert bits > 0 && bits <= ColumnType.GEOLONG_MAX_BITS;
- for (int i = bits - 1; i >= 0; --i) {
- sink.putAscii(((hash >> i) & 1) == 1 ? '1' : '0');
- }
- }
-
- public static void appendChars(long hash, int chars, CharSink sink) {
- if (hash != NULL) {
- appendCharsUnsafe(hash, chars, sink);
- }
- }
-
- public static void appendCharsUnsafe(long hash, int chars, CharSink sink) {
- // Below assertion can happen if there is corrupt metadata
- // which should not happen in production code since reader and writer check table metadata
- assert chars > 0 && chars <= MAX_STRING_LENGTH;
- for (int i = chars - 1; i >= 0; --i) {
- sink.putAscii(base32[(int) ((hash >> i * 5) & 0x1F)]);
- }
- }
-
- public static long fromCoordinatesDeg(double lat, double lon, int bits) throws NumericException {
- if (lat < -90.0 || lat > 90.0) {
- throw NumericException.instance();
- }
- if (lon < -180.0 || lon > 180.0) {
- throw NumericException.instance();
- }
- if (bits < 0 || bits > ColumnType.GEOLONG_MAX_BITS) {
- throw NumericException.instance();
- }
- return fromCoordinatesDegUnsafe(lat, lon, bits);
- }
-
- public static long fromCoordinatesDegUnsafe(double lat, double lon, int bits) {
- long latq = (long) Math.scalb((lat + 90.0) / 180.0, 32);
- long lngq = (long) Math.scalb((lon + 180.0) / 360.0, 32);
- return Numbers.interleaveBits(latq, lngq) >>> (64 - bits);
- }
-}
diff --git a/core/src/main/java/io/questdb/client/cairo/arr/BorrowedArray.java b/core/src/main/java/io/questdb/client/cairo/arr/BorrowedArray.java
deleted file mode 100644
index 947574a..0000000
--- a/core/src/main/java/io/questdb/client/cairo/arr/BorrowedArray.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*******************************************************************************
- * ___ _ ____ ____
- * / _ \ _ _ ___ ___| |_| _ \| __ )
- * | | | | | | |/ _ \/ __| __| | | | _ \
- * | |_| | |_| | __/\__ \ |_| |_| | |_) |
- * \__\_\\__,_|\___||___/\__|____/|____/
- *
- * Copyright (c) 2014-2019 Appsicle
- * Copyright (c) 2019-2026 QuestDB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- ******************************************************************************/
-
-package io.questdb.client.cairo.arr;
-
-import io.questdb.client.cairo.ColumnType;
-import io.questdb.client.std.Mutable;
-
-public class BorrowedArray extends MutableArray implements Mutable {
-
- public BorrowedArray() {
- this.flatView = new BorrowedFlatArrayView();
- }
-
- /**
- * Resets to an invalid array.
- */
- @Override
- public void clear() {
- this.type = ColumnType.UNDEFINED;
- borrowedFlatView().reset();
- shape.clear();
- strides.clear();
- }
-
-}
\ No newline at end of file
diff --git a/core/src/main/java/io/questdb/client/cutlass/http/HttpCookie.java b/core/src/main/java/io/questdb/client/cutlass/http/HttpCookie.java
deleted file mode 100644
index b653c2d..0000000
--- a/core/src/main/java/io/questdb/client/cutlass/http/HttpCookie.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*******************************************************************************
- * ___ _ ____ ____
- * / _ \ _ _ ___ ___| |_| _ \| __ )
- * | | | | | | |/ _ \/ __| __| | | | _ \
- * | |_| | |_| | __/\__ \ |_| |_| | |_) |
- * \__\_\\__,_|\___||___/\__|____/|____/
- *
- * Copyright (c) 2014-2019 Appsicle
- * Copyright (c) 2019-2026 QuestDB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- ******************************************************************************/
-
-package io.questdb.client.cutlass.http;
-
-import io.questdb.client.std.Mutable;
-import io.questdb.client.std.str.CharSink;
-import io.questdb.client.std.str.DirectUtf8String;
-import io.questdb.client.std.str.Sinkable;
-import org.jetbrains.annotations.NotNull;
-
-public class HttpCookie implements Mutable, Sinkable {
- public DirectUtf8String cookieName;
- public DirectUtf8String domain;
- public long expires = -1L;
- public boolean httpOnly;
- public long maxAge;
- public boolean partitioned;
- public DirectUtf8String path;
- public DirectUtf8String sameSite;
- public boolean secure;
- public DirectUtf8String value;
-
- @Override
- public void clear() {
- this.domain = null;
- this.expires = -1L;
- this.httpOnly = false;
- this.maxAge = 0L;
- this.partitioned = false;
- this.path = null;
- this.sameSite = null;
- this.secure = false;
- this.value = null;
- this.cookieName = null;
- }
-
- @Override
- public void toSink(@NotNull CharSink sink) {
- sink.put('{');
-
- sink.put("cookieName=").putQuoted(cookieName);
- sink.put(", value=").putQuoted(value);
- if (domain != null) {
- sink.put(", domain=").putQuoted(domain);
- }
- if (path != null) {
- sink.put(", path=").putQuoted(path);
- }
- sink.put(", secure=").put(secure);
- sink.put(", httpOnly=").put(httpOnly);
- sink.put(", partitioned=").put(partitioned);
- sink.put(", expires=").put(expires);
- sink.put(", maxAge=").put(maxAge);
- if (sameSite != null) {
- sink.put(", sameSite=").putQuoted(sameSite);
- }
- sink.put('}');
- }
-
-}
\ No newline at end of file
diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/HttpClientException.java b/core/src/main/java/io/questdb/client/cutlass/http/client/HttpClientException.java
index bd1f7f7..79f8185 100644
--- a/core/src/main/java/io/questdb/client/cutlass/http/client/HttpClientException.java
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/HttpClientException.java
@@ -31,6 +31,7 @@ public class HttpClientException extends RuntimeException {
private final StringSink message = new StringSink();
private int errno = Integer.MIN_VALUE;
+ private boolean isTimeout;
public HttpClientException(String message) {
this.message.put(message);
@@ -53,6 +54,10 @@ public String getMessage() {
return errNoRender + " " + message;
}
+ public boolean isTimeout() {
+ return isTimeout;
+ }
+
public HttpClientException put(char value) {
message.put(value);
return this;
@@ -77,4 +82,9 @@ public HttpClientException putSize(long value) {
message.putSize(value);
return this;
}
+
+ public HttpClientException flagAsTimeout() {
+ this.isTimeout = true;
+ return this;
+ }
}
\ No newline at end of file
diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
new file mode 100644
index 0000000..84c4329
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
@@ -0,0 +1,902 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.http.client;
+
+import io.questdb.client.HttpClientConfiguration;
+import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode;
+import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameParser;
+import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode;
+import io.questdb.client.network.IOOperation;
+import io.questdb.client.network.NetworkFacade;
+import io.questdb.client.network.Socket;
+import io.questdb.client.network.SocketFactory;
+import io.questdb.client.network.TlsSessionInitFailedException;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.Misc;
+import io.questdb.client.std.QuietCloseable;
+import io.questdb.client.std.SecureRnd;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.std.Vect;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Base64;
+
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+
+/**
+ * Zero-GC WebSocket client built on QuestDB's native socket infrastructure.
+ *
+ * This client uses native memory buffers and non-blocking I/O with
+ * platform-specific event notification (epoll/kqueue/select).
+ *
+ * Features:
+ *
+ * Thread safety: This class is NOT thread-safe. Each connection should be
+ * accessed from a single thread at a time.
+ */
+public abstract class WebSocketClient implements QuietCloseable {
+
+ private static final int DEFAULT_RECV_BUFFER_SIZE = 65536;
+ private static final int DEFAULT_SEND_BUFFER_SIZE = 65536;
+ private static final Logger LOG = LoggerFactory.getLogger(WebSocketClient.class);
+ private static final ThreadLocal
+ * Usage:
+ *
+ * Usage:
+ *
+ * WARNING: Only use this for testing. Production code should use proper TLS validation.
+ *
+ * @return a new WebSocket client with insecure TLS
+ */
+ public static WebSocketClient newInsecureTlsInstance() {
+ return newInstance(DefaultHttpClientConfiguration.INSTANCE, JavaTlsClientSocketFactory.INSECURE_NO_VALIDATION);
+ }
+
+ /**
+ * Creates a new WebSocket client with the specified configuration and socket factory.
+ *
+ * @param configuration the HTTP client configuration
+ * @param socketFactory the socket factory for creating sockets
+ * @return a new platform-specific WebSocket client
+ */
+ public static WebSocketClient newInstance(HttpClientConfiguration configuration, SocketFactory socketFactory) {
+ return switch (Os.type) {
+ case Os.LINUX -> new WebSocketClientLinux(configuration, socketFactory);
+ case Os.DARWIN, Os.FREEBSD -> new WebSocketClientOsx(configuration, socketFactory);
+ case Os.WINDOWS -> new WebSocketClientWindows(configuration, socketFactory);
+ default -> throw new UnsupportedOperationException("Unsupported platform: " + Os.type);
+ };
+ }
+
+ /**
+ * Creates a new plain text WebSocket client with default configuration.
+ *
+ * @return a new plain text WebSocket client
+ */
+ public static WebSocketClient newPlainTextInstance() {
+ return newPlainTextInstance(DefaultHttpClientConfiguration.INSTANCE);
+ }
+
+ /**
+ * Creates a new plain text WebSocket client with the specified configuration.
+ *
+ * @param configuration the HTTP client configuration
+ * @return a new plain text WebSocket client
+ */
+ public static WebSocketClient newPlainTextInstance(HttpClientConfiguration configuration) {
+ return newInstance(configuration, PlainSocketFactory.INSTANCE);
+ }
+
+ /**
+ * Creates a new TLS WebSocket client with the specified configuration.
+ *
+ * @param configuration the HTTP client configuration
+ * @param tlsConfig the TLS configuration
+ * @return a new TLS WebSocket client
+ */
+ public static WebSocketClient newTlsInstance(HttpClientConfiguration configuration, ClientTlsConfiguration tlsConfig) {
+ return newInstance(configuration, new JavaTlsClientSocketFactory(tlsConfig));
+ }
+
+ /**
+ * Creates a new TLS WebSocket client with default HTTP configuration.
+ *
+ * @param tlsConfig the TLS configuration
+ * @return a new TLS WebSocket client
+ */
+ public static WebSocketClient newTlsInstance(ClientTlsConfiguration tlsConfig) {
+ return newTlsInstance(DefaultHttpClientConfiguration.INSTANCE, tlsConfig);
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java
new file mode 100644
index 0000000..f4ac6ba
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java
@@ -0,0 +1,71 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.http.client;
+
+import io.questdb.client.HttpClientConfiguration;
+import io.questdb.client.network.Epoll;
+import io.questdb.client.network.EpollAccessor;
+import io.questdb.client.network.IOOperation;
+import io.questdb.client.network.SocketFactory;
+import io.questdb.client.std.Misc;
+
+/**
+ * Linux-specific WebSocket client using epoll for I/O waiting.
+ */
+public class WebSocketClientLinux extends WebSocketClient {
+ private Epoll epoll;
+
+ public WebSocketClientLinux(HttpClientConfiguration configuration, SocketFactory socketFactory) {
+ super(configuration, socketFactory);
+ epoll = new Epoll(
+ configuration.getEpollFacade(),
+ configuration.getWaitQueueCapacity()
+ );
+ }
+
+ @Override
+ public void close() {
+ super.close();
+ epoll = Misc.free(epoll);
+ }
+
+ @Override
+ protected void ioWait(int timeout, int op) {
+ final int event = op == IOOperation.WRITE ? EpollAccessor.EPOLLOUT : EpollAccessor.EPOLLIN;
+ if (epoll.control(socket.getFd(), 0, EpollAccessor.EPOLL_CTL_MOD, event) < 0) {
+ throw new HttpClientException("internal error: epoll_ctl failure [op=").put(op)
+ .put(", errno=").put(nf.errno())
+ .put(']');
+ }
+ dieWaiting(epoll.poll(timeout));
+ }
+
+ @Override
+ protected void setupIoWait() {
+ if (epoll.control(socket.getFd(), 0, EpollAccessor.EPOLL_CTL_ADD, EpollAccessor.EPOLLOUT) < 0) {
+ throw new HttpClientException("internal error: epoll_ctl failure [cmd=add, errno=").put(nf.errno()).put(']');
+ }
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java
new file mode 100644
index 0000000..d34df7c
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java
@@ -0,0 +1,75 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.http.client;
+
+import io.questdb.client.HttpClientConfiguration;
+import io.questdb.client.network.IOOperation;
+import io.questdb.client.network.Kqueue;
+import io.questdb.client.network.SocketFactory;
+import io.questdb.client.std.Misc;
+
+/**
+ * macOS-specific WebSocket client using kqueue for I/O waiting.
+ */
+public class WebSocketClientOsx extends WebSocketClient {
+ private Kqueue kqueue;
+
+ public WebSocketClientOsx(HttpClientConfiguration configuration, SocketFactory socketFactory) {
+ super(configuration, socketFactory);
+ this.kqueue = new Kqueue(
+ configuration.getKQueueFacade(),
+ configuration.getWaitQueueCapacity()
+ );
+ }
+
+ @Override
+ public void close() {
+ super.close();
+ this.kqueue = Misc.free(kqueue);
+ }
+
+ @Override
+ protected void ioWait(int timeout, int op) {
+ kqueue.setWriteOffset(0);
+ if (op == IOOperation.READ) {
+ kqueue.readFD(socket.getFd(), 0);
+ } else {
+ kqueue.writeFD(socket.getFd(), 0);
+ }
+
+ // 1 = always one FD, we are a single threaded network client
+ if (kqueue.register(1) != 0) {
+ throw new HttpClientException("could not register with kqueue [op=").put(op)
+ .put(", errno=").errno(nf.errno())
+ .put(']');
+ }
+ dieWaiting(kqueue.poll(timeout));
+ }
+
+ @Override
+ protected void setupIoWait() {
+ // no-op on macOS
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java
new file mode 100644
index 0000000..cdaec88
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java
@@ -0,0 +1,74 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.http.client;
+
+import io.questdb.client.HttpClientConfiguration;
+import io.questdb.client.network.FDSet;
+import io.questdb.client.network.IOOperation;
+import io.questdb.client.network.SelectFacade;
+import io.questdb.client.network.SocketFactory;
+import io.questdb.client.std.Misc;
+
+/**
+ * Windows-specific WebSocket client using select for I/O waiting.
+ */
+public class WebSocketClientWindows extends WebSocketClient {
+ private final SelectFacade sf;
+ private FDSet fdSet;
+
+ public WebSocketClientWindows(HttpClientConfiguration configuration, SocketFactory socketFactory) {
+ super(configuration, socketFactory);
+ this.fdSet = new FDSet(configuration.getWaitQueueCapacity());
+ this.sf = configuration.getSelectFacade();
+ }
+
+ @Override
+ public void close() {
+ super.close();
+ this.fdSet = Misc.free(fdSet);
+ }
+
+ @Override
+ protected void ioWait(int timeout, int op) {
+ final long readAddr;
+ final long writeAddr;
+ fdSet.clear();
+ fdSet.add(socket.getFd());
+ fdSet.setCount(1);
+ if (op == IOOperation.READ) {
+ readAddr = fdSet.address();
+ writeAddr = 0;
+ } else {
+ readAddr = 0;
+ writeAddr = fdSet.address();
+ }
+ dieWaiting(sf.select(readAddr, writeAddr, 0, timeout));
+ }
+
+ @Override
+ protected void setupIoWait() {
+ // no-op on Windows
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java
new file mode 100644
index 0000000..e3682f5
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java
@@ -0,0 +1,93 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.http.client;
+
+/**
+ * Callback interface for handling received WebSocket frames.
+ *
+ * Implementations should process received data efficiently and avoid blocking,
+ * as callbacks are invoked on the I/O thread.
+ *
+ * Thread safety: Callbacks are invoked from the thread that called receiveFrame().
+ * Implementations must handle their own synchronization if accessed from multiple threads.
+ */
+public interface WebSocketFrameHandler {
+
+ /**
+ * Called when a binary frame is received.
+ *
+ * @param payloadPtr pointer to the payload data in native memory
+ * @param payloadLen length of the payload in bytes
+ */
+ void onBinaryMessage(long payloadPtr, int payloadLen);
+
+ /**
+ * Called when a close frame is received from the server.
+ *
+ * After this callback, the connection will be closed. The handler should
+ * perform any necessary cleanup.
+ *
+ * @param code the close status code (e.g., 1000 for normal closure)
+ * @param reason the close reason (may be null or empty)
+ */
+ void onClose(int code, String reason);
+
+ /**
+ * Called when a ping frame is received.
+ *
+ * Default implementation does nothing. The WebSocketClient automatically
+ * sends a pong response, so this callback is for informational purposes only.
+ *
+ * @param payloadPtr pointer to the ping payload in native memory
+ * @param payloadLen length of the payload in bytes
+ */
+ default void onPing(long payloadPtr, int payloadLen) {
+ // Default: handled automatically by client
+ }
+
+ /**
+ * Called when a pong frame is received.
+ *
+ * Default implementation does nothing.
+ *
+ * @param payloadPtr pointer to the pong payload in native memory
+ * @param payloadLen length of the payload in bytes
+ */
+ default void onPong(long payloadPtr, int payloadLen) {
+ // Default: ignore pong frames
+ }
+
+ /**
+ * Called when a text frame is received.
+ *
+ * Default implementation does nothing. Override if text frames need handling.
+ *
+ * @param payloadPtr pointer to the UTF-8 encoded payload in native memory
+ * @param payloadLen length of the payload in bytes
+ */
+ default void onTextMessage(long payloadPtr, int payloadLen) {
+ // Default: ignore text frames
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java
new file mode 100644
index 0000000..d31044f
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java
@@ -0,0 +1,563 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.http.client;
+
+import io.questdb.client.cutlass.line.array.ArrayBufferAppender;
+import io.questdb.client.cutlass.qwp.client.NativeBufferWriter;
+import io.questdb.client.cutlass.qwp.client.QwpBufferWriter;
+import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameWriter;
+import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.Numbers;
+import io.questdb.client.std.QuietCloseable;
+import io.questdb.client.std.SecureRnd;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.std.Vect;
+
+/**
+ * Zero-GC WebSocket send buffer that implements {@link ArrayBufferAppender} for direct
+ * payload writing. Manages native memory with safe growth and handles WebSocket frame
+ * building (reserve header -> write payload -> patch header -> mask).
+ *
+ * Usage pattern:
+ *
+ * Thread safety: This class is NOT thread-safe. Each connection should have its own buffer.
+ */
+public class WebSocketSendBuffer implements QwpBufferWriter, QuietCloseable {
+
+ private static final int DEFAULT_INITIAL_CAPACITY = 65536;
+ private static final int MAX_BUFFER_SIZE = Integer.MAX_VALUE - 8; // Leave room for alignment
+ // Maximum header size: 2 (base) + 8 (64-bit length) + 4 (mask key)
+ private static final int MAX_HEADER_SIZE = 14;
+ private final FrameInfo frameInfo = new FrameInfo();
+ private final int maxBufferSize;
+ private final SecureRnd rnd;
+ private int bufCapacity;
+ private long bufPtr;
+ private int frameStartOffset; // Where current frame's reserved header starts
+ private int payloadStartOffset; // Where payload begins (frameStart + MAX_HEADER_SIZE)
+ private int writePos; // Current write position (offset from bufPtr)
+
+ /**
+ * Creates a new WebSocket send buffer with default initial capacity.
+ */
+ public WebSocketSendBuffer() {
+ this(DEFAULT_INITIAL_CAPACITY, MAX_BUFFER_SIZE);
+ }
+
+ /**
+ * Creates a new WebSocket send buffer with specified initial capacity.
+ *
+ * @param initialCapacity initial buffer size in bytes
+ */
+ public WebSocketSendBuffer(int initialCapacity) {
+ this(initialCapacity, MAX_BUFFER_SIZE);
+ }
+
+ /**
+ * Creates a new WebSocket send buffer with specified initial and max capacity.
+ *
+ * @param initialCapacity initial buffer size in bytes
+ * @param maxBufferSize maximum buffer size in bytes
+ */
+ public WebSocketSendBuffer(int initialCapacity, int maxBufferSize) {
+ this.bufCapacity = Math.max(initialCapacity, MAX_HEADER_SIZE * 2);
+ this.maxBufferSize = maxBufferSize;
+ this.bufPtr = Unsafe.malloc(bufCapacity, MemoryTag.NATIVE_DEFAULT);
+ this.writePos = 0;
+ this.frameStartOffset = 0;
+ this.payloadStartOffset = 0;
+ this.rnd = new SecureRnd();
+ }
+
+ /**
+ * Begins a new WebSocket frame. Reserves space for the maximum header size.
+ * The opcode is specified later when ending the frame via {@link #endFrame(int)}.
+ */
+ public void beginFrame() {
+ frameStartOffset = writePos;
+ // Reserve maximum header space
+ ensureCapacity(MAX_HEADER_SIZE);
+ writePos += MAX_HEADER_SIZE;
+ payloadStartOffset = writePos;
+ }
+
+ @Override
+ public void close() {
+ if (bufPtr != 0) {
+ Unsafe.free(bufPtr, bufCapacity, MemoryTag.NATIVE_DEFAULT);
+ bufPtr = 0;
+ bufCapacity = 0;
+ }
+ }
+
+ /**
+ * Finishes the current binary frame, writing the header and applying masking.
+ * Returns information about where to find the complete frame in the buffer.
+ *
+ * IMPORTANT: Only call this after all payload writes are complete. The buffer
+ * pointer is stable after this call (no more reallocations for this frame).
+ *
+ * @return frame info containing offset and length for sending
+ */
+ public FrameInfo endBinaryFrame() {
+ return endFrame(WebSocketOpcode.BINARY);
+ }
+
+ /**
+ * Finishes the current frame with the specified opcode.
+ *
+ * @param opcode the frame opcode
+ * @return frame info containing offset and length for sending
+ */
+ public FrameInfo endFrame(int opcode) {
+ int payloadLen = writePos - payloadStartOffset;
+
+ // Calculate actual header size (with mask key for client frames)
+ int actualHeaderSize = WebSocketFrameWriter.headerSize(payloadLen, true);
+ int unusedSpace = MAX_HEADER_SIZE - actualHeaderSize;
+ int actualFrameStart = frameStartOffset + unusedSpace;
+
+ // Generate mask key
+ int maskKey = rnd.nextInt();
+
+ // Write header at actual position (after unused space)
+ WebSocketFrameWriter.writeHeader(bufPtr + actualFrameStart, true, opcode, payloadLen, maskKey);
+
+ // Apply mask to payload
+ if (payloadLen > 0) {
+ WebSocketFrameWriter.maskPayload(bufPtr + payloadStartOffset, payloadLen, maskKey);
+ }
+
+ return frameInfo.set(actualFrameStart, actualHeaderSize + payloadLen);
+ }
+
+ /**
+ * Finishes the current text frame, writing the header and applying masking.
+ */
+ public FrameInfo endTextFrame() {
+ return endFrame(WebSocketOpcode.TEXT);
+ }
+
+ /**
+ * Ensures the buffer has capacity for the specified number of additional bytes.
+ * May reallocate the buffer if necessary.
+ *
+ * @param additionalBytes number of additional bytes needed
+ */
+ @Override
+ public void ensureCapacity(int additionalBytes) {
+ long requiredCapacity = (long) writePos + additionalBytes;
+ if (requiredCapacity > bufCapacity) {
+ grow(requiredCapacity);
+ }
+ }
+
+ /**
+ * Gets the buffer pointer. Only use this for reading after frame is complete.
+ */
+ public long getBufferPtr() {
+ return bufPtr;
+ }
+
+ /**
+ * Gets the current buffer capacity.
+ */
+ public int getCapacity() {
+ return bufCapacity;
+ }
+
+ /**
+ * Gets the payload length of the current frame being built.
+ */
+ public int getCurrentPayloadLength() {
+ return writePos - payloadStartOffset;
+ }
+
+ /**
+ * Gets the current write position (number of bytes written).
+ */
+ @Override
+ public int getPosition() {
+ return writePos;
+ }
+
+ /**
+ * Gets the current write position (total bytes written since last reset).
+ */
+ public int getWritePos() {
+ return writePos;
+ }
+
+ /**
+ * Patches an int value at the specified offset.
+ */
+ @Override
+ public void patchInt(int offset, int value) {
+ Unsafe.getUnsafe().putInt(bufPtr + offset, value);
+ }
+
+ /**
+ * Writes an ASCII string.
+ */
+ public void putAscii(CharSequence cs) {
+ if (cs == null) {
+ return;
+ }
+ int len = cs.length();
+ ensureCapacity(len);
+ for (int i = 0; i < len; i++) {
+ Unsafe.getUnsafe().putByte(bufPtr + writePos + i, (byte) cs.charAt(i));
+ }
+ writePos += len;
+ }
+
+ @Override
+ public void putBlockOfBytes(long from, long len) {
+ if (len <= 0) {
+ return;
+ }
+ if (len > Integer.MAX_VALUE) {
+ throw new IllegalArgumentException("len exceeds int range: " + len);
+ }
+ int intLen = (int) len;
+ ensureCapacity(intLen);
+ Vect.memcpy(bufPtr + writePos, from, intLen);
+ writePos += intLen;
+ }
+
+ @Override
+ public void putByte(byte b) {
+ ensureCapacity(1);
+ Unsafe.getUnsafe().putByte(bufPtr + writePos, b);
+ writePos++;
+ }
+
+ /**
+ * Writes raw bytes from a byte array.
+ */
+ public void putBytes(byte[] bytes, int offset, int length) {
+ if (length <= 0) {
+ return;
+ }
+ ensureCapacity(length);
+ for (int i = 0; i < length; i++) {
+ Unsafe.getUnsafe().putByte(bufPtr + writePos + i, bytes[offset + i]);
+ }
+ writePos += length;
+ }
+
+ @Override
+ public void putDouble(double value) {
+ ensureCapacity(8);
+ Unsafe.getUnsafe().putDouble(bufPtr + writePos, value);
+ writePos += 8;
+ }
+
+ /**
+ * Writes a float value.
+ */
+ public void putFloat(float value) {
+ ensureCapacity(4);
+ Unsafe.getUnsafe().putFloat(bufPtr + writePos, value);
+ writePos += 4;
+ }
+
+ @Override
+ public void putInt(int value) {
+ ensureCapacity(4);
+ Unsafe.getUnsafe().putInt(bufPtr + writePos, value);
+ writePos += 4;
+ }
+
+ @Override
+ public void putLong(long value) {
+ ensureCapacity(8);
+ Unsafe.getUnsafe().putLong(bufPtr + writePos, value);
+ writePos += 8;
+ }
+
+ /**
+ * Writes a long value in big-endian format.
+ */
+ public void putLongBE(long value) {
+ ensureCapacity(8);
+ Unsafe.getUnsafe().putLong(bufPtr + writePos, Long.reverseBytes(value));
+ writePos += 8;
+ }
+
+ /**
+ * Writes a short value in little-endian format.
+ */
+ public void putShort(short value) {
+ ensureCapacity(2);
+ Unsafe.getUnsafe().putShort(bufPtr + writePos, value);
+ writePos += 2;
+ }
+
+ /**
+ * Writes a length-prefixed UTF-8 string.
+ */
+ @Override
+ public void putString(String value) {
+ if (value == null || value.isEmpty()) {
+ putVarint(0);
+ return;
+ }
+ int utf8Len = NativeBufferWriter.utf8Length(value);
+ putVarint(utf8Len);
+ putUtf8(value);
+ }
+
+ /**
+ * Writes UTF-8 encoded bytes directly without length prefix.
+ */
+ @Override
+ public void putUtf8(String value) {
+ if (value == null || value.isEmpty()) {
+ return;
+ }
+ for (int i = 0, n = value.length(); i < n; i++) {
+ char c = value.charAt(i);
+ if (c < 0x80) {
+ putByte((byte) c);
+ } else if (c < 0x800) {
+ putByte((byte) (0xC0 | (c >> 6)));
+ putByte((byte) (0x80 | (c & 0x3F)));
+ } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) {
+ char c2 = value.charAt(++i);
+ if (Character.isLowSurrogate(c2)) {
+ int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00);
+ putByte((byte) (0xF0 | (codePoint >> 18)));
+ putByte((byte) (0x80 | ((codePoint >> 12) & 0x3F)));
+ putByte((byte) (0x80 | ((codePoint >> 6) & 0x3F)));
+ putByte((byte) (0x80 | (codePoint & 0x3F)));
+ } else {
+ putByte((byte) '?');
+ i--;
+ }
+ } else if (Character.isSurrogate(c)) {
+ putByte((byte) '?');
+ } else {
+ putByte((byte) (0xE0 | (c >> 12)));
+ putByte((byte) (0x80 | ((c >> 6) & 0x3F)));
+ putByte((byte) (0x80 | (c & 0x3F)));
+ }
+ }
+ }
+
+ /**
+ * Writes an unsigned variable-length integer (LEB128 encoding).
+ */
+ @Override
+ public void putVarint(long value) {
+ while (value > 0x7F) {
+ putByte((byte) ((value & 0x7F) | 0x80));
+ value >>>= 7;
+ }
+ putByte((byte) value);
+ }
+
+ /**
+ * Resets the buffer for reuse. Does not deallocate memory.
+ */
+ public void reset() {
+ writePos = 0;
+ frameStartOffset = 0;
+ payloadStartOffset = 0;
+ }
+
+ /**
+ * Skips the specified number of bytes, advancing the position.
+ */
+ @Override
+ public void skip(int bytes) {
+ ensureCapacity(bytes);
+ writePos += bytes;
+ }
+
+ /**
+ * Writes a complete close frame.
+ *
+ * @param code close status code (e.g., 1000 for normal closure)
+ * @param reason optional reason string (may be null)
+ * @return frame info for sending
+ */
+ public FrameInfo writeCloseFrame(int code, String reason) {
+ int payloadLen = 2; // status code
+ byte[] reasonBytes = null;
+ if (reason != null && !reason.isEmpty()) {
+ reasonBytes = reason.getBytes(java.nio.charset.StandardCharsets.UTF_8);
+ payloadLen += reasonBytes.length;
+ }
+
+ if (payloadLen > 125) {
+ throw new HttpClientException("Close payload too large [len=").put(payloadLen).put(']');
+ }
+
+ int frameStart = writePos;
+ int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true);
+ ensureCapacity(headerSize + payloadLen);
+
+ int maskKey = rnd.nextInt();
+ int written = WebSocketFrameWriter.writeHeader(bufPtr + writePos, true, WebSocketOpcode.CLOSE, payloadLen, maskKey);
+ writePos += written;
+
+ // Write status code (big-endian)
+ long payloadStart = bufPtr + writePos;
+ Unsafe.getUnsafe().putByte(payloadStart, (byte) ((code >> 8) & 0xFF));
+ Unsafe.getUnsafe().putByte(payloadStart + 1, (byte) (code & 0xFF));
+ writePos += 2;
+
+ // Write reason if present
+ if (reasonBytes != null) {
+ for (byte reasonByte : reasonBytes) {
+ Unsafe.getUnsafe().putByte(bufPtr + writePos++, reasonByte);
+ }
+ }
+
+ // Mask the payload (including status code and reason)
+ WebSocketFrameWriter.maskPayload(payloadStart, payloadLen, maskKey);
+
+ return frameInfo.set(frameStart, headerSize + payloadLen);
+ }
+
+ /**
+ * Writes a complete ping frame (control frame, no masking needed for server).
+ * Note: Client frames MUST be masked per RFC 6455. This writes a masked ping.
+ *
+ * @return frame info for sending
+ */
+ public FrameInfo writePingFrame() {
+ return writePingFrame(0, 0);
+ }
+
+ /**
+ * Writes a complete ping frame with payload.
+ *
+ * @param payloadPtr pointer to ping payload
+ * @param payloadLen length of payload (max 125 bytes for control frames)
+ * @return frame info for sending
+ */
+ public FrameInfo writePingFrame(long payloadPtr, int payloadLen) {
+ if (payloadLen > 125) {
+ throw new HttpClientException("Ping payload too large [len=").put(payloadLen).put(']');
+ }
+
+ int frameStart = writePos;
+ int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true);
+ ensureCapacity(headerSize + payloadLen);
+
+ int maskKey = rnd.nextInt();
+ int written = WebSocketFrameWriter.writeHeader(bufPtr + writePos, true, WebSocketOpcode.PING, payloadLen, maskKey);
+ writePos += written;
+
+ if (payloadLen > 0) {
+ Vect.memcpy(bufPtr + writePos, payloadPtr, payloadLen);
+ WebSocketFrameWriter.maskPayload(bufPtr + writePos, payloadLen, maskKey);
+ writePos += payloadLen;
+ }
+
+ return frameInfo.set(frameStart, headerSize + payloadLen);
+ }
+
+ /**
+ * Writes a complete pong frame.
+ *
+ * @param payloadPtr pointer to pong payload (should match received ping)
+ * @param payloadLen length of payload
+ * @return frame info for sending
+ */
+ public FrameInfo writePongFrame(long payloadPtr, int payloadLen) {
+ if (payloadLen > 125) {
+ throw new HttpClientException("Pong payload too large [len=").put(payloadLen).put(']');
+ }
+
+ int frameStart = writePos;
+ int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true);
+ ensureCapacity(headerSize + payloadLen);
+
+ int maskKey = rnd.nextInt();
+ int written = WebSocketFrameWriter.writeHeader(bufPtr + writePos, true, WebSocketOpcode.PONG, payloadLen, maskKey);
+ writePos += written;
+
+ if (payloadLen > 0) {
+ Vect.memcpy(bufPtr + writePos, payloadPtr, payloadLen);
+ WebSocketFrameWriter.maskPayload(bufPtr + writePos, payloadLen, maskKey);
+ writePos += payloadLen;
+ }
+
+ return frameInfo.set(frameStart, headerSize + payloadLen);
+ }
+
+ private void grow(long requiredCapacity) {
+ if (requiredCapacity > maxBufferSize) {
+ throw new HttpClientException("WebSocket buffer size exceeded maximum [required=")
+ .put(requiredCapacity)
+ .put(", max=")
+ .put(maxBufferSize)
+ .put(']');
+ }
+ int newCapacity = Math.min(
+ Math.max(Numbers.ceilPow2((int) requiredCapacity), (int) requiredCapacity),
+ maxBufferSize
+ );
+ bufPtr = Unsafe.realloc(bufPtr, bufCapacity, newCapacity, MemoryTag.NATIVE_DEFAULT);
+ bufCapacity = newCapacity;
+ }
+
+ /**
+ * Information about a completed WebSocket frame's location in the buffer.
+ * This class is mutable and reused to avoid allocations. Callers must
+ * extract values before calling any end*Frame() method again.
+ */
+ public static final class FrameInfo {
+ /**
+ * Total length of the frame (header + payload).
+ */
+ public int length;
+ /**
+ * Offset from buffer start where the frame begins.
+ */
+ public int offset;
+
+ FrameInfo set(int offset, int length) {
+ this.offset = offset;
+ this.length = length;
+ return this;
+ }
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/json/JsonLexer.java b/core/src/main/java/io/questdb/client/cutlass/json/JsonLexer.java
index 8b4caf0..1f0cc05 100644
--- a/core/src/main/java/io/questdb/client/cutlass/json/JsonLexer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/json/JsonLexer.java
@@ -65,6 +65,7 @@ public class JsonLexer implements Mutable, Closeable {
private boolean quoted = false;
private int state = S_START;
private boolean useCache = false;
+
public JsonLexer(int cacheSize, int cacheSizeLimit) {
this.cacheSizeLimit = cacheSizeLimit;
// if cacheSizeLimit is 0 or negative, the cache is disabled
@@ -398,4 +399,4 @@ private void utf8DecodeCacheAndBuffer(long lo, long hi, int position) throws Jso
unquotedTerminators.add('{');
unquotedTerminators.add('[');
}
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/line/AbstractLineSender.java b/core/src/main/java/io/questdb/client/cutlass/line/AbstractLineSender.java
index 0ed3937..11274a1 100644
--- a/core/src/main/java/io/questdb/client/cutlass/line/AbstractLineSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/line/AbstractLineSender.java
@@ -368,7 +368,7 @@ private static int findEOL(long ptr, int len) {
private byte[] receiveChallengeBytes() {
int n = 0;
- for (;;) {
+ for (; ; ) {
int rc = lineChannel.receive(ptr + n, capacity - n);
if (rc < 0) {
int errno = lineChannel.errno();
@@ -505,4 +505,4 @@ protected AbstractLineSender writeFieldName(CharSequence name) {
}
throw new LineSenderException("table expected");
}
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/line/LineSenderException.java b/core/src/main/java/io/questdb/client/cutlass/line/LineSenderException.java
index 9c6cc16..6fdaf9a 100644
--- a/core/src/main/java/io/questdb/client/cutlass/line/LineSenderException.java
+++ b/core/src/main/java/io/questdb/client/cutlass/line/LineSenderException.java
@@ -39,15 +39,9 @@
*
+ * @see io.questdb.client.Sender
*
- *
+ * This dictionary is shared across all tables and columns within a client instance.
+ * IDs are assigned sequentially starting from 0, ensuring contiguous ID space.
+ *
+ * Thread safety: This class is NOT thread-safe. External synchronization is required
+ * if accessed from multiple threads.
+ */
+public class GlobalSymbolDictionary {
+
+ private final ObjList
+ * After clearing, the next symbol added will get ID 0.
+ */
+ public void clear() {
+ symbolToId.clear();
+ idToSymbol.clear();
+ }
+
+ /**
+ * Checks if the dictionary contains the given symbol.
+ *
+ * @param symbol the symbol to check
+ * @return true if the symbol exists in the dictionary
+ */
+ public boolean contains(String symbol) {
+ return symbol != null && symbolToId.get(symbol) != CharSequenceIntHashMap.NO_ENTRY_VALUE;
+ }
+
+ /**
+ * Gets the ID for an existing symbol, or -1 if not found.
+ *
+ * @param symbol the symbol string
+ * @return the symbol ID, or -1 if not in dictionary
+ */
+ public int getId(String symbol) {
+ if (symbol == null) {
+ return -1;
+ }
+ int id = symbolToId.get(symbol);
+ return id == CharSequenceIntHashMap.NO_ENTRY_VALUE ? -1 : id;
+ }
+
+ /**
+ * Gets or adds a symbol to the dictionary.
+ *
+ * If the symbol already exists, returns its existing ID.
+ * If the symbol is new, assigns the next sequential ID and returns it.
+ *
+ * @param symbol the symbol string (must not be null)
+ * @return the global ID for this symbol (>= 0)
+ * @throws IllegalArgumentException if symbol is null
+ */
+ public int getOrAddSymbol(String symbol) {
+ if (symbol == null) {
+ throw new IllegalArgumentException("symbol cannot be null");
+ }
+
+ int existingId = symbolToId.get(symbol);
+ if (existingId != CharSequenceIntHashMap.NO_ENTRY_VALUE) {
+ return existingId;
+ }
+
+ // Assign new ID
+ int newId = idToSymbol.size();
+ symbolToId.put(symbol, newId);
+ idToSymbol.add(symbol);
+ return newId;
+ }
+
+ /**
+ * Gets the symbol string for a given ID.
+ *
+ * @param id the symbol ID
+ * @return the symbol string
+ * @throws IndexOutOfBoundsException if id is out of range
+ */
+ public String getSymbol(int id) {
+ if (id < 0 || id >= idToSymbol.size()) {
+ throw new IndexOutOfBoundsException("Invalid symbol ID: " + id + ", dictionary size: " + idToSymbol.size());
+ }
+ return idToSymbol.getQuick(id);
+ }
+
+ /**
+ * Checks if the dictionary is empty.
+ *
+ * @return true if no symbols have been added
+ */
+ public boolean isEmpty() {
+ return idToSymbol.size() == 0;
+ }
+
+ /**
+ * Returns the number of symbols in the dictionary.
+ *
+ * @return dictionary size
+ */
+ public int size() {
+ return idToSymbol.size();
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java
new file mode 100644
index 0000000..6447cb8
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java
@@ -0,0 +1,479 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client;
+
+import io.questdb.client.cutlass.line.LineSenderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.VarHandle;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.LockSupport;
+
+/**
+ * Lock-free in-flight batch tracker for the sliding window protocol.
+ *
+ * Concurrency model (lock-free):
+ *
+ * For sequential batch IDs, this is a cumulative acknowledgment -
+ * acknowledging batch N means all batches up to N are acknowledged.
+ *
+ * Called by: acker (WebSocket I/O thread) after receiving an ACK.
+ *
+ * @param batchId the batch ID that was acknowledged
+ * @return true if the batch was in flight, false if already acknowledged
+ */
+ public boolean acknowledge(long batchId) {
+ return acknowledgeUpTo(batchId) > 0 || highestAcked >= batchId;
+ }
+
+ /**
+ * Acknowledges all batches up to and including the given sequence (cumulative ACK).
+ * Lock-free with single consumer.
+ *
+ * Called by: acker (WebSocket I/O thread) after receiving an ACK.
+ *
+ * @param sequence the highest acknowledged sequence
+ * @return the number of batches acknowledged
+ */
+ public int acknowledgeUpTo(long sequence) {
+ long sent = highestSent;
+
+ // Nothing to acknowledge if window is empty or sequence is beyond what's sent
+ if (sent < 0) {
+ return 0; // No batches have been sent
+ }
+
+ // Cap sequence at highestSent - can't acknowledge what hasn't been sent
+ long effectiveSequence = Math.min(sequence, sent);
+
+ long prevAcked = highestAcked;
+ if (effectiveSequence <= prevAcked) {
+ // Already acknowledged up to this point
+ return 0;
+ }
+ highestAcked = effectiveSequence;
+
+ int acknowledged = (int) (effectiveSequence - prevAcked);
+ TOTAL_ACKED.getAndAdd(this, (long) acknowledged);
+
+ LOG.debug("Cumulative ACK [upTo={}, acknowledged={}, remaining={}]", sequence, acknowledged, getInFlightCount());
+
+ // Wake up waiting threads
+ Thread waiter = waitingForSpace;
+ if (waiter != null) {
+ LockSupport.unpark(waiter);
+ }
+
+ waiter = waitingForEmpty;
+ if (waiter != null && getInFlightCount() == 0) {
+ LockSupport.unpark(waiter);
+ }
+
+ return acknowledged;
+ }
+
+ /**
+ * Adds a batch to the in-flight window.
+ *
+ * Blocks if the window is full until space becomes available or timeout.
+ * Uses spin-wait with exponential backoff, then parks. Blocking is only expected
+ * in modes where another actor can make progress on acknowledgments. In normal
+ * sync usage the window size is 1 and the same thread immediately waits for the
+ * ACK, so this should never actually park. If a caller uses a larger window here
+ * it must ensure ACKs are processed on another thread; a single-threaded caller
+ * with window>1 would deadlock by parking while also being the only thread that
+ * can advance {@link #acknowledgeUpTo(long)}.
+ *
+ * Called by: sync sender thread before sending a batch (window=1).
+ *
+ * @param batchId the batch ID to track
+ * @throws LineSenderException if timeout occurs or an error was reported
+ */
+ public void addInFlight(long batchId) {
+ // Check for errors first
+ checkError();
+
+ // Fast path: try to add without waiting
+ if (tryAddInFlightInternal(batchId)) {
+ return;
+ }
+
+ // Slow path: need to wait for space
+ long deadline = System.currentTimeMillis() + timeoutMs;
+ int spins = 0;
+
+ // Register as waiting thread
+ waitingForSpace = Thread.currentThread();
+ try {
+ while (true) {
+ // Check for errors
+ checkError();
+
+ // Try to add
+ if (tryAddInFlightInternal(batchId)) {
+ return;
+ }
+
+ // Check timeout
+ long remaining = deadline - System.currentTimeMillis();
+ if (remaining <= 0) {
+ throw new LineSenderException("Timeout waiting for window space, window full with " +
+ getInFlightCount() + " batches");
+ }
+
+ // Spin or park
+ if (spins < SPIN_TRIES) {
+ Thread.onSpinWait();
+ spins++;
+ } else {
+ // Park with timeout
+ LockSupport.parkNanos(Math.min(PARK_NANOS, remaining * 1_000_000));
+ if (Thread.interrupted()) {
+ throw new LineSenderException("Interrupted while waiting for window space");
+ }
+ }
+ }
+ } finally {
+ waitingForSpace = null;
+ }
+ }
+
+ /**
+ * Waits until all in-flight batches are acknowledged.
+ *
+ * Called by flush() to ensure all data is confirmed.
+ *
+ * Called by: waiter (flush thread), while producer/acker thread progresses.
+ *
+ * @throws LineSenderException if timeout occurs or an error was reported
+ */
+ public void awaitEmpty() {
+ checkError();
+
+ // Fast path: already empty
+ if (getInFlightCount() == 0) {
+ LOG.debug("Window already empty");
+ return;
+ }
+
+ long deadline = System.currentTimeMillis() + timeoutMs;
+ int spins = 0;
+
+ // Register as waiting thread
+ waitingForEmpty = Thread.currentThread();
+ try {
+ while (getInFlightCount() > 0) {
+ checkError();
+
+ long remaining = deadline - System.currentTimeMillis();
+ if (remaining <= 0) {
+ throw new LineSenderException("Timeout waiting for batch acknowledgments, " +
+ getInFlightCount() + " batches still in flight");
+ }
+
+ if (spins < SPIN_TRIES) {
+ Thread.onSpinWait();
+ spins++;
+ } else {
+ LockSupport.parkNanos(Math.min(PARK_NANOS, remaining * 1_000_000));
+ if (Thread.interrupted()) {
+ throw new LineSenderException("Interrupted while waiting for acknowledgments");
+ }
+ }
+ }
+
+ LOG.debug("Window empty, all batches ACKed");
+ } finally {
+ waitingForEmpty = null;
+ }
+ }
+
+ /**
+ * Clears the error state.
+ */
+ public void clearError() {
+ lastError.set(null);
+ failedBatchId = -1;
+ }
+
+ /**
+ * Marks a batch as failed, setting an error that will be propagated to waiters.
+ *
+ * Called by: acker (WebSocket I/O thread) on error response or send failure.
+ *
+ * @param batchId the batch ID that failed
+ * @param error the error that occurred
+ */
+ public void fail(long batchId, Throwable error) {
+ this.failedBatchId = batchId;
+ this.lastError.set(error);
+ TOTAL_FAILED.getAndAdd(this, 1L);
+
+ LOG.error("Batch failed [batchId={}, error={}]", batchId, String.valueOf(error));
+
+ wakeWaiters();
+ }
+
+ /**
+ * Marks all currently in-flight batches as failed.
+ *
+ * Used for transport-level failures (disconnect/protocol violation) where
+ * no further ACKs are expected and all waiters must be released.
+ *
+ * @param error terminal error to propagate
+ */
+ public void failAll(Throwable error) {
+ long sent = highestSent;
+ long acked = highestAcked;
+ long inFlight = Math.max(0, sent - acked);
+
+ this.failedBatchId = sent;
+ this.lastError.set(error);
+ TOTAL_FAILED.getAndAdd(this, Math.max(1L, inFlight));
+
+ LOG.error("All in-flight batches failed [inFlight={}, error={}]", inFlight, String.valueOf(error));
+
+ wakeWaiters();
+ }
+
+ /**
+ * Returns the current number of batches in flight.
+ * Wait-free operation.
+ */
+ public int getInFlightCount() {
+ long sent = highestSent;
+ long acked = highestAcked;
+ // Ensure non-negative (can happen during initialization)
+ return (int) Math.max(0, sent - acked);
+ }
+
+ /**
+ * Returns the last error, or null if no error.
+ */
+ public Throwable getLastError() {
+ return lastError.get();
+ }
+
+ /**
+ * Returns the maximum window size.
+ */
+ public int getMaxWindowSize() {
+ return maxWindowSize;
+ }
+
+ /**
+ * Returns the total number of batches acknowledged.
+ */
+ public long getTotalAcked() {
+ return (long) TOTAL_ACKED.getOpaque(this);
+ }
+
+ /**
+ * Returns the total number of batches that failed.
+ */
+ public long getTotalFailed() {
+ return (long) TOTAL_FAILED.getOpaque(this);
+ }
+
+ /**
+ * Checks if there's space in the window for another batch.
+ * Wait-free operation.
+ *
+ * @return true if there's space, false if window is full
+ */
+ public boolean hasWindowSpace() {
+ return getInFlightCount() < maxWindowSize;
+ }
+
+ /**
+ * Returns true if the window is empty.
+ * Wait-free operation.
+ */
+ public boolean isEmpty() {
+ return getInFlightCount() == 0;
+ }
+
+ /**
+ * Returns true if the window is full.
+ * Wait-free operation.
+ */
+ public boolean isFull() {
+ return getInFlightCount() >= maxWindowSize;
+ }
+
+ /**
+ * Resets the window, clearing all state.
+ */
+ public void reset() {
+ highestSent = -1;
+ highestAcked = -1;
+ lastError.set(null);
+ failedBatchId = -1;
+
+ wakeWaiters();
+ }
+
+ /**
+ * Tries to add a batch to the in-flight window without blocking.
+ * Lock-free, assuming single producer for highestSent.
+ *
+ * Called by: async producer (WebSocket I/O thread) before sending a batch.
+ *
+ * @param batchId the batch ID to track (must be sequential)
+ * @return true if added, false if window is full
+ */
+ public boolean tryAddInFlight(long batchId) {
+ // Check window space first
+ long sent = highestSent;
+ long acked = highestAcked;
+
+ if (sent - acked >= maxWindowSize) {
+ return false;
+ }
+
+ // Sequential caller: just publish the new highestSent
+ highestSent = batchId;
+
+ LOG.debug("Added to window [batchId={}, windowSize={}]", batchId, getInFlightCount());
+ return true;
+ }
+
+ private void checkError() {
+ Throwable error = lastError.get();
+ if (error != null) {
+ throw new LineSenderException("Batch " + failedBatchId + " failed: " + error.getMessage(), error);
+ }
+ }
+
+ private boolean tryAddInFlightInternal(long batchId) {
+ long sent = highestSent;
+ long acked = highestAcked;
+
+ if (sent - acked >= maxWindowSize) {
+ return false;
+ }
+
+ // For sequential IDs, we just update highestSent
+ // The caller guarantees batchId is the next in sequence
+ highestSent = batchId;
+
+ LOG.debug("Added to window [batchId={}, windowSize={}]", batchId, getInFlightCount());
+ return true;
+ }
+
+ private void wakeWaiters() {
+ Thread waiter = waitingForSpace;
+ if (waiter != null) {
+ LockSupport.unpark(waiter);
+ }
+ waiter = waitingForEmpty;
+ if (waiter != null) {
+ LockSupport.unpark(waiter);
+ }
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java
new file mode 100644
index 0000000..4f5bfbf
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java
@@ -0,0 +1,471 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client;
+
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.QuietCloseable;
+import io.questdb.client.std.Unsafe;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * A buffer for accumulating ILP data into microbatches before sending.
+ *
+ * This class implements a state machine for buffer lifecycle management in the
+ * double-buffering scheme used by {@link QwpWebSocketSender}:
+ *
+ * Thread safety: This class is NOT thread-safe for concurrent writes. However, it
+ * supports safe hand-over between user thread and I/O thread through the state
+ * machine. State transitions use volatile fields to ensure visibility.
+ */
+public class MicrobatchBuffer implements QuietCloseable {
+
+ // Buffer states
+ public static final int STATE_FILLING = 0;
+ public static final int STATE_RECYCLED = 3;
+ public static final int STATE_SEALED = 1;
+ public static final int STATE_SENDING = 2;
+ private static final AtomicLong nextBatchId = new AtomicLong();
+ private final long maxAgeNanos;
+ private final int maxBytes;
+ // Flush trigger thresholds
+ private final int maxRows;
+ // Batch identification
+ private long batchId;
+ private int bufferCapacity;
+ private int bufferPos;
+ // Native memory buffer
+ private long bufferPtr;
+ private long firstRowTimeNanos;
+ // Symbol tracking for delta encoding
+ private int maxSymbolId = -1;
+ // For waiting on recycle (user thread waits for I/O thread to finish)
+ // CountDownLatch is not resettable, so we create a new instance on reset()
+ private volatile CountDownLatch recycleLatch = new CountDownLatch(1);
+ // Row tracking
+ private int rowCount;
+ // State machine
+ private volatile int state = STATE_FILLING;
+
+ /**
+ * Creates a new MicrobatchBuffer with specified flush thresholds.
+ *
+ * @param initialCapacity initial buffer size in bytes
+ * @param maxRows maximum rows before auto-flush (0 = unlimited)
+ * @param maxBytes maximum bytes before auto-flush (0 = unlimited)
+ * @param maxAgeNanos maximum age in nanoseconds before auto-flush (0 = unlimited)
+ */
+ public MicrobatchBuffer(int initialCapacity, int maxRows, int maxBytes, long maxAgeNanos) {
+ if (initialCapacity <= 0) {
+ throw new IllegalArgumentException("initialCapacity must be positive");
+ }
+ this.bufferCapacity = initialCapacity;
+ this.bufferPtr = Unsafe.malloc(initialCapacity, MemoryTag.NATIVE_ILP_RSS);
+ this.bufferPos = 0;
+ this.rowCount = 0;
+ this.firstRowTimeNanos = 0;
+ this.maxRows = maxRows;
+ this.maxBytes = maxBytes;
+ this.maxAgeNanos = maxAgeNanos;
+ this.batchId = nextBatchId.getAndIncrement();
+ }
+
+ /**
+ * Creates a new MicrobatchBuffer with default thresholds (no auto-flush).
+ *
+ * @param initialCapacity initial buffer size in bytes
+ */
+ public MicrobatchBuffer(int initialCapacity) {
+ this(initialCapacity, 0, 0, 0);
+ }
+
+ /**
+ * Returns a human-readable name for the given state.
+ */
+ public static String stateName(int state) {
+ return switch (state) {
+ case STATE_FILLING -> "FILLING";
+ case STATE_SEALED -> "SEALED";
+ case STATE_SENDING -> "SENDING";
+ case STATE_RECYCLED -> "RECYCLED";
+ default -> "UNKNOWN(" + state + ")";
+ };
+ }
+
+ /**
+ * Waits for the buffer to be recycled (transition to RECYCLED state).
+ * Only the user thread should call this.
+ */
+ public void awaitRecycled() {
+ try {
+ recycleLatch.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ /**
+ * Waits for the buffer to be recycled with a timeout.
+ *
+ * @param timeout the maximum time to wait
+ * @param unit the time unit
+ * @return true if recycled, false if timeout elapsed
+ */
+ public boolean awaitRecycled(long timeout, TimeUnit unit) {
+ try {
+ return recycleLatch.await(timeout, unit);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return false;
+ }
+ }
+
+ @Override
+ public void close() {
+ if (bufferPtr != 0) {
+ Unsafe.free(bufferPtr, bufferCapacity, MemoryTag.NATIVE_ILP_RSS);
+ bufferPtr = 0;
+ bufferCapacity = 0;
+ }
+ }
+
+ /**
+ * Ensures the buffer has at least the specified capacity.
+ * Grows the buffer if necessary.
+ *
+ * @param requiredCapacity minimum required capacity
+ */
+ public void ensureCapacity(int requiredCapacity) {
+ if (state != STATE_FILLING) {
+ throw new IllegalStateException("Cannot resize when state is " + stateName(state));
+ }
+ if (requiredCapacity > bufferCapacity) {
+ int newCapacity = (int) Math.min(Math.max((long) bufferCapacity * 2, requiredCapacity), Integer.MAX_VALUE);
+ bufferPtr = Unsafe.realloc(bufferPtr, bufferCapacity, newCapacity, MemoryTag.NATIVE_ILP_RSS);
+ bufferCapacity = newCapacity;
+ }
+ }
+
+ /**
+ * Returns the age of the first row in nanoseconds, or 0 if no rows.
+ */
+ public long getAgeNanos() {
+ if (rowCount == 0) {
+ return 0;
+ }
+ return System.nanoTime() - firstRowTimeNanos;
+ }
+
+ /**
+ * Returns the batch ID for this buffer.
+ */
+ public long getBatchId() {
+ return batchId;
+ }
+
+ /**
+ * Returns the buffer capacity.
+ */
+ public int getBufferCapacity() {
+ return bufferCapacity;
+ }
+
+ /**
+ * Returns the current write position in the buffer.
+ */
+ public int getBufferPos() {
+ return bufferPos;
+ }
+
+ /**
+ * Returns the buffer pointer for writing data.
+ * Only valid when state is FILLING.
+ */
+ public long getBufferPtr() {
+ return bufferPtr;
+ }
+
+ /**
+ * Returns the number of rows in this buffer.
+ */
+ public int getRowCount() {
+ return rowCount;
+ }
+
+ /**
+ * Returns the current state.
+ */
+ public int getState() {
+ return state;
+ }
+
+ /**
+ * Returns true if the buffer has any data.
+ */
+ public boolean hasData() {
+ return bufferPos > 0;
+ }
+
+ /**
+ * Increments the row count and records the first row time if this is the first row.
+ */
+ public void incrementRowCount() {
+ if (state != STATE_FILLING) {
+ throw new IllegalStateException("Cannot increment row count when state is " + stateName(state));
+ }
+ if (rowCount == 0) {
+ firstRowTimeNanos = System.nanoTime();
+ }
+ rowCount++;
+ }
+
+ /**
+ * Checks if the age limit has been exceeded.
+ */
+ public boolean isAgeLimitExceeded() {
+ if (maxAgeNanos <= 0 || rowCount == 0) {
+ return false;
+ }
+ long ageNanos = System.nanoTime() - firstRowTimeNanos;
+ return ageNanos >= maxAgeNanos;
+ }
+
+ /**
+ * Checks if the byte size limit has been exceeded.
+ */
+ public boolean isByteLimitExceeded() {
+ return maxBytes > 0 && bufferPos >= maxBytes;
+ }
+
+ /**
+ * Returns true if the buffer is in FILLING state (available for writing).
+ */
+ public boolean isFilling() {
+ return state == STATE_FILLING;
+ }
+
+ /**
+ * Returns true if the buffer is currently in use (not available for the user thread).
+ */
+ public boolean isInUse() {
+ int s = state;
+ return s == STATE_SEALED || s == STATE_SENDING;
+ }
+
+ /**
+ * Returns true if the buffer is in RECYCLED state (available for reset).
+ */
+ public boolean isRecycled() {
+ return state == STATE_RECYCLED;
+ }
+
+ /**
+ * Checks if the row count limit has been exceeded.
+ */
+ public boolean isRowLimitExceeded() {
+ return maxRows > 0 && rowCount >= maxRows;
+ }
+
+ /**
+ * Returns true if the buffer is in SEALED state (ready to send).
+ */
+ public boolean isSealed() {
+ return state == STATE_SEALED;
+ }
+
+ /**
+ * Returns true if the buffer is in SENDING state (being sent by I/O thread).
+ */
+ public boolean isSending() {
+ return state == STATE_SENDING;
+ }
+
+ /**
+ * Marks the buffer as recycled, transitioning from SENDING to RECYCLED.
+ * This signals to the user thread that the buffer can be reused.
+ * Only the I/O thread should call this.
+ *
+ * @throws IllegalStateException if not in SENDING state
+ */
+ public void markRecycled() {
+ if (state != STATE_SENDING) {
+ throw new IllegalStateException("Cannot mark recycled in state " + stateName(state));
+ }
+ state = STATE_RECYCLED;
+ recycleLatch.countDown();
+ }
+
+ /**
+ * Marks the buffer as being sent, transitioning from SEALED to SENDING.
+ * Only the I/O thread should call this.
+ *
+ * @throws IllegalStateException if not in SEALED state
+ */
+ public void markSending() {
+ if (state != STATE_SEALED) {
+ throw new IllegalStateException("Cannot mark sending in state " + stateName(state));
+ }
+ state = STATE_SENDING;
+ }
+
+ /**
+ * Resets the buffer to FILLING state, clearing all data.
+ * Only valid when in RECYCLED state or when the buffer is fresh.
+ *
+ * @throws IllegalStateException if in SEALED or SENDING state
+ */
+ public void reset() {
+ int s = state;
+ if (s == STATE_SEALED || s == STATE_SENDING) {
+ throw new IllegalStateException("Cannot reset buffer in state " + stateName(s));
+ }
+ bufferPos = 0;
+ rowCount = 0;
+ firstRowTimeNanos = 0;
+ maxSymbolId = -1;
+ batchId = nextBatchId.getAndIncrement();
+ state = STATE_FILLING;
+ recycleLatch = new CountDownLatch(1);
+ }
+
+ /**
+ * Rolls back a seal operation, transitioning from SEALED back to FILLING.
+ *
+ * Used when enqueue fails after a buffer has been sealed but before ownership
+ * was transferred to the I/O thread.
+ *
+ * @throws IllegalStateException if not in SEALED state
+ */
+ public void rollbackSealForRetry() {
+ if (state != STATE_SEALED) {
+ throw new IllegalStateException("Cannot rollback seal in state " + stateName(state));
+ }
+ state = STATE_FILLING;
+ }
+
+ /**
+ * Seals the buffer, transitioning from FILLING to SEALED.
+ * After sealing, no more data can be written.
+ * Only the user thread should call this.
+ *
+ * @throws IllegalStateException if not in FILLING state
+ */
+ public void seal() {
+ if (state != STATE_FILLING) {
+ throw new IllegalStateException("Cannot seal buffer in state " + stateName(state));
+ }
+ state = STATE_SEALED;
+ }
+
+ /**
+ * Sets the buffer position after external writes.
+ * Only valid when state is FILLING.
+ *
+ * @param pos new position
+ */
+ public void setBufferPos(int pos) {
+ if (state != STATE_FILLING) {
+ throw new IllegalStateException("Cannot set position when state is " + stateName(state));
+ }
+ if (pos < 0 || pos > bufferCapacity) {
+ throw new IllegalArgumentException("Position out of bounds: " + pos);
+ }
+ this.bufferPos = pos;
+ }
+
+ /**
+ * Sets the maximum symbol ID used in this batch.
+ * Used for delta symbol dictionary tracking.
+ */
+ public void setMaxSymbolId(int maxSymbolId) {
+ this.maxSymbolId = maxSymbolId;
+ }
+
+ /**
+ * Checks if the buffer should be flushed based on configured thresholds.
+ *
+ * @return true if any flush threshold is exceeded
+ */
+ public boolean shouldFlush() {
+ if (!hasData()) {
+ return false;
+ }
+ return isRowLimitExceeded() || isByteLimitExceeded() || isAgeLimitExceeded();
+ }
+
+ @Override
+ public String toString() {
+ return "MicrobatchBuffer{" +
+ "batchId=" + batchId +
+ ", state=" + stateName(state) +
+ ", rows=" + rowCount +
+ ", bytes=" + bufferPos +
+ ", capacity=" + bufferCapacity +
+ '}';
+ }
+
+ /**
+ * Writes bytes to the buffer at the current position.
+ * Grows the buffer if necessary.
+ *
+ * @param src source address
+ * @param length number of bytes to write
+ */
+ public void write(long src, int length) {
+ if (state != STATE_FILLING) {
+ throw new IllegalStateException("Cannot write when state is " + stateName(state));
+ }
+ ensureCapacity((int) Math.min((long) bufferPos + length, Integer.MAX_VALUE));
+ Unsafe.getUnsafe().copyMemory(src, bufferPtr + bufferPos, length);
+ bufferPos += length;
+ }
+
+ /**
+ * Writes a single byte to the buffer.
+ *
+ * @param b byte to write
+ */
+ public void writeByte(byte b) {
+ if (state != STATE_FILLING) {
+ throw new IllegalStateException("Cannot write when state is " + stateName(state));
+ }
+ ensureCapacity((int) Math.min((long) bufferPos + 1, Integer.MAX_VALUE));
+ Unsafe.getUnsafe().putByte(bufferPtr + bufferPos, b);
+ bufferPos++;
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java
new file mode 100644
index 0000000..e538769
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java
@@ -0,0 +1,306 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client;
+
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.QuietCloseable;
+import io.questdb.client.std.Unsafe;
+
+/**
+ * A simple native memory buffer writer for encoding ILP v4 messages.
+ *
+ * This class provides write methods similar to HttpClient.Request but writes
+ * to a native memory buffer that can be sent over WebSocket.
+ *
+ * All multi-byte values are written in little-endian format unless otherwise specified.
+ */
+public class NativeBufferWriter implements QwpBufferWriter, QuietCloseable {
+
+ private static final int DEFAULT_CAPACITY = 8192;
+
+ private long bufferPtr;
+ private int capacity;
+ private int position;
+
+ public NativeBufferWriter() {
+ this(DEFAULT_CAPACITY);
+ }
+
+ public NativeBufferWriter(int initialCapacity) {
+ this.capacity = initialCapacity;
+ this.bufferPtr = Unsafe.malloc(capacity, MemoryTag.NATIVE_DEFAULT);
+ this.position = 0;
+ }
+
+ /**
+ * Returns the UTF-8 encoded length of a string.
+ *
+ * @param s the string (may be null)
+ * @return the number of bytes needed to encode the string as UTF-8
+ */
+ public static int utf8Length(String s) {
+ if (s == null) return 0;
+ int len = 0;
+ for (int i = 0, n = s.length(); i < n; i++) {
+ char c = s.charAt(i);
+ if (c < 0x80) {
+ len++;
+ } else if (c < 0x800) {
+ len += 2;
+ } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n && Character.isLowSurrogate(s.charAt(i + 1))) {
+ i++;
+ len += 4;
+ } else if (Character.isSurrogate(c)) {
+ len++;
+ } else {
+ len += 3;
+ }
+ }
+ return len;
+ }
+
+ @Override
+ public void close() {
+ if (bufferPtr != 0) {
+ Unsafe.free(bufferPtr, capacity, MemoryTag.NATIVE_DEFAULT);
+ bufferPtr = 0;
+ }
+ }
+
+ /**
+ * Ensures the buffer has at least the specified additional capacity.
+ *
+ * @param needed additional bytes needed beyond current position
+ */
+ @Override
+ public void ensureCapacity(int needed) {
+ if (position + needed > capacity) {
+ int newCapacity = Math.max(capacity * 2, position + needed);
+ bufferPtr = Unsafe.realloc(bufferPtr, capacity, newCapacity, MemoryTag.NATIVE_DEFAULT);
+ capacity = newCapacity;
+ }
+ }
+
+ /**
+ * Returns the buffer pointer.
+ */
+ @Override
+ public long getBufferPtr() {
+ return bufferPtr;
+ }
+
+ /**
+ * Returns the current buffer capacity.
+ */
+ @Override
+ public int getCapacity() {
+ return capacity;
+ }
+
+ /**
+ * Returns the current write position (number of bytes written).
+ */
+ @Override
+ public int getPosition() {
+ return position;
+ }
+
+ /**
+ * Patches an int value at the specified offset.
+ * Used for updating length fields after writing content.
+ */
+ @Override
+ public void patchInt(int offset, int value) {
+ Unsafe.getUnsafe().putInt(bufferPtr + offset, value);
+ }
+
+ /**
+ * Writes a block of bytes from native memory.
+ */
+ @Override
+ public void putBlockOfBytes(long from, long len) {
+ if (len < 0 || len > Integer.MAX_VALUE) {
+ throw new IllegalArgumentException("len exceeds int range: " + len);
+ }
+ int intLen = (int) len;
+ ensureCapacity(intLen);
+ Unsafe.getUnsafe().copyMemory(from, bufferPtr + position, intLen);
+ position += intLen;
+ }
+
+ /**
+ * Writes a single byte.
+ */
+ @Override
+ public void putByte(byte value) {
+ ensureCapacity(1);
+ Unsafe.getUnsafe().putByte(bufferPtr + position, value);
+ position++;
+ }
+
+ /**
+ * Writes a double (8 bytes, little-endian).
+ */
+ @Override
+ public void putDouble(double value) {
+ ensureCapacity(8);
+ Unsafe.getUnsafe().putDouble(bufferPtr + position, value);
+ position += 8;
+ }
+
+ /**
+ * Writes a float (4 bytes, little-endian).
+ */
+ @Override
+ public void putFloat(float value) {
+ ensureCapacity(4);
+ Unsafe.getUnsafe().putFloat(bufferPtr + position, value);
+ position += 4;
+ }
+
+ /**
+ * Writes an int (4 bytes, little-endian).
+ */
+ @Override
+ public void putInt(int value) {
+ ensureCapacity(4);
+ Unsafe.getUnsafe().putInt(bufferPtr + position, value);
+ position += 4;
+ }
+
+ /**
+ * Writes a long (8 bytes, little-endian).
+ */
+ @Override
+ public void putLong(long value) {
+ ensureCapacity(8);
+ Unsafe.getUnsafe().putLong(bufferPtr + position, value);
+ position += 8;
+ }
+
+ /**
+ * Writes a long in big-endian order.
+ */
+ @Override
+ public void putLongBE(long value) {
+ ensureCapacity(8);
+ Unsafe.getUnsafe().putLong(bufferPtr + position, Long.reverseBytes(value));
+ position += 8;
+ }
+
+ /**
+ * Writes a short (2 bytes, little-endian).
+ */
+ @Override
+ public void putShort(short value) {
+ ensureCapacity(2);
+ Unsafe.getUnsafe().putShort(bufferPtr + position, value);
+ position += 2;
+ }
+
+ /**
+ * Writes a length-prefixed UTF-8 string.
+ */
+ @Override
+ public void putString(String value) {
+ if (value == null || value.isEmpty()) {
+ putVarint(0);
+ return;
+ }
+
+ int utf8Len = utf8Length(value);
+ putVarint(utf8Len);
+ putUtf8(value);
+ }
+
+ /**
+ * Writes UTF-8 bytes directly without length prefix.
+ */
+ @Override
+ public void putUtf8(String value) {
+ if (value == null || value.isEmpty()) {
+ return;
+ }
+ for (int i = 0, n = value.length(); i < n; i++) {
+ char c = value.charAt(i);
+ if (c < 0x80) {
+ putByte((byte) c);
+ } else if (c < 0x800) {
+ putByte((byte) (0xC0 | (c >> 6)));
+ putByte((byte) (0x80 | (c & 0x3F)));
+ } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) {
+ char c2 = value.charAt(++i);
+ if (Character.isLowSurrogate(c2)) {
+ int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00);
+ putByte((byte) (0xF0 | (codePoint >> 18)));
+ putByte((byte) (0x80 | ((codePoint >> 12) & 0x3F)));
+ putByte((byte) (0x80 | ((codePoint >> 6) & 0x3F)));
+ putByte((byte) (0x80 | (codePoint & 0x3F)));
+ } else {
+ putByte((byte) '?');
+ i--;
+ }
+ } else if (Character.isSurrogate(c)) {
+ putByte((byte) '?');
+ } else {
+ putByte((byte) (0xE0 | (c >> 12)));
+ putByte((byte) (0x80 | ((c >> 6) & 0x3F)));
+ putByte((byte) (0x80 | (c & 0x3F)));
+ }
+ }
+ }
+
+ /**
+ * Writes a varint (unsigned LEB128).
+ */
+ @Override
+ public void putVarint(long value) {
+ while (value > 0x7F) {
+ putByte((byte) ((value & 0x7F) | 0x80));
+ value >>>= 7;
+ }
+ putByte((byte) value);
+ }
+
+ /**
+ * Resets the buffer for reuse.
+ */
+ @Override
+ public void reset() {
+ position = 0;
+ }
+
+ /**
+ * Skips the specified number of bytes, advancing the position.
+ * Used when data has been written directly to the buffer via getBufferPtr().
+ *
+ * @param bytes number of bytes to skip
+ */
+ @Override
+ public void skip(int bytes) {
+ ensureCapacity(bytes);
+ position += bytes;
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java
new file mode 100644
index 0000000..644fdf8
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java
@@ -0,0 +1,138 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client;
+
+import io.questdb.client.cutlass.line.array.ArrayBufferAppender;
+
+/**
+ * Buffer writer interface for ILP v4 message encoding.
+ *
+ * This interface extends {@link ArrayBufferAppender} with additional methods
+ * required for encoding ILP v4 messages, including varint encoding, string
+ * handling, and buffer manipulation.
+ *
+ * Implementations include:
+ *
+ * All multi-byte values are written in little-endian format unless the method
+ * name explicitly indicates big-endian (e.g., {@link #putLongBE}).
+ */
+public interface QwpBufferWriter extends ArrayBufferAppender {
+
+ /**
+ * Ensures the buffer has capacity for at least the specified
+ * additional bytes beyond the current position.
+ *
+ * @param additionalBytes number of additional bytes needed
+ */
+ void ensureCapacity(int additionalBytes);
+
+ /**
+ * Returns the native memory pointer to the buffer start.
+ *
+ * The returned pointer is valid until the next buffer growth operation.
+ * Use with care and only for reading completed data.
+ */
+ long getBufferPtr();
+
+ /**
+ * Returns the current buffer capacity in bytes.
+ */
+ int getCapacity();
+
+ /**
+ * Returns the current write position (number of bytes written).
+ */
+ int getPosition();
+
+ /**
+ * Patches an int value at the specified offset in the buffer.
+ *
+ * Used for updating length fields after writing content.
+ *
+ * @param offset the byte offset from buffer start
+ * @param value the int value to write
+ */
+ void patchInt(int offset, int value);
+
+ /**
+ * Writes a float (4 bytes, little-endian).
+ */
+ void putFloat(float value);
+
+ /**
+ * Writes a long in big-endian byte order.
+ */
+ void putLongBE(long value);
+
+ /**
+ * Writes a short (2 bytes, little-endian).
+ */
+ void putShort(short value);
+
+ /**
+ * Writes a length-prefixed UTF-8 string.
+ *
+ * Format: varint length + UTF-8 bytes
+ *
+ * @param value the string to write (may be null or empty)
+ */
+ void putString(String value);
+
+ /**
+ * Writes UTF-8 encoded bytes directly without length prefix.
+ *
+ * @param value the string to encode (may be null or empty)
+ */
+ void putUtf8(String value);
+
+ /**
+ * Writes an unsigned variable-length integer (LEB128 encoding).
+ *
+ * Each byte contains 7 bits of data with the high bit indicating
+ * whether more bytes follow.
+ */
+ void putVarint(long value);
+
+ /**
+ * Resets the buffer for reuse, setting the position to 0.
+ *
+ * Does not deallocate memory.
+ */
+ void reset();
+
+ /**
+ * Skips the specified number of bytes, advancing the position.
+ *
+ * Used when data has been written directly to the buffer via
+ * {@link #getBufferPtr()}.
+ *
+ * @param bytes number of bytes to skip
+ */
+ void skip(int bytes);
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java
new file mode 100644
index 0000000..632e4c8
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java
@@ -0,0 +1,477 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client;
+
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef;
+import io.questdb.client.cutlass.qwp.protocol.QwpGorillaEncoder;
+import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer;
+import io.questdb.client.std.QuietCloseable;
+import io.questdb.client.std.Unsafe;
+
+import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*;
+
+/**
+ * Encodes ILP v4 messages for WebSocket transport.
+ *
+ * This encoder reads column data from off-heap {@link io.questdb.client.cutlass.qwp.protocol.OffHeapAppendMemory}
+ * buffers in {@link QwpTableBuffer.ColumnBuffer} and uses bulk {@code putBlockOfBytes} for fixed-width
+ * types where wire format matches native byte order.
+ *
+ * Types that use bulk copy (native byte-order on wire):
+ * BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, DATE, UUID, LONG256
+ *
+ * Types that require element-by-element encoding:
+ * BOOLEAN (bit-packed on wire), TIMESTAMP (Gorilla), DECIMAL64/128/256 (big-endian on wire)
+ */
+public class QwpWebSocketEncoder implements QuietCloseable {
+
+ public static final byte ENCODING_GORILLA = 0x01;
+ public static final byte ENCODING_UNCOMPRESSED = 0x00;
+ private final QwpGorillaEncoder gorillaEncoder = new QwpGorillaEncoder();
+ private NativeBufferWriter buffer;
+ private byte flags;
+
+ public QwpWebSocketEncoder() {
+ this.buffer = new NativeBufferWriter();
+ this.flags = 0;
+ }
+
+ public QwpWebSocketEncoder(int bufferSize) {
+ this.buffer = new NativeBufferWriter(bufferSize);
+ this.flags = 0;
+ }
+
+ @Override
+ public void close() {
+ if (buffer != null) {
+ buffer.close();
+ buffer = null;
+ }
+ }
+
+ public int encode(QwpTableBuffer tableBuffer, boolean useSchemaRef) {
+ buffer.reset();
+ writeHeader(1, 0);
+ int payloadStart = buffer.getPosition();
+ encodeTable(tableBuffer, useSchemaRef, false);
+ int payloadLength = buffer.getPosition() - payloadStart;
+ buffer.patchInt(8, payloadLength);
+ return buffer.getPosition();
+ }
+
+ public int encodeWithDeltaDict(
+ QwpTableBuffer tableBuffer,
+ GlobalSymbolDictionary globalDict,
+ int confirmedMaxId,
+ int batchMaxId,
+ boolean useSchemaRef
+ ) {
+ buffer.reset();
+ int deltaStart = confirmedMaxId + 1;
+ int deltaCount = Math.max(0, batchMaxId - confirmedMaxId);
+ byte savedFlags = flags;
+ flags |= FLAG_DELTA_SYMBOL_DICT;
+ writeHeader(1, 0);
+ int payloadStart = buffer.getPosition();
+ buffer.putVarint(deltaStart);
+ buffer.putVarint(deltaCount);
+ for (int id = deltaStart; id < deltaStart + deltaCount; id++) {
+ String symbol = globalDict.getSymbol(id);
+ buffer.putString(symbol);
+ }
+ encodeTable(tableBuffer, useSchemaRef, true);
+ int payloadLength = buffer.getPosition() - payloadStart;
+ buffer.patchInt(8, payloadLength);
+ flags = savedFlags;
+ return buffer.getPosition();
+ }
+
+ public QwpBufferWriter getBuffer() {
+ return buffer;
+ }
+
+ public boolean isGorillaEnabled() {
+ return (flags & FLAG_GORILLA) != 0;
+ }
+
+ public void setGorillaEnabled(boolean enabled) {
+ if (enabled) {
+ flags |= FLAG_GORILLA;
+ } else {
+ flags &= ~FLAG_GORILLA;
+ }
+ }
+
+ public void writeHeader(int tableCount, int payloadLength) {
+ buffer.putByte((byte) 'Q');
+ buffer.putByte((byte) 'W');
+ buffer.putByte((byte) 'P');
+ buffer.putByte((byte) '1');
+ buffer.putByte(VERSION_1);
+ buffer.putByte(flags);
+ buffer.putShort((short) tableCount);
+ buffer.putInt(payloadLength);
+ }
+
+ private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, int rowCount, boolean useGorilla, boolean useGlobalSymbols) {
+ int valueCount = col.getValueCount();
+ long dataAddr = col.getDataAddress();
+
+ if (colDef.isNullable()) {
+ writeNullBitmap(col, rowCount);
+ }
+
+ switch (col.getType()) {
+ case TYPE_BOOLEAN:
+ writeBooleanColumn(dataAddr, valueCount);
+ break;
+ case TYPE_BYTE:
+ buffer.putBlockOfBytes(dataAddr, valueCount);
+ break;
+ case TYPE_SHORT:
+ case TYPE_CHAR:
+ buffer.putBlockOfBytes(dataAddr, (long) valueCount * 2);
+ break;
+ case TYPE_INT, TYPE_FLOAT:
+ buffer.putBlockOfBytes(dataAddr, (long) valueCount * 4);
+ break;
+ case TYPE_LONG, TYPE_DATE, TYPE_DOUBLE:
+ buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8);
+ break;
+ case TYPE_TIMESTAMP:
+ case TYPE_TIMESTAMP_NANOS:
+ writeTimestampColumn(dataAddr, valueCount, useGorilla);
+ break;
+ case TYPE_GEOHASH:
+ writeGeoHashColumn(dataAddr, valueCount, col.getGeoHashPrecision());
+ break;
+ case TYPE_STRING:
+ case TYPE_VARCHAR:
+ writeStringColumn(col, valueCount);
+ break;
+ case TYPE_SYMBOL:
+ if (useGlobalSymbols) {
+ writeSymbolColumnWithGlobalIds(col, valueCount);
+ } else {
+ writeSymbolColumn(col, valueCount);
+ }
+ break;
+ case TYPE_UUID:
+ // Stored as lo+hi contiguously, matching wire order
+ buffer.putBlockOfBytes(dataAddr, (long) valueCount * 16);
+ break;
+ case TYPE_LONG256:
+ // Stored as 4 contiguous longs per value
+ buffer.putBlockOfBytes(dataAddr, (long) valueCount * 32);
+ break;
+ case TYPE_DOUBLE_ARRAY:
+ writeDoubleArrayColumn(col, valueCount);
+ break;
+ case TYPE_LONG_ARRAY:
+ writeLongArrayColumn(col, valueCount);
+ break;
+ case TYPE_DECIMAL64:
+ writeDecimal64Column(col.getDecimalScale(), dataAddr, valueCount);
+ break;
+ case TYPE_DECIMAL128:
+ writeDecimal128Column(col.getDecimalScale(), dataAddr, valueCount);
+ break;
+ case TYPE_DECIMAL256:
+ writeDecimal256Column(col.getDecimalScale(), dataAddr, valueCount);
+ break;
+ default:
+ throw new LineSenderException("Unknown column type: " + col.getType());
+ }
+ }
+
+ private void encodeTable(QwpTableBuffer tableBuffer, boolean useSchemaRef, boolean useGlobalSymbols) {
+ QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs();
+ int rowCount = tableBuffer.getRowCount();
+
+ if (useSchemaRef) {
+ writeTableHeaderWithSchemaRef(
+ tableBuffer.getTableName(),
+ rowCount,
+ tableBuffer.getSchemaHash(),
+ columnDefs.length
+ );
+ } else {
+ writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs);
+ }
+
+ boolean useGorilla = isGorillaEnabled();
+ for (int i = 0; i < tableBuffer.getColumnCount(); i++) {
+ QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i);
+ QwpColumnDef colDef = columnDefs[i];
+ encodeColumn(col, colDef, rowCount, useGorilla, useGlobalSymbols);
+ }
+ }
+
+ /**
+ * Writes boolean column data (bit-packed on wire).
+ * Reads individual bytes from off-heap and packs into bits.
+ */
+ private void writeBooleanColumn(long addr, int count) {
+ int packedSize = (count + 7) / 8;
+ for (int i = 0; i < packedSize; i++) {
+ byte b = 0;
+ for (int bit = 0; bit < 8; bit++) {
+ int idx = i * 8 + bit;
+ if (idx < count && Unsafe.getUnsafe().getByte(addr + idx) != 0) {
+ b |= (1 << bit);
+ }
+ }
+ buffer.putByte(b);
+ }
+ }
+
+ /**
+ * Writes Decimal128 values in big-endian wire format.
+ * Reads hi/lo pairs from off-heap (stored as hi, lo per value).
+ */
+ private void writeDecimal128Column(byte scale, long addr, int count) {
+ buffer.putByte(scale);
+ for (int i = 0; i < count; i++) {
+ long offset = (long) i * 16;
+ long hi = Unsafe.getUnsafe().getLong(addr + offset);
+ long lo = Unsafe.getUnsafe().getLong(addr + offset + 8);
+ buffer.putLongBE(hi);
+ buffer.putLongBE(lo);
+ }
+ }
+
+ /**
+ * Writes Decimal256 values in big-endian wire format.
+ * Reads hh/hl/lh/ll quads from off-heap (stored contiguously per value).
+ */
+ private void writeDecimal256Column(byte scale, long addr, int count) {
+ buffer.putByte(scale);
+ for (int i = 0; i < count; i++) {
+ long offset = (long) i * 32;
+ buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset));
+ buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset + 8));
+ buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset + 16));
+ buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset + 24));
+ }
+ }
+
+ /**
+ * Writes Decimal64 values in big-endian wire format.
+ * Reads longs from off-heap.
+ */
+ private void writeDecimal64Column(byte scale, long addr, int count) {
+ buffer.putByte(scale);
+ for (int i = 0; i < count; i++) {
+ buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + (long) i * 8));
+ }
+ }
+
+ private void writeDoubleArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) {
+ byte[] dims = col.getArrayDims();
+ int[] shapes = col.getArrayShapes();
+ double[] data = col.getDoubleArrayData();
+
+ int shapeIdx = 0;
+ int dataIdx = 0;
+ for (int row = 0; row < count; row++) {
+ int nDims = dims[row];
+ buffer.putByte((byte) nDims);
+
+ int elemCount = 1;
+ for (int d = 0; d < nDims; d++) {
+ int dimLen = shapes[shapeIdx++];
+ buffer.putInt(dimLen);
+ elemCount = Math.multiplyExact(elemCount, dimLen);
+ }
+
+ for (int e = 0; e < elemCount; e++) {
+ buffer.putDouble(data[dataIdx++]);
+ }
+ }
+ }
+
+ /**
+ * Writes a GeoHash column in variable-width wire format.
+ *
+ * Wire format: [precision varint] [packed values: ceil(precision/8) bytes each]
+ * Values are stored as 8-byte longs in the off-heap buffer but only the
+ * lower ceil(precision/8) bytes are written to the wire.
+ */
+ private void writeGeoHashColumn(long addr, int count, int precision) {
+ if (precision < 1) {
+ // All values are null: use minimum valid precision.
+ // The decoder will skip all values via the null bitmap,
+ // so the precision only needs to be structurally valid.
+ precision = 1;
+ }
+ buffer.putVarint(precision);
+ int valueSize = (precision + 7) / 8;
+ for (int i = 0; i < count; i++) {
+ long value = Unsafe.getUnsafe().getLong(addr + (long) i * 8);
+ for (int b = 0; b < valueSize; b++) {
+ buffer.putByte((byte) (value >>> (b * 8)));
+ }
+ }
+ }
+
+ private void writeLongArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) {
+ byte[] dims = col.getArrayDims();
+ int[] shapes = col.getArrayShapes();
+ long[] data = col.getLongArrayData();
+
+ int shapeIdx = 0;
+ int dataIdx = 0;
+ for (int row = 0; row < count; row++) {
+ int nDims = dims[row];
+ buffer.putByte((byte) nDims);
+
+ int elemCount = 1;
+ for (int d = 0; d < nDims; d++) {
+ int dimLen = shapes[shapeIdx++];
+ buffer.putInt(dimLen);
+ elemCount = Math.multiplyExact(elemCount, dimLen);
+ }
+
+ for (int e = 0; e < elemCount; e++) {
+ buffer.putLong(data[dataIdx++]);
+ }
+ }
+ }
+
+ /**
+ * Writes a null bitmap from off-heap memory.
+ * On little-endian platforms, the byte layout of the long-packed bitmap
+ * in memory matches the wire format, enabling bulk copy.
+ */
+ private void writeNullBitmap(QwpTableBuffer.ColumnBuffer col, int rowCount) {
+ long nullAddr = col.getNullBitmapAddress();
+ if (nullAddr != 0) {
+ int bitmapSize = (rowCount + 7) / 8;
+ buffer.putBlockOfBytes(nullAddr, bitmapSize);
+ } else {
+ // Non-nullable column shouldn't reach here, but write zeros as fallback
+ int bitmapSize = (rowCount + 7) / 8;
+ for (int i = 0; i < bitmapSize; i++) {
+ buffer.putByte((byte) 0);
+ }
+ }
+ }
+
+ private void writeStringColumn(QwpTableBuffer.ColumnBuffer col, int valueCount) {
+ // Offset array: (valueCount + 1) int32 values, pre-built in wire format
+ buffer.putBlockOfBytes(col.getStringOffsetsAddress(), (long) (valueCount + 1) * 4);
+ // UTF-8 data: raw bytes, contiguous
+ buffer.putBlockOfBytes(col.getStringDataAddress(), col.getStringDataSize());
+ }
+
+ /**
+ * Writes a symbol column with dictionary.
+ * Reads local symbol indices from off-heap data buffer.
+ */
+ private void writeSymbolColumn(QwpTableBuffer.ColumnBuffer col, int count) {
+ long dataAddr = col.getDataAddress();
+ String[] dictionary = col.getSymbolDictionary();
+
+ buffer.putVarint(dictionary.length);
+ for (String symbol : dictionary) {
+ buffer.putString(symbol);
+ }
+
+ for (int i = 0; i < count; i++) {
+ int idx = Unsafe.getUnsafe().getInt(dataAddr + (long) i * 4);
+ buffer.putVarint(idx);
+ }
+ }
+
+ /**
+ * Writes a symbol column using global IDs (for delta dictionary mode).
+ * Reads from auxiliary data buffer if available, otherwise falls back to local indices.
+ */
+ private void writeSymbolColumnWithGlobalIds(QwpTableBuffer.ColumnBuffer col, int count) {
+ long auxAddr = col.getAuxDataAddress();
+ if (auxAddr == 0) {
+ // Fall back to local indices
+ long dataAddr = col.getDataAddress();
+ for (int i = 0; i < count; i++) {
+ int idx = Unsafe.getUnsafe().getInt(dataAddr + (long) i * 4);
+ buffer.putVarint(idx);
+ }
+ } else {
+ for (int i = 0; i < count; i++) {
+ int globalId = Unsafe.getUnsafe().getInt(auxAddr + (long) i * 4);
+ buffer.putVarint(globalId);
+ }
+ }
+ }
+
+ private void writeTableHeaderWithSchema(String tableName, int rowCount, QwpColumnDef[] columns) {
+ buffer.putString(tableName);
+ buffer.putVarint(rowCount);
+ buffer.putVarint(columns.length);
+ buffer.putByte(SCHEMA_MODE_FULL);
+ for (QwpColumnDef col : columns) {
+ buffer.putString(col.getName());
+ buffer.putByte(col.getWireTypeCode());
+ }
+ }
+
+ private void writeTableHeaderWithSchemaRef(String tableName, int rowCount, long schemaHash, int columnCount) {
+ buffer.putString(tableName);
+ buffer.putVarint(rowCount);
+ buffer.putVarint(columnCount);
+ buffer.putByte(SCHEMA_MODE_REFERENCE);
+ buffer.putLong(schemaHash);
+ }
+
+ /**
+ * Writes a timestamp column with optional Gorilla compression.
+ * Reads longs directly from off-heap — zero heap allocation.
+ */
+ private void writeTimestampColumn(long addr, int count, boolean useGorilla) {
+ if (useGorilla && count > 2) {
+ if (QwpGorillaEncoder.canUseGorilla(addr, count)) {
+ buffer.putByte(ENCODING_GORILLA);
+ int encodedSize = QwpGorillaEncoder.calculateEncodedSize(addr, count);
+ buffer.ensureCapacity(encodedSize);
+ int bytesWritten = gorillaEncoder.encodeTimestamps(
+ buffer.getBufferPtr() + buffer.getPosition(),
+ buffer.getCapacity() - buffer.getPosition(),
+ addr,
+ count
+ );
+ buffer.skip(bytesWritten);
+ } else {
+ buffer.putByte(ENCODING_UNCOMPRESSED);
+ buffer.putBlockOfBytes(addr, (long) count * 8);
+ }
+ } else {
+ if (useGorilla) {
+ buffer.putByte(ENCODING_UNCOMPRESSED);
+ }
+ buffer.putBlockOfBytes(addr, (long) count * 8);
+ }
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
new file mode 100644
index 0000000..3b77d73
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -0,0 +1,1389 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.http.client.WebSocketClient;
+import io.questdb.client.cutlass.http.client.WebSocketClientFactory;
+import io.questdb.client.cutlass.http.client.WebSocketFrameHandler;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.line.array.DoubleArray;
+import io.questdb.client.cutlass.line.array.LongArray;
+import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer;
+import io.questdb.client.std.CharSequenceObjHashMap;
+import io.questdb.client.std.Chars;
+import io.questdb.client.std.Decimal128;
+import io.questdb.client.std.Decimal256;
+import io.questdb.client.std.Decimal64;
+import io.questdb.client.std.LongHashSet;
+import io.questdb.client.std.ObjList;
+import io.questdb.client.std.bytes.DirectByteSlice;
+import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.concurrent.TimeUnit;
+
+import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*;
+
+
+/**
+ * ILP v4 WebSocket client sender for streaming data to QuestDB.
+ *
+ * This sender uses a double-buffering scheme with asynchronous I/O for high throughput:
+ *
+ * Configuration options:
+ *
+ * Example usage:
+ *
+ * Fast-path API for high-throughput generators
+ *
+ * For maximum throughput, bypass the fluent API to avoid per-row overhead
+ * (no column-name hashmap lookups, no {@code checkNotClosed()}/{@code checkTableSelected()}
+ * per column, direct access to column buffers). Use {@link #getTableBuffer(String)},
+ * {@link #getOrAddGlobalSymbol(String)}, and {@link #incrementPendingRowCount()}:
+ *
+ * This allows unit tests to test sender logic without requiring a real server.
+ * Uses default auto-flush settings.
+ *
+ * @param host server host (not connected)
+ * @param port server port (not connected)
+ * @param inFlightWindowSize window size: 1 for sync behavior, >1 for async
+ * @return unconnected sender
+ */
+ public static QwpWebSocketSender createForTesting(String host, int port, int inFlightWindowSize) {
+ return new QwpWebSocketSender(
+ host, port, false, DEFAULT_BUFFER_SIZE,
+ DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS,
+ inFlightWindowSize
+ );
+ // Note: does NOT call ensureConnected()
+ }
+
+ /**
+ * Creates a sender with custom flow control settings without connecting. For testing only.
+ *
+ * @param host server host (not connected)
+ * @param port server port (not connected)
+ * @param autoFlushRows rows per batch (0 = no limit)
+ * @param autoFlushBytes bytes per batch (0 = no limit)
+ * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit)
+ * @param inFlightWindowSize window size: 1 for sync behavior, >1 for async
+ * @return unconnected sender
+ */
+ public static QwpWebSocketSender createForTesting(
+ String host, int port,
+ int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos,
+ int inFlightWindowSize) {
+ return new QwpWebSocketSender(
+ host, port, false, DEFAULT_BUFFER_SIZE,
+ autoFlushRows, autoFlushBytes, autoFlushIntervalNanos,
+ inFlightWindowSize
+ );
+ // Note: does NOT call ensureConnected()
+ }
+
+ @Override
+ public void at(long timestamp, ChronoUnit unit) {
+ checkNotClosed();
+ checkTableSelected();
+ if (unit == ChronoUnit.NANOS) {
+ atNanos(timestamp);
+ } else {
+ long micros = toMicros(timestamp, unit);
+ atMicros(micros);
+ }
+ }
+
+ @Override
+ public void at(Instant timestamp) {
+ checkNotClosed();
+ checkTableSelected();
+ long micros = timestamp.getEpochSecond() * 1_000_000L + timestamp.getNano() / 1000L;
+ atMicros(micros);
+ }
+
+ @Override
+ public void atNow() {
+ checkNotClosed();
+ checkTableSelected();
+ // Server-assigned timestamp - just send the row without designated timestamp
+ sendRow();
+ }
+
+ @Override
+ public QwpWebSocketSender boolColumn(CharSequence columnName, boolean value) {
+ checkNotClosed();
+ checkTableSelected();
+ QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BOOLEAN, false);
+ col.addBoolean(value);
+ return this;
+ }
+
+ @Override
+ public DirectByteSlice bufferView() {
+ throw new LineSenderException("bufferView() is not supported for WebSocket sender");
+ }
+
+ /**
+ * Adds a BYTE column value to the current row.
+ *
+ * @param columnName the column name
+ * @param value the byte value
+ * @return this sender for method chaining
+ */
+ public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) {
+ checkNotClosed();
+ checkTableSelected();
+ QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BYTE, false);
+ col.addByte(value);
+ return this;
+ }
+
+ @Override
+ public void cancelRow() {
+ checkNotClosed();
+ if (currentTableBuffer != null) {
+ currentTableBuffer.cancelCurrentRow();
+ }
+ }
+
+ /**
+ * Adds a CHAR column value to the current row.
+ *
+ * CHAR is stored as a 2-byte UTF-16 code unit in QuestDB.
+ *
+ * @param columnName the column name
+ * @param value the character value
+ * @return this sender for method chaining
+ */
+ public QwpWebSocketSender charColumn(CharSequence columnName, char value) {
+ checkNotClosed();
+ checkTableSelected();
+ QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_CHAR, false);
+ col.addShort((short) value);
+ return this;
+ }
+
+ @Override
+ public void close() {
+ if (!closed) {
+ closed = true;
+
+ // Flush any remaining data
+ try {
+ if (inFlightWindowSize > 1) {
+ // Async mode (window > 1): flush accumulated rows in table buffers first
+ flushPendingRows();
+
+ if (activeBuffer != null && activeBuffer.hasData()) {
+ sealAndSwapBuffer();
+ }
+ // Wait for all batches to be sent and acknowledged before closing
+ if (sendQueue != null) {
+ sendQueue.flush();
+ }
+ if (inFlightWindow != null) {
+ inFlightWindow.awaitEmpty();
+ }
+ } else {
+ // Sync mode (window=1): flush pending rows synchronously
+ if (pendingRowCount > 0 && client != null && client.isConnected()) {
+ flushSync();
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("Error during close: {}", String.valueOf(e));
+ }
+
+ // Shut down the I/O thread before closing the socket or buffers
+ // it may be using. This must run even if the flush above failed.
+ if (sendQueue != null) {
+ try {
+ sendQueue.close();
+ } catch (Exception e) {
+ LOG.error("Error closing send queue: {}", String.valueOf(e));
+ }
+ }
+
+ // Close buffers (async mode only, window > 1)
+ if (buffer0 != null) {
+ buffer0.close();
+ }
+ if (buffer1 != null) {
+ buffer1.close();
+ }
+
+ if (client != null) {
+ client.close();
+ client = null;
+ }
+ encoder.close();
+ // Close all table buffers to free off-heap column memory
+ ObjList
+ * Response format (little-endian):
+ *
+ * Status codes:
+ *
+ * The sequence number allows correlation with the original request.
+ * Error message is only present when status != 0.
+ */
+public class WebSocketResponse {
+
+ public static final int MAX_ERROR_MESSAGE_LENGTH = 1024;
+ public static final int MIN_ERROR_RESPONSE_SIZE = 11; // status + sequence + error length
+ // Minimum response size: status (1) + sequence (8)
+ public static final int MIN_RESPONSE_SIZE = 9;
+ public static final byte STATUS_INTERNAL_ERROR = (byte) 255;
+ // Status codes
+ public static final byte STATUS_OK = 0;
+ public static final byte STATUS_PARSE_ERROR = 1;
+ public static final byte STATUS_SCHEMA_ERROR = 2;
+ public static final byte STATUS_SECURITY_ERROR = 4;
+ public static final byte STATUS_WRITE_ERROR = 3;
+ private String errorMessage;
+ private long sequence;
+ private byte status;
+
+ public WebSocketResponse() {
+ this.status = STATUS_OK;
+ this.sequence = 0;
+ this.errorMessage = null;
+ }
+
+ /**
+ * Creates an error response.
+ */
+ public static WebSocketResponse error(long sequence, byte status, String errorMessage) {
+ WebSocketResponse response = new WebSocketResponse();
+ response.status = status;
+ response.sequence = sequence;
+ response.errorMessage = errorMessage;
+ return response;
+ }
+
+ /**
+ * Validates binary response framing without allocating.
+ *
+ * Accepted formats:
+ *
+ * This class manages a dedicated I/O thread that handles both:
+ *
+ * Thread safety:
+ *
+ * Backpressure:
+ *
+ * This method:
+ * 1. Stops accepting new batches
+ * 2. Waits for pending batches to be sent
+ * 3. Stops the I/O thread
+ *
+ * Note: This does NOT close the WebSocket channel - that's the caller's responsibility.
+ */
+ @Override
+ public void close() {
+ if (!running) {
+ return;
+ }
+
+ LOG.info("Closing WebSocket send queue [pending={}]", getPendingSize());
+
+ // Signal shutdown
+ shuttingDown = true;
+
+ // Wait for pending batches to be sent
+ long startTime = System.currentTimeMillis();
+ synchronized (processingLock) {
+ while (!isPendingEmpty()) {
+ long elapsed = System.currentTimeMillis() - startTime;
+ if (elapsed >= shutdownTimeoutMs) {
+ LOG.error("Shutdown timeout, {} batches not sent", getPendingSize());
+ break;
+ }
+ try {
+ processingLock.wait(shutdownTimeoutMs - elapsed);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ break;
+ }
+ }
+ }
+
+ // Stop the I/O thread
+ running = false;
+
+ // Wake up I/O thread if it's blocked on processingLock.wait()
+ synchronized (processingLock) {
+ processingLock.notifyAll();
+ }
+ ioThread.interrupt();
+
+ // Wait for I/O thread to finish
+ try {
+ shutdownLatch.await(shutdownTimeoutMs, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+
+ LOG.info("WebSocket send queue closed [totalBatches={}, totalBytes={}]", totalBatchesSent.get(), totalBytesSent.get());
+ }
+
+ /**
+ * Enqueues a sealed buffer for sending.
+ *
+ * The buffer must be in SEALED state. After this method returns successfully,
+ * ownership of the buffer transfers to the send queue.
+ *
+ * @param buffer the sealed buffer to send
+ * @return true if enqueued successfully
+ * @throws LineSenderException if the buffer is not sealed or an error occurred
+ */
+ public boolean enqueue(MicrobatchBuffer buffer) {
+ if (buffer == null) {
+ throw new IllegalArgumentException("buffer cannot be null");
+ }
+ if (!buffer.isSealed()) {
+ throw new LineSenderException("Buffer must be sealed before enqueue, state=" +
+ MicrobatchBuffer.stateName(buffer.getState()));
+ }
+ if (!running || shuttingDown) {
+ throw new LineSenderException("Send queue is not running");
+ }
+
+ // Check for errors from I/O thread
+ checkError();
+
+ final long deadline = System.currentTimeMillis() + enqueueTimeoutMs;
+ synchronized (processingLock) {
+ while (true) {
+ if (!running || shuttingDown) {
+ throw new LineSenderException("Send queue is not running");
+ }
+ checkError();
+
+ if (offerPending(buffer)) {
+ processingLock.notifyAll();
+ break;
+ }
+
+ long remaining = deadline - System.currentTimeMillis();
+ if (remaining <= 0) {
+ throw new LineSenderException("Enqueue timeout after " + enqueueTimeoutMs + "ms");
+ }
+ try {
+ processingLock.wait(Math.min(10, remaining));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new LineSenderException("Interrupted while enqueueing", e);
+ }
+ }
+ }
+
+ LOG.debug("Enqueued batch [id={}, bytes={}, rows={}]", buffer.getBatchId(), buffer.getBufferPos(), buffer.getRowCount());
+ return true;
+ }
+
+ /**
+ * Waits for all pending batches to be sent.
+ *
+ * This method blocks until the queue is empty and all in-flight sends complete.
+ * It does not close the queue - new batches can still be enqueued after flush.
+ *
+ * @throws LineSenderException if an error occurs during flush
+ */
+ public void flush() {
+ checkError();
+
+ long startTime = System.currentTimeMillis();
+
+ // Wait under lock - I/O thread will notify when processingCount decrements
+ synchronized (processingLock) {
+ while (running) {
+ // Atomically check: queue empty AND not processing
+ if (isPendingEmpty() && processingCount.get() == 0) {
+ break; // All done
+ }
+
+ try {
+ processingLock.wait(10);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new LineSenderException("Interrupted while flushing", e);
+ }
+
+ // Check timeout
+ if (System.currentTimeMillis() - startTime > enqueueTimeoutMs) {
+ throw new LineSenderException("Flush timeout after " + enqueueTimeoutMs + "ms, " +
+ "queue=" + getPendingSize() + ", processing=" + processingCount.get());
+ }
+
+ // Check for errors
+ checkError();
+ }
+ }
+
+ // If loop exited because running=false we still need to surface the root cause.
+ checkError();
+
+ LOG.debug("Flush complete");
+ }
+
+ /**
+ * Returns the last error that occurred in the I/O thread, or null if no error.
+ */
+ public Throwable getLastError() {
+ return lastError;
+ }
+
+ /**
+ * Returns the total number of batches sent.
+ */
+ public long getTotalBatchesSent() {
+ return totalBatchesSent.get();
+ }
+
+ /**
+ * Returns the total number of bytes sent.
+ */
+ public long getTotalBytesSent() {
+ return totalBytesSent.get();
+ }
+
+ /**
+ * Checks if an error occurred in the I/O thread and throws if so.
+ */
+ private void checkError() {
+ Throwable error = lastError;
+ if (error != null) {
+ throw new LineSenderException("Error in send queue I/O thread: " + error.getMessage(), error);
+ }
+ }
+
+ /**
+ * Computes the current I/O state based on queue and in-flight status.
+ */
+ private IoState computeState(boolean hasInFlight) {
+ if (!isPendingEmpty()) {
+ return IoState.ACTIVE;
+ } else if (hasInFlight) {
+ return IoState.DRAINING;
+ } else {
+ return IoState.IDLE;
+ }
+ }
+
+ private void failTransport(LineSenderException error) {
+ Throwable rootError = lastError;
+ if (rootError == null) {
+ lastError = error;
+ rootError = error;
+ }
+ running = false;
+ shuttingDown = true;
+ if (inFlightWindow != null) {
+ inFlightWindow.failAll(rootError);
+ }
+ synchronized (processingLock) {
+ MicrobatchBuffer dropped = pollPending();
+ if (dropped != null) {
+ if (dropped.isSealed()) {
+ dropped.markSending();
+ }
+ if (dropped.isSending()) {
+ dropped.markRecycled();
+ }
+ }
+ processingLock.notifyAll();
+ }
+ }
+
+ private int getPendingSize() {
+ return pendingBuffer == null ? 0 : 1;
+ }
+
+ /**
+ * The main I/O loop that handles both sending batches and receiving ACKs.
+ *
+ * Uses a state machine:
+ *
+ * This buffer provides typed append operations (putByte, putShort, etc.) backed by
+ * native memory allocated via {@link Unsafe}. Memory is tracked under
+ * {@link MemoryTag#NATIVE_ILP_RSS} for precise accounting.
+ *
+ * Growth strategy: capacity doubles on each resize via {@link Unsafe#realloc}.
+ */
+public class OffHeapAppendMemory implements QuietCloseable {
+
+ private static final int DEFAULT_INITIAL_CAPACITY = 128;
+ private long appendAddress;
+ private long capacity;
+ private long pageAddress;
+
+ public OffHeapAppendMemory() {
+ this(DEFAULT_INITIAL_CAPACITY);
+ }
+
+ public OffHeapAppendMemory(long initialCapacity) {
+ this.capacity = Math.max(initialCapacity, 8);
+ this.pageAddress = Unsafe.malloc(this.capacity, MemoryTag.NATIVE_ILP_RSS);
+ this.appendAddress = pageAddress;
+ }
+
+ /**
+ * Returns the address at the given byte offset from the start.
+ */
+ public long addressOf(long offset) {
+ return pageAddress + offset;
+ }
+
+ @Override
+ public void close() {
+ if (pageAddress != 0) {
+ Unsafe.free(pageAddress, capacity, MemoryTag.NATIVE_ILP_RSS);
+ pageAddress = 0;
+ appendAddress = 0;
+ capacity = 0;
+ }
+ }
+
+ /**
+ * Returns the append offset (number of bytes written).
+ */
+ public long getAppendOffset() {
+ return appendAddress - pageAddress;
+ }
+
+ /**
+ * Sets the append position to the given byte offset.
+ * Used for truncateTo operations on column buffers.
+ */
+ public void jumpTo(long offset) {
+ assert offset >= 0 && offset <= getAppendOffset();
+ appendAddress = pageAddress + offset;
+ }
+
+ /**
+ * Returns the base address of the buffer.
+ */
+ public long pageAddress() {
+ return pageAddress;
+ }
+
+ public void putBoolean(boolean value) {
+ putByte(value ? (byte) 1 : (byte) 0);
+ }
+
+ public void putByte(byte value) {
+ ensureCapacity(1);
+ Unsafe.getUnsafe().putByte(appendAddress, value);
+ appendAddress++;
+ }
+
+ public void putDouble(double value) {
+ ensureCapacity(8);
+ Unsafe.getUnsafe().putDouble(appendAddress, value);
+ appendAddress += 8;
+ }
+
+ public void putFloat(float value) {
+ ensureCapacity(4);
+ Unsafe.getUnsafe().putFloat(appendAddress, value);
+ appendAddress += 4;
+ }
+
+ public void putInt(int value) {
+ ensureCapacity(4);
+ Unsafe.getUnsafe().putInt(appendAddress, value);
+ appendAddress += 4;
+ }
+
+ public void putLong(long value) {
+ ensureCapacity(8);
+ Unsafe.getUnsafe().putLong(appendAddress, value);
+ appendAddress += 8;
+ }
+
+ public void putShort(short value) {
+ ensureCapacity(2);
+ Unsafe.getUnsafe().putShort(appendAddress, value);
+ appendAddress += 2;
+ }
+
+ /**
+ * Encodes a Java String to UTF-8 directly into the off-heap buffer.
+ * Pre-ensures worst-case capacity to avoid per-byte checks.
+ */
+ public void putUtf8(String value) {
+ if (value == null || value.isEmpty()) {
+ return;
+ }
+ int len = value.length();
+ ensureCapacity((long) len * 4); // worst case: all supplementary chars
+ for (int i = 0; i < len; i++) {
+ char c = value.charAt(i);
+ if (c < 0x80) {
+ Unsafe.getUnsafe().putByte(appendAddress++, (byte) c);
+ } else if (c < 0x800) {
+ Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0xC0 | (c >> 6)));
+ Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | (c & 0x3F)));
+ } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < len) {
+ char c2 = value.charAt(++i);
+ if (Character.isLowSurrogate(c2)) {
+ int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00);
+ Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0xF0 | (codePoint >> 18)));
+ Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((codePoint >> 12) & 0x3F)));
+ Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((codePoint >> 6) & 0x3F)));
+ Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | (codePoint & 0x3F)));
+ } else {
+ Unsafe.getUnsafe().putByte(appendAddress++, (byte) '?');
+ i--;
+ }
+ } else if (Character.isSurrogate(c)) {
+ Unsafe.getUnsafe().putByte(appendAddress++, (byte) '?');
+ } else {
+ Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0xE0 | (c >> 12)));
+ Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((c >> 6) & 0x3F)));
+ Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | (c & 0x3F)));
+ }
+ }
+ }
+
+ /**
+ * Advances the append position by the given number of bytes without writing.
+ */
+ public void skip(long bytes) {
+ ensureCapacity(bytes);
+ appendAddress += bytes;
+ }
+
+ /**
+ * Resets the append position to 0 without freeing memory.
+ */
+ public void truncate() {
+ appendAddress = pageAddress;
+ }
+
+ private void ensureCapacity(long needed) {
+ long used = appendAddress - pageAddress;
+ if (used + needed > capacity) {
+ long newCapacity = Math.max(capacity * 2, used + needed);
+ pageAddress = Unsafe.realloc(pageAddress, capacity, newCapacity, MemoryTag.NATIVE_ILP_RSS);
+ capacity = newCapacity;
+ appendAddress = pageAddress + used;
+ }
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java
new file mode 100644
index 0000000..fbe6efd
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java
@@ -0,0 +1,235 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.protocol;
+
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.std.Unsafe;
+
+/**
+ * Bit-level writer for ILP v4 protocol.
+ *
+ * This class writes bits to a buffer in LSB-first order within each byte.
+ * Bits are packed sequentially, spanning byte boundaries as needed.
+ *
+ * The implementation buffers up to 64 bits before flushing to the output buffer
+ * to minimize memory operations. All writes are to direct memory for performance.
+ *
+ * Usage pattern:
+ *
+ * This method flushes any remaining bits and returns the total byte count.
+ *
+ * @return bytes written since reset
+ */
+ public int finish() {
+ flush();
+ return (int) (currentAddress - startAddress);
+ }
+
+ /**
+ * Flushes any remaining bits in the buffer to memory.
+ *
+ * If there are partial bits (less than 8), they are written as the last byte
+ * with the remaining high bits set to zero.
+ *
+ * Must be called before reading the output or getting the final position.
+ */
+ public void flush() {
+ if (bitsInBuffer > 0) {
+ if (currentAddress >= endAddress) {
+ throw new LineSenderException("QwpBitWriter buffer overflow");
+ }
+ Unsafe.getUnsafe().putByte(currentAddress++, (byte) bitBuffer);
+ bitBuffer = 0;
+ bitsInBuffer = 0;
+ }
+ }
+
+ /**
+ * Returns the current write position (address).
+ * Note: Call {@link #flush()} first to ensure all buffered bits are written.
+ *
+ * @return the current address after all written data
+ */
+ public long getPosition() {
+ return currentAddress;
+ }
+
+ /**
+ * Resets the writer to write to the specified memory region.
+ *
+ * @param address the starting address
+ * @param capacity the maximum number of bytes to write
+ */
+ public void reset(long address, long capacity) {
+ this.startAddress = address;
+ this.currentAddress = address;
+ this.endAddress = address + capacity;
+ this.bitBuffer = 0;
+ this.bitsInBuffer = 0;
+ }
+
+ /**
+ * Writes a single bit.
+ *
+ * @param bit the bit value (0 or 1, only LSB is used)
+ */
+ public void writeBit(int bit) {
+ writeBits(bit & 1, 1);
+ }
+
+ /**
+ * Writes multiple bits from the given value.
+ *
+ * Bits are taken from the LSB of the value. For example, if value=0b1101
+ * and numBits=4, the bits written are 1, 0, 1, 1 (LSB to MSB order).
+ *
+ * @param value the value containing the bits (LSB-aligned)
+ * @param numBits number of bits to write (1-64)
+ */
+ public void writeBits(long value, int numBits) {
+ if (numBits <= 0 || numBits > 64) {
+ throw new AssertionError("Asked to write more than 64 bits of a long");
+ }
+
+ // Mask the value to only include the requested bits
+ if (numBits < 64) {
+ value &= (1L << numBits) - 1;
+ }
+
+ int bitsToWrite = numBits;
+
+ while (bitsToWrite > 0) {
+ // How many bits can fit into the current buffer (max 64 total)
+ int availableInBuffer = 64 - bitsInBuffer;
+ int bitsThisRound = Math.min(bitsToWrite, availableInBuffer);
+
+ // Add bits to the buffer
+ long mask = bitsThisRound == 64 ? -1L : (1L << bitsThisRound) - 1;
+ bitBuffer |= (value & mask) << bitsInBuffer;
+ bitsInBuffer += bitsThisRound;
+ value >>>= bitsThisRound;
+ bitsToWrite -= bitsThisRound;
+
+ // Flush complete bytes from the buffer
+ while (bitsInBuffer >= 8) {
+ if (currentAddress >= endAddress) {
+ throw new LineSenderException("QwpBitWriter buffer overflow");
+ }
+ Unsafe.getUnsafe().putByte(currentAddress++, (byte) bitBuffer);
+ bitBuffer >>>= 8;
+ bitsInBuffer -= 8;
+ }
+ }
+ }
+
+ /**
+ * Writes a complete byte, ensuring byte alignment first.
+ *
+ * @param value the byte value
+ */
+ public void writeByte(int value) {
+ alignToByte();
+ if (currentAddress >= endAddress) {
+ throw new LineSenderException("QwpBitWriter buffer overflow");
+ }
+ Unsafe.getUnsafe().putByte(currentAddress++, (byte) value);
+ }
+
+ /**
+ * Writes a complete 32-bit integer in little-endian order, ensuring byte alignment first.
+ *
+ * @param value the integer value
+ */
+ public void writeInt(int value) {
+ alignToByte();
+ if (currentAddress + 4 > endAddress) {
+ throw new LineSenderException("QwpBitWriter buffer overflow");
+ }
+ Unsafe.getUnsafe().putInt(currentAddress, value);
+ currentAddress += 4;
+ }
+
+ /**
+ * Writes a complete 64-bit long in little-endian order, ensuring byte alignment first.
+ *
+ * @param value the long value
+ */
+ public void writeLong(long value) {
+ alignToByte();
+ if (currentAddress + 8 > endAddress) {
+ throw new LineSenderException("QwpBitWriter buffer overflow");
+ }
+ Unsafe.getUnsafe().putLong(currentAddress, value);
+ currentAddress += 8;
+ }
+
+ /**
+ * Writes a signed value using two's complement representation.
+ *
+ * @param value the signed value
+ * @param numBits number of bits to use for the representation
+ */
+ public void writeSigned(long value, int numBits) {
+ // Two's complement is automatic in Java for the bit pattern
+ writeBits(value, numBits);
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java
new file mode 100644
index 0000000..a2f4f50
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java
@@ -0,0 +1,148 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.protocol;
+
+import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_BOOLEAN;
+import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_CHAR;
+
+/**
+ * Represents a column definition in an ILP v4 schema.
+ *
+ * This class is immutable and safe for caching.
+ */
+public final class QwpColumnDef {
+ private final String name;
+ private final boolean nullable;
+ private final byte typeCode;
+
+ /**
+ * Creates a column definition.
+ *
+ * @param name the column name (UTF-8)
+ * @param typeCode the ILP v4 type code (0x01-0x0F, optionally OR'd with 0x80 for nullable)
+ */
+ public QwpColumnDef(String name, byte typeCode) {
+ this.name = name;
+ // Extract nullable flag (high bit) and base type
+ this.nullable = (typeCode & 0x80) != 0;
+ this.typeCode = (byte) (typeCode & 0x7F);
+ }
+
+ /**
+ * Creates a column definition with explicit nullable flag.
+ *
+ * @param name the column name
+ * @param typeCode the base type code (0x01-0x0F)
+ * @param nullable whether the column is nullable
+ */
+ public QwpColumnDef(String name, byte typeCode, boolean nullable) {
+ this.name = name;
+ this.typeCode = (byte) (typeCode & 0x7F);
+ this.nullable = nullable;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ QwpColumnDef that = (QwpColumnDef) o;
+ return typeCode == that.typeCode &&
+ nullable == that.nullable &&
+ name.equals(that.name);
+ }
+
+ /**
+ * Gets the column name.
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Gets the base type code (without nullable flag).
+ *
+ * @return type code 0x01-0x0F
+ */
+ public byte getTypeCode() {
+ return typeCode;
+ }
+
+ /**
+ * Gets the type name for display purposes.
+ */
+ public String getTypeName() {
+ return QwpConstants.getTypeName(typeCode);
+ }
+
+ /**
+ * Gets the wire type code (with nullable flag if applicable).
+ *
+ * @return type code as sent on wire
+ */
+ public byte getWireTypeCode() {
+ return nullable ? (byte) (typeCode | 0x80) : typeCode;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = name.hashCode();
+ result = 31 * result + typeCode;
+ result = 31 * result + (nullable ? 1 : 0);
+ return result;
+ }
+
+ /**
+ * Returns true if this column is nullable.
+ */
+ public boolean isNullable() {
+ return nullable;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(name).append(':').append(getTypeName());
+ if (nullable) {
+ sb.append('?');
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Validates that this column definition has a valid type code.
+ *
+ * @throws IllegalArgumentException if type code is invalid
+ */
+ public void validate() {
+ // Valid type codes: TYPE_BOOLEAN (0x01) through TYPE_CHAR (0x16)
+ // This includes all basic types, arrays, decimals, and char
+ boolean valid = (typeCode >= TYPE_BOOLEAN && typeCode <= TYPE_CHAR);
+ if (!valid) {
+ throw new IllegalArgumentException(
+ "invalid column type code: 0x" + Integer.toHexString(typeCode)
+ );
+ }
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java
new file mode 100644
index 0000000..216f2fa
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java
@@ -0,0 +1,348 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.protocol;
+
+/**
+ * Constants for the ILP v4 binary protocol.
+ */
+public final class QwpConstants {
+
+ /**
+ * Default initial receive buffer size (64 KB).
+ */
+ public static final int DEFAULT_INITIAL_RECV_BUFFER_SIZE = 64 * 1024;
+ /**
+ * Default maximum batch size in bytes (16 MB).
+ */
+ public static final int DEFAULT_MAX_BATCH_SIZE = 16 * 1024 * 1024;
+
+ /**
+ * Maximum in-flight batches for pipelining.
+ */
+ public static final int DEFAULT_MAX_IN_FLIGHT_BATCHES = 4;
+ /**
+ * Default maximum rows per table in a batch.
+ */
+ public static final int DEFAULT_MAX_ROWS_PER_TABLE = 1_000_000;
+ /**
+ * Default maximum tables per batch.
+ */
+ public static final int DEFAULT_MAX_TABLES_PER_BATCH = 256;
+ /**
+ * Flag bit: Delta symbol dictionary encoding enabled.
+ * When set, symbol columns use global IDs and send only new dictionary entries.
+ */
+ public static final byte FLAG_DELTA_SYMBOL_DICT = 0x08;
+ /**
+ * Flag bit: Gorilla timestamp encoding enabled.
+ */
+ public static final byte FLAG_GORILLA = 0x04;
+
+ /**
+ * Flag bit: LZ4 compression enabled.
+ */
+ public static final byte FLAG_LZ4 = 0x01;
+
+ /**
+ * Flag bit: Zstd compression enabled.
+ */
+ public static final byte FLAG_ZSTD = 0x02;
+ /**
+ * Mask for compression flags (bits 0-1).
+ */
+ public static final byte FLAG_COMPRESSION_MASK = FLAG_LZ4 | FLAG_ZSTD;
+ /**
+ * Offset of flags byte in header.
+ */
+ public static final int HEADER_OFFSET_FLAGS = 5;
+ /**
+ * Size of the message header in bytes.
+ */
+ public static final int HEADER_SIZE = 12;
+ /**
+ * Magic bytes for capability request: "ILP?" (ASCII).
+ */
+ public static final int MAGIC_CAPABILITY_REQUEST = 0x3F504C49; // "ILP?" in little-endian
+ /**
+ * Magic bytes for capability response: "ILP!" (ASCII).
+ */
+ public static final int MAGIC_CAPABILITY_RESPONSE = 0x21504C49; // "ILP!" in little-endian
+ /**
+ * Magic bytes for fallback response (old server): "ILP0" (ASCII).
+ */
+ public static final int MAGIC_FALLBACK = 0x30504C49; // "ILP0" in little-endian
+ /**
+ * Magic bytes for QWP v1 message: "QWP1" (ASCII).
+ */
+ public static final int MAGIC_MESSAGE = 0x31505751; // "QWP1" in little-endian
+ /**
+ * Maximum columns per table (QuestDB limit).
+ */
+ public static final int MAX_COLUMNS_PER_TABLE = 2048;
+ /**
+ * Schema mode: Full schema included.
+ */
+ public static final byte SCHEMA_MODE_FULL = 0x00;
+ /**
+ * Schema mode: Schema reference (hash lookup).
+ */
+ public static final byte SCHEMA_MODE_REFERENCE = 0x01;
+ /**
+ * Status: Server error.
+ */
+ public static final byte STATUS_INTERNAL_ERROR = 0x06;
+ /**
+ * Status: Batch accepted successfully.
+ */
+ public static final byte STATUS_OK = 0x00;
+ /**
+ * Status: Back-pressure, retry later.
+ */
+ public static final byte STATUS_OVERLOADED = 0x07;
+ /**
+ * Status: Malformed message.
+ */
+ public static final byte STATUS_PARSE_ERROR = 0x05;
+ /**
+ * Status: Some rows failed (partial failure).
+ */
+ public static final byte STATUS_PARTIAL = 0x01;
+ /**
+ * Status: Column type incompatible.
+ */
+ public static final byte STATUS_SCHEMA_MISMATCH = 0x03;
+ /**
+ * Status: Schema hash not recognized.
+ */
+ public static final byte STATUS_SCHEMA_REQUIRED = 0x02;
+ /**
+ * Status: Table doesn't exist (auto-create disabled).
+ */
+ public static final byte STATUS_TABLE_NOT_FOUND = 0x04;
+ /**
+ * Column type: BOOLEAN (1 bit per value, packed).
+ */
+ public static final byte TYPE_BOOLEAN = 0x01;
+ /**
+ * Column type: BYTE (int8).
+ */
+ public static final byte TYPE_BYTE = 0x02;
+ /**
+ * Column type: CHAR (2-byte UTF-16 code unit).
+ */
+ public static final byte TYPE_CHAR = 0x16;
+ /**
+ * Column type: DATE (int64 milliseconds since epoch).
+ */
+ public static final byte TYPE_DATE = 0x0B;
+
+ /**
+ * Column type: DECIMAL128 (16 bytes, 38 digits precision).
+ * Wire format: [scale (1B in schema)] + [big-endian unscaled value (16B)]
+ */
+ public static final byte TYPE_DECIMAL128 = 0x14;
+ /**
+ * Column type: DECIMAL256 (32 bytes, 77 digits precision).
+ * Wire format: [scale (1B in schema)] + [big-endian unscaled value (32B)]
+ */
+ public static final byte TYPE_DECIMAL256 = 0x15;
+
+ /**
+ * Column type: DECIMAL64 (8 bytes, 18 digits precision).
+ * Wire format: [scale (1B in schema)] + [big-endian unscaled value (8B)]
+ */
+ public static final byte TYPE_DECIMAL64 = 0x13;
+ /**
+ * Column type: DOUBLE (IEEE 754 float64).
+ */
+ public static final byte TYPE_DOUBLE = 0x07;
+ /**
+ * Column type: DOUBLE_ARRAY (N-dimensional array of IEEE 754 float64).
+ * Wire format: [nDims (1B)] [dim1_len (4B)]...[dimN_len (4B)] [flattened values (LE)]
+ */
+ public static final byte TYPE_DOUBLE_ARRAY = 0x11;
+ /**
+ * Column type: FLOAT (IEEE 754 float32).
+ */
+ public static final byte TYPE_FLOAT = 0x06;
+ /**
+ * Column type: GEOHASH (varint bits + packed geohash).
+ */
+ public static final byte TYPE_GEOHASH = 0x0E;
+ /**
+ * Column type: INT (int32, little-endian).
+ */
+ public static final byte TYPE_INT = 0x04;
+ /**
+ * Column type: LONG (int64, little-endian).
+ */
+ public static final byte TYPE_LONG = 0x05;
+ /**
+ * Column type: LONG256 (32 bytes, big-endian).
+ */
+ public static final byte TYPE_LONG256 = 0x0D;
+
+ /**
+ * Column type: LONG_ARRAY (N-dimensional array of int64).
+ * Wire format: [nDims (1B)] [dim1_len (4B)]...[dimN_len (4B)] [flattened values (LE)]
+ */
+ public static final byte TYPE_LONG_ARRAY = 0x12;
+ /**
+ * Mask for type code without nullable flag.
+ */
+ public static final byte TYPE_MASK = 0x7F;
+ /**
+ * High bit indicating nullable column.
+ */
+ public static final byte TYPE_NULLABLE_FLAG = (byte) 0x80;
+ /**
+ * Column type: SHORT (int16, little-endian).
+ */
+ public static final byte TYPE_SHORT = 0x03;
+ /**
+ * Column type: STRING (length-prefixed UTF-8).
+ */
+ public static final byte TYPE_STRING = 0x08;
+ /**
+ * Column type: SYMBOL (dictionary-encoded string).
+ */
+ public static final byte TYPE_SYMBOL = 0x09;
+ /**
+ * Column type: TIMESTAMP (int64 microseconds since epoch).
+ * Use this for timestamps beyond nanosecond range (year > 2262).
+ */
+ public static final byte TYPE_TIMESTAMP = 0x0A;
+ /**
+ * Column type: TIMESTAMP_NANOS (int64 nanoseconds since epoch).
+ * Use this for full nanosecond precision (limited to years 1677-2262).
+ */
+ public static final byte TYPE_TIMESTAMP_NANOS = 0x10;
+ /**
+ * Column type: UUID (16 bytes, big-endian).
+ */
+ public static final byte TYPE_UUID = 0x0C;
+
+ /**
+ * Column type: VARCHAR (length-prefixed UTF-8, aux storage).
+ */
+ public static final byte TYPE_VARCHAR = 0x0F;
+ /**
+ * Current protocol version.
+ */
+ public static final byte VERSION_1 = 1;
+
+ private QwpConstants() {
+ // utility class
+ }
+
+ /**
+ * Returns the per-value size in bytes as encoded on the wire. BOOLEAN returns 0
+ * because it is bit-packed (1 bit per value). GEOHASH returns -1 because it uses
+ * variable-width encoding (varint precision + ceil(precision/8) bytes per value).
+ *
+ * This is distinct from the in-memory buffer stride used by the client's
+ * {@code QwpTableBuffer.elementSizeInBuffer()}.
+ *
+ * @param typeCode the column type code (without nullable flag)
+ * @return size in bytes, 0 for bit-packed (BOOLEAN), or -1 for variable-width types
+ */
+ public static int getFixedTypeSize(byte typeCode) {
+ int code = typeCode & TYPE_MASK;
+ return switch (code) {
+ case TYPE_BOOLEAN -> 0; // Special: bit-packed
+ case TYPE_BYTE -> 1;
+ case TYPE_SHORT, TYPE_CHAR -> 2;
+ case TYPE_INT, TYPE_FLOAT -> 4;
+ case TYPE_LONG, TYPE_DOUBLE, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS, TYPE_DATE, TYPE_DECIMAL64 -> 8;
+ case TYPE_UUID, TYPE_DECIMAL128 -> 16;
+ case TYPE_LONG256, TYPE_DECIMAL256 -> 32;
+ case TYPE_GEOHASH -> -1; // Variable width: varint precision + packed values
+ default -> -1; // Variable width
+ };
+ }
+
+ /**
+ * Returns a human-readable name for the type code.
+ *
+ * @param typeCode the column type code
+ * @return type name
+ */
+ public static String getTypeName(byte typeCode) {
+ int code = typeCode & TYPE_MASK;
+ boolean nullable = (typeCode & TYPE_NULLABLE_FLAG) != 0;
+ String name = switch (code) {
+ case TYPE_BOOLEAN -> "BOOLEAN";
+ case TYPE_BYTE -> "BYTE";
+ case TYPE_SHORT -> "SHORT";
+ case TYPE_CHAR -> "CHAR";
+ case TYPE_INT -> "INT";
+ case TYPE_LONG -> "LONG";
+ case TYPE_FLOAT -> "FLOAT";
+ case TYPE_DOUBLE -> "DOUBLE";
+ case TYPE_STRING -> "STRING";
+ case TYPE_SYMBOL -> "SYMBOL";
+ case TYPE_TIMESTAMP -> "TIMESTAMP";
+ case TYPE_TIMESTAMP_NANOS -> "TIMESTAMP_NANOS";
+ case TYPE_DATE -> "DATE";
+ case TYPE_UUID -> "UUID";
+ case TYPE_LONG256 -> "LONG256";
+ case TYPE_GEOHASH -> "GEOHASH";
+ case TYPE_VARCHAR -> "VARCHAR";
+ case TYPE_DOUBLE_ARRAY -> "DOUBLE_ARRAY";
+ case TYPE_LONG_ARRAY -> "LONG_ARRAY";
+ case TYPE_DECIMAL64 -> "DECIMAL64";
+ case TYPE_DECIMAL128 -> "DECIMAL128";
+ case TYPE_DECIMAL256 -> "DECIMAL256";
+ default -> "UNKNOWN(" + code + ")";
+ };
+ return nullable ? name + "?" : name;
+ }
+
+ /**
+ * Returns true if the type code represents a fixed-width type.
+ *
+ * @param typeCode the column type code (without nullable flag)
+ * @return true if fixed-width
+ */
+ public static boolean isFixedWidthType(byte typeCode) {
+ int code = typeCode & TYPE_MASK;
+ return code == TYPE_BOOLEAN ||
+ code == TYPE_BYTE ||
+ code == TYPE_SHORT ||
+ code == TYPE_CHAR ||
+ code == TYPE_INT ||
+ code == TYPE_LONG ||
+ code == TYPE_FLOAT ||
+ code == TYPE_DOUBLE ||
+ code == TYPE_TIMESTAMP ||
+ code == TYPE_TIMESTAMP_NANOS ||
+ code == TYPE_DATE ||
+ code == TYPE_UUID ||
+ code == TYPE_LONG256 ||
+ code == TYPE_DECIMAL64 ||
+ code == TYPE_DECIMAL128 ||
+ code == TYPE_DECIMAL256;
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java
new file mode 100644
index 0000000..302299c
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java
@@ -0,0 +1,288 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.protocol;
+
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.std.Unsafe;
+
+/**
+ * Gorilla delta-of-delta encoder for timestamps in ILP v4 format.
+ *
+ * This encoder is used by the WebSocket encoder to compress timestamp columns.
+ * It uses delta-of-delta compression where:
+ *
+ * The encoder writes first two timestamps uncompressed, then encodes
+ * remaining timestamps using delta-of-delta compression.
+ */
+public class QwpGorillaEncoder {
+
+ private static final int BUCKET_12BIT_MAX = 2047;
+ private static final int BUCKET_12BIT_MIN = -2048;
+ private static final int BUCKET_7BIT_MAX = 63;
+ // Bucket boundaries (two's complement signed ranges)
+ private static final int BUCKET_7BIT_MIN = -64;
+ private static final int BUCKET_9BIT_MAX = 255;
+ private static final int BUCKET_9BIT_MIN = -256;
+ private final QwpBitWriter bitWriter = new QwpBitWriter();
+
+ /**
+ * Creates a new Gorilla encoder.
+ */
+ public QwpGorillaEncoder() {
+ }
+
+ /**
+ * Calculates the encoded size in bytes for Gorilla-encoded timestamps stored off-heap.
+ *
+ * Note: This does NOT include the encoding flag byte. Add 1 byte if
+ * the encoding flag is needed.
+ *
+ * @param srcAddress source address of contiguous int64 timestamps in native memory
+ * @param count number of timestamps
+ * @return encoded size in bytes (excluding encoding flag)
+ */
+ public static int calculateEncodedSize(long srcAddress, int count) {
+ if (count == 0) {
+ return 0;
+ }
+
+ int size = 8; // first timestamp
+
+ if (count == 1) {
+ return size;
+ }
+
+ size += 8; // second timestamp
+
+ if (count == 2) {
+ return size;
+ }
+
+ // Calculate bits for delta-of-delta encoding
+ long prevTimestamp = Unsafe.getUnsafe().getLong(srcAddress + 8);
+ long prevDelta = prevTimestamp - Unsafe.getUnsafe().getLong(srcAddress);
+ int totalBits = 0;
+
+ for (int i = 2; i < count; i++) {
+ long ts = Unsafe.getUnsafe().getLong(srcAddress + (long) i * 8);
+ long delta = ts - prevTimestamp;
+ long deltaOfDelta = delta - prevDelta;
+
+ totalBits += getBitsRequired(deltaOfDelta);
+
+ prevDelta = delta;
+ prevTimestamp = ts;
+ }
+
+ // Round up to bytes
+ size += (totalBits + 7) / 8;
+
+ return size;
+ }
+
+ /**
+ * Checks if Gorilla encoding can be used for timestamps stored off-heap.
+ *
+ * Gorilla encoding uses 32-bit signed integers for delta-of-delta values,
+ * so it cannot encode timestamps where the delta-of-delta exceeds the
+ * 32-bit signed integer range.
+ *
+ * @param srcAddress source address of contiguous int64 timestamps in native memory
+ * @param count number of timestamps
+ * @return true if Gorilla encoding can be used, false otherwise
+ */
+ public static boolean canUseGorilla(long srcAddress, int count) {
+ if (count < 3) {
+ return true; // No DoD encoding needed for 0, 1, or 2 timestamps
+ }
+
+ long prevDelta = Unsafe.getUnsafe().getLong(srcAddress + 8) - Unsafe.getUnsafe().getLong(srcAddress);
+ for (int i = 2; i < count; i++) {
+ long delta = Unsafe.getUnsafe().getLong(srcAddress + (long) i * 8)
+ - Unsafe.getUnsafe().getLong(srcAddress + (long) (i - 1) * 8);
+ long dod = delta - prevDelta;
+ if (dod < Integer.MIN_VALUE || dod > Integer.MAX_VALUE) {
+ return false;
+ }
+ prevDelta = delta;
+ }
+ return true;
+ }
+
+ /**
+ * Returns the number of bits required to encode a delta-of-delta value.
+ *
+ * @param deltaOfDelta the delta-of-delta value
+ * @return bits required
+ */
+ public static int getBitsRequired(long deltaOfDelta) {
+ int bucket = getBucket(deltaOfDelta);
+ return switch (bucket) {
+ case 0 -> 1;
+ case 1 -> 9;
+ case 2 -> 12;
+ case 3 -> 16;
+ default -> 36;
+ };
+ }
+
+ /**
+ * Determines which bucket a delta-of-delta value falls into.
+ *
+ * @param deltaOfDelta the delta-of-delta value
+ * @return bucket number (0 = 1-bit, 1 = 9-bit, 2 = 12-bit, 3 = 16-bit, 4 = 36-bit)
+ */
+ public static int getBucket(long deltaOfDelta) {
+ if (deltaOfDelta == 0) {
+ return 0; // 1-bit
+ } else if (deltaOfDelta >= BUCKET_7BIT_MIN && deltaOfDelta <= BUCKET_7BIT_MAX) {
+ return 1; // 9-bit (2 prefix + 7 value)
+ } else if (deltaOfDelta >= BUCKET_9BIT_MIN && deltaOfDelta <= BUCKET_9BIT_MAX) {
+ return 2; // 12-bit (3 prefix + 9 value)
+ } else if (deltaOfDelta >= BUCKET_12BIT_MIN && deltaOfDelta <= BUCKET_12BIT_MAX) {
+ return 3; // 16-bit (4 prefix + 12 value)
+ } else {
+ return 4; // 36-bit (4 prefix + 32 value)
+ }
+ }
+
+ /**
+ * Encodes a delta-of-delta value using bucket selection.
+ *
+ * Prefix patterns are written LSB-first to match the decoder's read order:
+ *
+ * Format:
+ *
+ * Precondition: the caller must verify that {@link #canUseGorilla(long, int)}
+ * returns {@code true} before calling this method. The largest delta-of-delta
+ * bucket uses 32-bit signed encoding, so values outside the {@code int} range
+ * are silently truncated, producing corrupt output on decode.
+ *
+ * Note: This method does NOT write the encoding flag byte. The caller is
+ * responsible for writing the ENCODING_GORILLA flag before calling this method.
+ *
+ * @param destAddress destination address in native memory
+ * @param capacity maximum number of bytes to write
+ * @param srcAddress source address of contiguous int64 timestamps in native memory
+ * @param count number of timestamps to encode
+ * @return number of bytes written
+ */
+ public int encodeTimestamps(long destAddress, long capacity, long srcAddress, int count) {
+ if (count == 0) {
+ return 0;
+ }
+
+ int pos;
+
+ // Write first timestamp uncompressed
+ if (capacity < 8) {
+ throw new LineSenderException("Gorilla encoder buffer overflow");
+ }
+ long ts0 = Unsafe.getUnsafe().getLong(srcAddress);
+ Unsafe.getUnsafe().putLong(destAddress, ts0);
+ pos = 8;
+
+ if (count == 1) {
+ return pos;
+ }
+
+ // Write second timestamp uncompressed
+ if (capacity < pos + 8) {
+ throw new LineSenderException("Gorilla encoder buffer overflow");
+ }
+ long ts1 = Unsafe.getUnsafe().getLong(srcAddress + 8);
+ Unsafe.getUnsafe().putLong(destAddress + pos, ts1);
+ pos += 8;
+
+ if (count == 2) {
+ return pos;
+ }
+
+ // Encode remaining with delta-of-delta
+ bitWriter.reset(destAddress + pos, capacity - pos);
+ long prevTs = ts1;
+ long prevDelta = ts1 - ts0;
+
+ for (int i = 2; i < count; i++) {
+ long ts = Unsafe.getUnsafe().getLong(srcAddress + (long) i * 8);
+ long delta = ts - prevTs;
+ long dod = delta - prevDelta;
+ encodeDoD(dod);
+ prevDelta = delta;
+ prevTs = ts;
+ }
+
+ return pos + bitWriter.finish();
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java
new file mode 100644
index 0000000..63d005d
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java
@@ -0,0 +1,519 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.protocol;
+
+
+import io.questdb.client.std.Unsafe;
+
+/**
+ * XXHash64 implementation for schema hashing in ILP v4 protocol.
+ *
+ * The schema hash is computed over column definitions (name + type) to enable
+ * schema caching. When a client sends a schema reference (hash), the server
+ * can look up the cached schema instead of re-parsing the full schema each time.
+ *
+ * This is a pure Java implementation of XXHash64 based on the original algorithm
+ * by Yann Collet. It's optimized for small inputs typical of schema hashing.
+ *
+ * @see xxHash
+ */
+public final class QwpSchemaHash {
+
+ // Default seed (0 for ILP v4)
+ private static final long DEFAULT_SEED = 0L;
+ // XXHash64 constants
+ private static final long PRIME64_1 = 0x9E3779B185EBCA87L;
+ private static final long PRIME64_2 = 0xC2B2AE3D27D4EB4FL;
+ // Thread-local Hasher to avoid allocation on every computeSchemaHash call
+ private static final ThreadLocal
+ * This is useful when building the schema hash incrementally
+ * as columns are processed.
+ */
+ public static class Hasher {
+ private final byte[] buffer = new byte[32];
+ private int bufferPos;
+ private long seed;
+ private long totalLen;
+ private long v1, v2, v3, v4;
+
+ public Hasher() {
+ reset(DEFAULT_SEED);
+ }
+
+ /**
+ * Finalizes and returns the hash value.
+ *
+ * @return the 64-bit hash
+ */
+ public long getValue() {
+ long h64;
+
+ if (totalLen >= 32) {
+ h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) +
+ Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18);
+ h64 = mergeRound(h64, v1);
+ h64 = mergeRound(h64, v2);
+ h64 = mergeRound(h64, v3);
+ h64 = mergeRound(h64, v4);
+ } else {
+ h64 = seed + PRIME64_5;
+ }
+
+ h64 += totalLen;
+
+ // Process buffered data
+ int pos = 0;
+ while (pos + 8 <= bufferPos) {
+ long k1 = getLong(buffer, pos);
+ k1 *= PRIME64_2;
+ k1 = Long.rotateLeft(k1, 31);
+ k1 *= PRIME64_1;
+ h64 ^= k1;
+ h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4;
+ pos += 8;
+ }
+
+ if (pos + 4 <= bufferPos) {
+ h64 ^= (getInt(buffer, pos) & 0xFFFFFFFFL) * PRIME64_1;
+ h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3;
+ pos += 4;
+ }
+
+ while (pos < bufferPos) {
+ h64 ^= (buffer[pos] & 0xFFL) * PRIME64_5;
+ h64 = Long.rotateLeft(h64, 11) * PRIME64_1;
+ pos++;
+ }
+
+ return avalanche(h64);
+ }
+
+ /**
+ * Resets the hasher with the given seed.
+ *
+ * @param seed the hash seed
+ */
+ public void reset(long seed) {
+ this.seed = seed;
+ v1 = seed + PRIME64_1 + PRIME64_2;
+ v2 = seed + PRIME64_2;
+ v3 = seed;
+ v4 = seed - PRIME64_1;
+ totalLen = 0;
+ bufferPos = 0;
+ }
+
+ /**
+ * Updates the hash with a byte array.
+ *
+ * @param data the bytes to add
+ */
+ public void update(byte[] data) {
+ update(data, 0, data.length);
+ }
+
+ /**
+ * Updates the hash with a byte array region.
+ *
+ * @param data the bytes to add
+ * @param offset starting offset
+ * @param length number of bytes
+ */
+ public void update(byte[] data, int offset, int length) {
+ totalLen += length;
+
+ // Fill buffer first
+ if (bufferPos > 0) {
+ int toCopy = Math.min(32 - bufferPos, length);
+ System.arraycopy(data, offset, buffer, bufferPos, toCopy);
+ bufferPos += toCopy;
+ offset += toCopy;
+ length -= toCopy;
+
+ if (bufferPos == 32) {
+ processBuffer();
+ }
+ }
+
+ // Process 32-byte blocks directly
+ while (length >= 32) {
+ v1 = round(v1, getLong(data, offset));
+ v2 = round(v2, getLong(data, offset + 8));
+ v3 = round(v3, getLong(data, offset + 16));
+ v4 = round(v4, getLong(data, offset + 24));
+ offset += 32;
+ length -= 32;
+ }
+
+ // Buffer remaining
+ if (length > 0) {
+ System.arraycopy(data, offset, buffer, 0, length);
+ bufferPos = length;
+ }
+ }
+
+ /**
+ * Updates the hash with a single byte.
+ *
+ * @param b the byte to add
+ */
+ public void update(byte b) {
+ buffer[bufferPos++] = b;
+ totalLen++;
+
+ if (bufferPos == 32) {
+ processBuffer();
+ }
+ }
+
+ private void processBuffer() {
+ v1 = round(v1, getLong(buffer, 0));
+ v2 = round(v2, getLong(buffer, 8));
+ v3 = round(v3, getLong(buffer, 16));
+ v4 = round(v4, getLong(buffer, 24));
+ bufferPos = 0;
+ }
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
new file mode 100644
index 0000000..f3d7fe1
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
@@ -0,0 +1,1284 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.protocol;
+
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.line.array.ArrayBufferAppender;
+import io.questdb.client.cutlass.line.array.DoubleArray;
+import io.questdb.client.cutlass.line.array.LongArray;
+import io.questdb.client.std.CharSequenceIntHashMap;
+import io.questdb.client.std.Decimal128;
+import io.questdb.client.std.Decimal256;
+import io.questdb.client.std.Decimal64;
+import io.questdb.client.std.Decimals;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.ObjList;
+import io.questdb.client.std.QuietCloseable;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.std.Vect;
+
+import java.util.Arrays;
+
+import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*;
+
+/**
+ * Buffers rows for a single table in columnar format.
+ *
+ * Fixed-width column data is stored off-heap via {@link OffHeapAppendMemory} for zero-GC
+ * buffering and bulk copy to network buffers. Variable-width data (strings, symbol
+ * dictionaries, arrays) remains on-heap.
+ */
+public class QwpTableBuffer implements QuietCloseable {
+
+ private final CharSequenceIntHashMap columnNameToIndex;
+ private final ObjList
+ * This removes any column values added since the last {@link #nextRow()} call.
+ * If no values have been added for the current row, this is a no-op.
+ */
+ public void cancelCurrentRow() {
+ // Reset sequential access cursor
+ columnAccessCursor = 0;
+ // Truncate each column back to the committed row count
+ for (int i = 0, n = columns.size(); i < n; i++) {
+ ColumnBuffer col = fastColumns[i];
+ col.truncateTo(rowCount);
+ }
+ }
+
+ /**
+ * Clears the buffer completely, including column definitions.
+ * Frees all off-heap memory.
+ */
+ public void clear() {
+ for (int i = 0, n = columns.size(); i < n; i++) {
+ columns.get(i).close();
+ }
+ columns.clear();
+ columnNameToIndex.clear();
+ fastColumns = null;
+ columnAccessCursor = 0;
+ rowCount = 0;
+ schemaHash = 0;
+ schemaHashComputed = false;
+ columnDefsCacheValid = false;
+ cachedColumnDefs = null;
+ }
+
+ @Override
+ public void close() {
+ clear();
+ }
+
+ /**
+ * Returns the total bytes buffered across all columns.
+ * This queries actual buffer sizes, not estimates.
+ */
+ public long getBufferedBytes() {
+ long bytes = 0;
+ for (int i = 0, n = columns.size(); i < n; i++) {
+ bytes += fastColumns[i].getBufferedBytes();
+ }
+ return bytes;
+ }
+
+ /**
+ * Returns the column at the given index.
+ */
+ public ColumnBuffer getColumn(int index) {
+ return columns.get(index);
+ }
+
+ /**
+ * Returns the number of columns.
+ */
+ public int getColumnCount() {
+ return columns.size();
+ }
+
+ /**
+ * Returns the column definitions (cached for efficiency).
+ */
+ public QwpColumnDef[] getColumnDefs() {
+ if (!columnDefsCacheValid || cachedColumnDefs == null || cachedColumnDefs.length != columns.size()) {
+ cachedColumnDefs = new QwpColumnDef[columns.size()];
+ for (int i = 0; i < columns.size(); i++) {
+ ColumnBuffer col = columns.get(i);
+ cachedColumnDefs[i] = new QwpColumnDef(col.name, col.type, col.nullable);
+ }
+ columnDefsCacheValid = true;
+ }
+ return cachedColumnDefs;
+ }
+
+ /**
+ * Gets or creates a column with the given name and type.
+ *
+ * Optimized for the common case where columns are accessed in the same
+ * order every row: a sequential cursor avoids hash map lookups entirely.
+ */
+ public ColumnBuffer getOrCreateColumn(String name, byte type, boolean nullable) {
+ // Fast path: predict next column in sequence
+ int n = columns.size();
+ if (columnAccessCursor < n) {
+ ColumnBuffer candidate = fastColumns[columnAccessCursor];
+ if (candidate.name.equals(name)) {
+ columnAccessCursor++;
+ if (candidate.type != type) {
+ throw new LineSenderException(
+ "Column type mismatch for " + name + ": existing=" + candidate.type + " new=" + type
+ );
+ }
+ return candidate;
+ }
+ }
+
+ // Slow path: hash map lookup
+ int idx = columnNameToIndex.get(name);
+ if (idx != CharSequenceIntHashMap.NO_ENTRY_VALUE) {
+ ColumnBuffer existing = columns.get(idx);
+ if (existing.type != type) {
+ throw new LineSenderException(
+ "Column type mismatch for " + name + ": existing=" + existing.type + " new=" + type
+ );
+ }
+ return existing;
+ }
+
+ // Create new column
+ ColumnBuffer col = new ColumnBuffer(name, type, nullable);
+ int index = columns.size();
+ columns.add(col);
+ columnNameToIndex.put(name, index);
+ // Update fast access array
+ if (fastColumns == null || index >= fastColumns.length) {
+ int newLen = Math.max(8, index + 4);
+ ColumnBuffer[] newArr = new ColumnBuffer[newLen];
+ if (fastColumns != null) {
+ System.arraycopy(fastColumns, 0, newArr, 0, index);
+ }
+ fastColumns = newArr;
+ }
+ fastColumns[index] = col;
+ schemaHashComputed = false;
+ columnDefsCacheValid = false;
+ return col;
+ }
+
+ /**
+ * Returns the number of rows buffered.
+ */
+ public int getRowCount() {
+ return rowCount;
+ }
+
+ /**
+ * Returns the schema hash for this table.
+ *
+ * The hash is computed to match what QwpSchema.computeSchemaHash() produces:
+ * - Uses wire type codes (with nullable bit)
+ * - Hash is over name bytes + type code for each column
+ */
+ public long getSchemaHash() {
+ if (!schemaHashComputed) {
+ // Compute hash directly from column buffers without intermediate arrays
+ schemaHash = QwpSchemaHash.computeSchemaHashDirect(columns);
+ schemaHashComputed = true;
+ }
+ return schemaHash;
+ }
+
+ /**
+ * Returns the table name.
+ */
+ public String getTableName() {
+ return tableName;
+ }
+
+ /**
+ * Advances to the next row.
+ *
+ * This should be called after all column values for the current row have been set.
+ */
+ public void nextRow() {
+ // Reset sequential access cursor for the next row
+ columnAccessCursor = 0;
+ // Ensure all columns have the same row count
+ for (int i = 0, n = columns.size(); i < n; i++) {
+ ColumnBuffer col = fastColumns[i];
+ // If column wasn't set for this row, add a null
+ while (col.size < rowCount + 1) {
+ col.addNull();
+ }
+ }
+ rowCount++;
+ }
+
+ /**
+ * Resets the buffer for reuse. Keeps column definitions and allocated memory.
+ */
+ public void reset() {
+ for (int i = 0, n = columns.size(); i < n; i++) {
+ fastColumns[i].reset();
+ }
+ columnAccessCursor = 0;
+ rowCount = 0;
+ }
+
+ /**
+ * Returns the in-memory buffer element stride in bytes. This is the size used
+ * to store each value in the client's off-heap {@link OffHeapAppendMemory} buffer.
+ * This is different from element size on the wire.
+ *
+ * For example, BOOLEAN is stored as 1 byte per value here (for easy indexed access)
+ * but bit-packed on the wire; GEOHASH is stored as 8-byte longs here but uses
+ * variable-width encoding on the wire.
+ *
+ * Returns 0 for variable-width types (string, arrays) that do not use a fixed-stride
+ * data buffer.
+ *
+ * @see QwpConstants#getFixedTypeSize(byte) for wire-format sizes
+ */
+ static int elementSizeInBuffer(byte type) {
+ return switch (type) {
+ case TYPE_BOOLEAN, TYPE_BYTE -> 1;
+ case TYPE_SHORT, TYPE_CHAR -> 2;
+ case TYPE_INT, TYPE_SYMBOL, TYPE_FLOAT -> 4;
+ case TYPE_GEOHASH, TYPE_LONG, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS,
+ TYPE_DATE, TYPE_DECIMAL64, TYPE_DOUBLE -> 8;
+ case TYPE_UUID, TYPE_DECIMAL128 -> 16;
+ case TYPE_LONG256, TYPE_DECIMAL256 -> 32;
+ default -> 0;
+ };
+ }
+
+ /**
+ * Helper class to capture array data from DoubleArray/LongArray.appendToBufPtr().
+ */
+ private static class ArrayCapture implements ArrayBufferAppender {
+ final int[] shape = new int[32];
+ double[] doubleData;
+ int doubleDataOffset;
+ long[] longData;
+ int longDataOffset;
+ byte nDims;
+ private boolean forLong;
+ private int shapeIndex;
+
+ @Override
+ public void putBlockOfBytes(long from, long len) {
+ int count = (int) (len / 8);
+ if (forLong) {
+ if (longData == null || longData.length < count) {
+ longData = new long[count];
+ }
+ for (int i = 0; i < count; i++) {
+ longData[longDataOffset++] = Unsafe.getUnsafe().getLong(from + i * 8L);
+ }
+ } else {
+ if (doubleData == null || doubleData.length < count) {
+ doubleData = new double[count];
+ }
+ for (int i = 0; i < count; i++) {
+ doubleData[doubleDataOffset++] = Unsafe.getUnsafe().getDouble(from + i * 8L);
+ }
+ }
+ }
+
+ @Override
+ public void putByte(byte b) {
+ if (shapeIndex == 0) {
+ nDims = b;
+ }
+ }
+
+ @Override
+ public void putDouble(double value) {
+ if (doubleData != null && doubleDataOffset < doubleData.length) {
+ doubleData[doubleDataOffset++] = value;
+ }
+ }
+
+ @Override
+ public void putInt(int value) {
+ if (shapeIndex < nDims) {
+ shape[shapeIndex++] = value;
+ if (shapeIndex == nDims) {
+ int totalElements = 1;
+ for (int i = 0; i < nDims; i++) {
+ totalElements *= shape[i];
+ }
+ if (forLong) {
+ if (longData == null || longData.length < totalElements) {
+ longData = new long[totalElements];
+ }
+ } else {
+ if (doubleData == null || doubleData.length < totalElements) {
+ doubleData = new double[totalElements];
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public void putLong(long value) {
+ if (longData != null && longDataOffset < longData.length) {
+ longData[longDataOffset++] = value;
+ }
+ }
+
+ void reset(boolean forLong) {
+ this.forLong = forLong;
+ shapeIndex = 0;
+ nDims = 0;
+ doubleDataOffset = 0;
+ longDataOffset = 0;
+ }
+ }
+
+ /**
+ * Column buffer for a single column.
+ *
+ * Fixed-width data is stored off-heap in {@link OffHeapAppendMemory} for zero-GC
+ * operation and efficient bulk copy to network buffers.
+ */
+ public static class ColumnBuffer implements QuietCloseable {
+ final int elemSize;
+ final String name;
+ final boolean nullable;
+ final byte type;
+ private final Decimal256 rescaleTemp = new Decimal256();
+ private ArrayCapture arrayCapture;
+ private int arrayDataOffset;
+ // Array storage (double/long arrays - variable length per row)
+ private byte[] arrayDims;
+ private int arrayShapeOffset;
+ private int[] arrayShapes;
+ // Off-heap auxiliary buffer for global symbol IDs (SYMBOL type only)
+ private OffHeapAppendMemory auxBuffer;
+ // Off-heap data buffer for fixed-width types
+ private OffHeapAppendMemory dataBuffer;
+ // Decimal storage
+ private byte decimalScale = -1;
+ private double[] doubleArrayData;
+ // GeoHash precision (number of bits, 1-60)
+ private int geohashPrecision = -1;
+ private boolean hasNulls;
+ private long[] longArrayData;
+ private int maxGlobalSymbolId = -1;
+ private int nullBufCapRows;
+ // Off-heap null bitmap (bit-packed, 1 bit per row)
+ private long nullBufPtr;
+ private int size; // Total row count (including nulls)
+ private OffHeapAppendMemory stringData;
+ // Off-heap storage for string/varchar column data
+ private OffHeapAppendMemory stringOffsets;
+ // Symbol specific (dictionary stays on-heap)
+ private CharSequenceIntHashMap symbolDict;
+ private ObjList The parser operates on raw memory buffers and maintains minimal state.
+ * It can parse frames incrementally when data arrives in chunks.
+ *
+ * Thread safety: This class is NOT thread-safe. Each connection should
+ * have its own parser instance.
+ */
+public class WebSocketFrameParser {
+ /**
+ * Frame completely parsed.
+ */
+ public static final int STATE_COMPLETE = 3;
+ /**
+ * Error state - frame is invalid.
+ */
+ public static final int STATE_ERROR = 4;
+ /**
+ * Initial state, waiting for frame header.
+ */
+ public static final int STATE_HEADER = 0;
+ /**
+ * Need more data to complete parsing.
+ */
+ public static final int STATE_NEED_MORE = 1;
+ /**
+ * Header parsed, need payload data.
+ */
+ public static final int STATE_NEED_PAYLOAD = 2;
+ // Frame header bits
+ private static final int FIN_BIT = 0x80;
+ private static final int LENGTH_MASK = 0x7F;
+ private static final int MASK_BIT = 0x80;
+ // Control frame max payload size (RFC 6455)
+ private static final int MAX_CONTROL_FRAME_PAYLOAD = 125;
+ private static final int OPCODE_MASK = 0x0F;
+ private static final int RSV_BITS = 0x70;
+ private int errorCode;
+ // Parsed frame data
+ private boolean fin;
+ private int headerSize;
+ private int maskKey;
+ private boolean masked;
+ private int opcode;
+ private long payloadLength;
+ // Parser state
+ private int state = STATE_HEADER;
+
+ public int getErrorCode() {
+ return errorCode;
+ }
+
+ public int getHeaderSize() {
+ return headerSize;
+ }
+
+ public int getOpcode() {
+ return opcode;
+ }
+
+ public long getPayloadLength() {
+ return payloadLength;
+ }
+
+ public int getState() {
+ return state;
+ }
+
+ public boolean isFin() {
+ return fin;
+ }
+
+ public boolean isMasked() {
+ return masked;
+ }
+
+ /**
+ * Parses a WebSocket frame from the given buffer.
+ *
+ * @param buf the start of the buffer
+ * @param limit the end of the buffer (exclusive)
+ * @return the number of bytes consumed, or 0 if more data is needed
+ */
+ public int parse(long buf, long limit) {
+ long available = limit - buf;
+
+ if (available < 2) {
+ state = STATE_NEED_MORE;
+ return 0;
+ }
+
+ // Parse first two bytes
+ int byte0 = Unsafe.getUnsafe().getByte(buf) & 0xFF;
+ int byte1 = Unsafe.getUnsafe().getByte(buf + 1) & 0xFF;
+
+ // Check reserved bits (must be 0 unless extension negotiated)
+ if ((byte0 & RSV_BITS) != 0) {
+ state = STATE_ERROR;
+ errorCode = WebSocketCloseCode.PROTOCOL_ERROR;
+ return 0;
+ }
+
+ fin = (byte0 & FIN_BIT) != 0;
+ opcode = byte0 & OPCODE_MASK;
+
+ // Validate opcode
+ if (!WebSocketOpcode.isValid(opcode)) {
+ state = STATE_ERROR;
+ errorCode = WebSocketCloseCode.PROTOCOL_ERROR;
+ return 0;
+ }
+
+ // Control frames must not be fragmented
+ if (WebSocketOpcode.isControlFrame(opcode) && !fin) {
+ state = STATE_ERROR;
+ errorCode = WebSocketCloseCode.PROTOCOL_ERROR;
+ return 0;
+ }
+
+ final boolean masked = (byte1 & MASK_BIT) != 0;
+ this.masked = masked;
+ int lengthField = byte1 & LENGTH_MASK;
+
+ // Validate masking based on mode
+ // Configuration
+ // If true, expect masked frames from clients
+ if (masked) {
+ // Server frames MUST NOT be masked
+ state = STATE_ERROR;
+ errorCode = WebSocketCloseCode.PROTOCOL_ERROR;
+ return 0;
+ }
+
+ // Calculate header size and payload length
+ int offset = 2;
+
+ // If true, reject non-minimal length encodings
+ if (lengthField <= 125) {
+ payloadLength = lengthField;
+ } else if (lengthField == 126) {
+ // 16-bit extended length
+ if (available < 4) {
+ state = STATE_NEED_MORE;
+ return 0;
+ }
+ int high = Unsafe.getUnsafe().getByte(buf + 2) & 0xFF;
+ int low = Unsafe.getUnsafe().getByte(buf + 3) & 0xFF;
+ payloadLength = (high << 8) | low;
+
+ // Strict mode: reject non-minimal encodings
+
+ offset = 4;
+ } else {
+ // 64-bit extended length
+ if (available < 10) {
+ state = STATE_NEED_MORE;
+ return 0;
+ }
+ payloadLength = Long.reverseBytes(Unsafe.getUnsafe().getLong(buf + 2));
+
+ // Strict mode: reject non-minimal encodings
+
+ // MSB must be 0 (no negative lengths)
+ if (payloadLength < 0) {
+ state = STATE_ERROR;
+ errorCode = WebSocketCloseCode.PROTOCOL_ERROR;
+ return 0;
+ }
+
+ offset = 10;
+ }
+
+ // Control frames must not have payload > 125 bytes
+ if (WebSocketOpcode.isControlFrame(opcode) && payloadLength > MAX_CONTROL_FRAME_PAYLOAD) {
+ state = STATE_ERROR;
+ errorCode = WebSocketCloseCode.PROTOCOL_ERROR;
+ return 0;
+ }
+
+ // Close frame with 1 byte payload is invalid (must be 0 or >= 2)
+ if (opcode == WebSocketOpcode.CLOSE && payloadLength == 1) {
+ state = STATE_ERROR;
+ errorCode = WebSocketCloseCode.PROTOCOL_ERROR;
+ return 0;
+ }
+
+ maskKey = 0;
+ headerSize = offset;
+
+ // Check if we have the complete payload
+ long totalFrameSize = headerSize + payloadLength;
+ if (available < totalFrameSize) {
+ state = STATE_NEED_PAYLOAD;
+ return headerSize;
+ }
+
+ state = STATE_COMPLETE;
+ return (int) totalFrameSize;
+ }
+
+ /**
+ * Resets the parser state for parsing a new frame.
+ */
+ public void reset() {
+ state = STATE_HEADER;
+ fin = false;
+ opcode = 0;
+ masked = false;
+ maskKey = 0;
+ payloadLength = 0;
+ headerSize = 0;
+ errorCode = 0;
+ }
+
+ /**
+ * Unmasks the payload data in place.
+ *
+ * @param buf the start of the payload data
+ * @param len the length of the payload
+ */
+ public void unmaskPayload(long buf, long len) {
+ if (!masked || maskKey == 0) {
+ // a zero maskKey is a no-op (makes no change to the data)
+ return;
+ }
+
+ // Process 8 bytes at a time when possible for better performance
+ long i = 0;
+ long longMask = ((long) maskKey << 32) | (maskKey & 0xFFFFFFFFL);
+
+ // Process 8-byte chunks
+ while (i + 8 <= len) {
+ long value = Unsafe.getUnsafe().getLong(buf + i);
+ Unsafe.getUnsafe().putLong(buf + i, value ^ longMask);
+ i += 8;
+ }
+
+ // Process 4-byte chunk if remaining
+ if (i + 4 <= len) {
+ int value = Unsafe.getUnsafe().getInt(buf + i);
+ Unsafe.getUnsafe().putInt(buf + i, value ^ maskKey);
+ i += 4;
+ }
+
+ // Process remaining bytes
+ while (i < len) {
+ byte b = Unsafe.getUnsafe().getByte(buf + i);
+ int shift = ((int) (i % 4)) << 3; // 0, 8, 16, or 24
+ byte maskByte = (byte) ((maskKey >> shift) & 0xFF);
+ Unsafe.getUnsafe().putByte(buf + i, (byte) (b ^ maskByte));
+ i++;
+ }
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java
new file mode 100644
index 0000000..2f3c653
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java
@@ -0,0 +1,150 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.websocket;
+
+import io.questdb.client.std.Unsafe;
+
+/**
+ * Zero-allocation WebSocket frame writer.
+ * Writes WebSocket frames according to RFC 6455.
+ *
+ * All methods are static utilities that write directly to memory buffers.
+ *
+ * Thread safety: This class is thread-safe as it contains no mutable state.
+ */
+public final class WebSocketFrameWriter {
+ // Frame header bits
+ private static final int FIN_BIT = 0x80;
+ private static final int MASK_BIT = 0x80;
+
+ private WebSocketFrameWriter() {
+ // Static utility class
+ }
+
+ /**
+ * Calculates the header size for a given payload length and masking.
+ *
+ * @param payloadLength the payload length
+ * @param masked true if the payload will be masked
+ * @return the header size in bytes
+ */
+ public static int headerSize(long payloadLength, boolean masked) {
+ int size;
+ if (payloadLength <= 125) {
+ size = 2;
+ } else if (payloadLength <= 65535) {
+ size = 4;
+ } else {
+ size = 10;
+ }
+ return masked ? size + 4 : size;
+ }
+
+ /**
+ * Masks payload data in place using XOR with the given mask key.
+ *
+ * @param buf the payload buffer
+ * @param len the payload length
+ * @param maskKey the 4-byte mask key
+ */
+ public static void maskPayload(long buf, long len, int maskKey) {
+ // Process 8 bytes at a time when possible
+ long i = 0;
+ long longMask = ((long) maskKey << 32) | (maskKey & 0xFFFFFFFFL);
+
+ // Process 8-byte chunks
+ while (i + 8 <= len) {
+ long value = Unsafe.getUnsafe().getLong(buf + i);
+ Unsafe.getUnsafe().putLong(buf + i, value ^ longMask);
+ i += 8;
+ }
+
+ // Process 4-byte chunk if remaining
+ if (i + 4 <= len) {
+ int value = Unsafe.getUnsafe().getInt(buf + i);
+ Unsafe.getUnsafe().putInt(buf + i, value ^ maskKey);
+ i += 4;
+ }
+
+ // Process remaining bytes (0-3 bytes) - extract mask byte inline to avoid allocation
+ while (i < len) {
+ byte b = Unsafe.getUnsafe().getByte(buf + i);
+ int maskByte = (maskKey >> (((int) i & 3) << 3)) & 0xFF;
+ Unsafe.getUnsafe().putByte(buf + i, (byte) (b ^ maskByte));
+ i++;
+ }
+ }
+
+ /**
+ * Writes a WebSocket frame header to the buffer.
+ *
+ * @param buf the buffer to write to
+ * @param fin true if this is the final frame
+ * @param opcode the frame opcode
+ * @param payloadLength the payload length
+ * @param masked true if the payload should be masked
+ * @return the number of bytes written (header size)
+ */
+ public static int writeHeader(long buf, boolean fin, int opcode, long payloadLength, boolean masked) {
+ int offset = 0;
+
+ // First byte: FIN + opcode
+ int byte0 = (fin ? FIN_BIT : 0) | (opcode & 0x0F);
+ Unsafe.getUnsafe().putByte(buf + offset++, (byte) byte0);
+
+ // Second byte: MASK + payload length
+ int maskBit = masked ? MASK_BIT : 0;
+
+ if (payloadLength <= 125) {
+ Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | payloadLength));
+ } else if (payloadLength <= 65535) {
+ Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | 126));
+ Unsafe.getUnsafe().putByte(buf + offset++, (byte) ((payloadLength >> 8) & 0xFF));
+ Unsafe.getUnsafe().putByte(buf + offset++, (byte) (payloadLength & 0xFF));
+ } else {
+ Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | 127));
+ Unsafe.getUnsafe().putLong(buf + offset, Long.reverseBytes(payloadLength));
+ offset += 8;
+ }
+
+ return offset;
+ }
+
+ /**
+ * Writes a WebSocket frame header with optional mask key.
+ *
+ * @param buf the buffer to write to
+ * @param fin true if this is the final frame
+ * @param opcode the frame opcode
+ * @param payloadLength the payload length
+ * @param maskKey the mask key (only used if masked is true)
+ * @return the number of bytes written (header size including mask key)
+ */
+ public static int writeHeader(long buf, boolean fin, int opcode, long payloadLength, int maskKey) {
+ int offset = writeHeader(buf, fin, opcode, payloadLength, true);
+ Unsafe.getUnsafe().putInt(buf + offset, maskKey);
+ return offset + 4;
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java
new file mode 100644
index 0000000..8668644
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java
@@ -0,0 +1,107 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.websocket;
+
+/**
+ * WebSocket frame opcodes as defined in RFC 6455.
+ */
+public final class WebSocketOpcode {
+ /**
+ * Binary frame (0x2).
+ * Payload is arbitrary binary data.
+ */
+ public static final int BINARY = 0x02;
+ /**
+ * Connection close frame (0x8).
+ * Indicates that the endpoint wants to close the connection.
+ */
+ public static final int CLOSE = 0x08;
+ /**
+ * Continuation frame (0x0).
+ * Used for fragmented messages after the initial frame.
+ */
+ public static final int CONTINUATION = 0x00;
+
+ // Reserved non-control frames: 0x3-0x7
+ /**
+ * Ping frame (0x9).
+ * Used for keep-alive and connection health checks.
+ */
+ public static final int PING = 0x09;
+ /**
+ * Pong frame (0xA).
+ * Response to a ping frame.
+ */
+ public static final int PONG = 0x0A;
+ /**
+ * Text frame (0x1).
+ * Payload is UTF-8 encoded text.
+ */
+ public static final int TEXT = 0x01;
+
+ // Reserved control frames: 0xB-0xF
+
+ private WebSocketOpcode() {
+ // Constants class
+ }
+
+ /**
+ * Checks if the opcode is a control frame.
+ * Control frames are CLOSE (0x8), PING (0x9), and PONG (0xA).
+ *
+ * @param opcode the opcode to check
+ * @return true if the opcode is a control frame
+ */
+ public static boolean isControlFrame(int opcode) {
+ return (opcode & 0x08) != 0;
+ }
+
+ /**
+ * Checks if the opcode is a data frame.
+ * Data frames are CONTINUATION (0x0), TEXT (0x1), and BINARY (0x2).
+ *
+ * @param opcode the opcode to check
+ * @return true if the opcode is a data frame
+ */
+ public static boolean isDataFrame(int opcode) {
+ return opcode <= 0x02;
+ }
+
+ /**
+ * Checks if the opcode is valid according to RFC 6455.
+ *
+ * @param opcode the opcode to check
+ * @return true if the opcode is valid
+ */
+ public static boolean isValid(int opcode) {
+ return opcode == CONTINUATION
+ || opcode == TEXT
+ || opcode == BINARY
+ || opcode == CLOSE
+ || opcode == PING
+ || opcode == PONG;
+ }
+
+}
diff --git a/core/src/main/java/io/questdb/client/network/IOOperation.java b/core/src/main/java/io/questdb/client/network/IOOperation.java
index dca5ac3..b600280 100644
--- a/core/src/main/java/io/questdb/client/network/IOOperation.java
+++ b/core/src/main/java/io/questdb/client/network/IOOperation.java
@@ -25,7 +25,6 @@
package io.questdb.client.network;
public final class IOOperation {
- public static final int HEARTBEAT = 8;
public static final int READ = 1;
public static final int WRITE = 4;
diff --git a/core/src/main/java/io/questdb/client/network/Net.java b/core/src/main/java/io/questdb/client/network/Net.java
index 1f35299..b1c4721 100644
--- a/core/src/main/java/io/questdb/client/network/Net.java
+++ b/core/src/main/java/io/questdb/client/network/Net.java
@@ -80,9 +80,9 @@ public static void configureKeepAlive(int fd) {
public static native int configureNonBlocking(int fd);
- public native static int connect(int fd, long sockaddr);
+ public static native int connect(int fd, long sockaddr);
- public native static int connectAddrInfo(int fd, long lpAddrInfo);
+ public static native int connectAddrInfo(int fd, long lpAddrInfo);
public static void freeAddrInfo(long pAddrInfo) {
if (pAddrInfo != 0) {
@@ -106,63 +106,59 @@ public static long getAddrInfo(CharSequence host, int port) {
}
}
- private static long getAddrInfo(DirectUtf8Sequence host, int port) {
- return getAddrInfo(host.ptr(), port);
- }
-
- private static long getAddrInfo(long lpszHost, int port) {
- long addrInfo = getAddrInfo0(lpszHost, port);
- if (addrInfo != -1) {
- ADDR_INFO_COUNTER.incrementAndGet();
- }
- return addrInfo;
- }
-
- public native static int getSndBuf(int fd);
+ public static native int getSndBuf(int fd);
public static void init() {
// no-op
}
- public native static boolean join(int fd, int bindIPv4Address, int groupIPv4Address);
+ public static native boolean join(int fd, int bindIPv4Address, int groupIPv4Address);
public static native int peek(int fd, long ptr, int len);
public static native int recv(int fd, long ptr, int len);
- public static int send(long fd, long ptr, int len) {
- return send(fd, ptr, len);
- }
-
public static native int send(int fd, long ptr, int len);
- public native static int sendTo(int fd, long ptr, int len, long sockaddr);
+ public static native int sendTo(int fd, long ptr, int len, long sockaddr);
public static native int setKeepAlive0(int fd, int seconds);
- public native static int setMulticastInterface(int fd, int ipv4address);
+ public static native int setMulticastInterface(int fd, int ipv4address);
- public native static int setMulticastTtl(int fd, int ttl);
+ public static native int setMulticastTtl(int fd, int ttl);
- public native static int setSndBuf(int fd, int size);
+ public static native int setSndBuf(int fd, int size);
- public native static int setTcpNoDelay(int fd, boolean noDelay);
+ public static native int setTcpNoDelay(int fd, boolean noDelay);
public static long sockaddr(int ipv4address, int port) {
SOCK_ADDR_COUNTER.incrementAndGet();
return sockaddr0(ipv4address, port);
}
- public native static long sockaddr0(int ipv4address, int port);
+ public static native long sockaddr0(int ipv4address, int port);
- public native static int socketTcp(boolean blocking);
+ public static native int socketTcp(boolean blocking);
- public native static int socketUdp();
+ public static native int socketUdp();
private static native void freeAddrInfo0(long pAddrInfo);
private static native void freeSockAddr0(long sockaddr);
+ private static long getAddrInfo(DirectUtf8Sequence host, int port) {
+ return getAddrInfo(host.ptr(), port);
+ }
+
+ private static long getAddrInfo(long lpszHost, int port) {
+ long addrInfo = getAddrInfo0(lpszHost, port);
+ if (addrInfo != -1) {
+ ADDR_INFO_COUNTER.incrementAndGet();
+ }
+ return addrInfo;
+ }
+
private static native long getAddrInfo0(long lpszHost, int port);
private static native int getEwouldblock();
@@ -186,4 +182,4 @@ public static long sockaddr(int ipv4address, int port) {
MMSGHDR_BUFFER_LENGTH_OFFSET = -1L;
}
}
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/io/questdb/client/std/AbstractLowerCaseCharSequenceHashSet.java b/core/src/main/java/io/questdb/client/std/AbstractLowerCaseCharSequenceHashSet.java
deleted file mode 100644
index 4bb8cf9..0000000
--- a/core/src/main/java/io/questdb/client/std/AbstractLowerCaseCharSequenceHashSet.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*******************************************************************************
- * ___ _ ____ ____
- * / _ \ _ _ ___ ___| |_| _ \| __ )
- * | | | | | | |/ _ \/ __| __| | | | _ \
- * | |_| | |_| | __/\__ \ |_| |_| | |_) |
- * \__\_\\__,_|\___||___/\__|____/|____/
- *
- * Copyright (c) 2014-2019 Appsicle
- * Copyright (c) 2019-2026 QuestDB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- ******************************************************************************/
-
-package io.questdb.client.std;
-
-import java.util.Arrays;
-
-public abstract class AbstractLowerCaseCharSequenceHashSet implements Mutable {
- protected static final int MIN_INITIAL_CAPACITY = 16;
- protected static final CharSequence noEntryKey = null;
- protected final double loadFactor;
- protected int capacity;
- protected int free;
- protected CharSequence[] keys;
- protected int mask;
-
- public AbstractLowerCaseCharSequenceHashSet(int initialCapacity, double loadFactor) {
- if (loadFactor <= 0d || loadFactor >= 1d) {
- throw new IllegalArgumentException("0 < loadFactor < 1");
- }
-
- free = this.capacity = Math.max(initialCapacity, MIN_INITIAL_CAPACITY);
- this.loadFactor = loadFactor;
- keys = new CharSequence[Numbers.ceilPow2((int) (this.capacity / loadFactor))];
- mask = keys.length - 1;
- }
-
- @Override
- public void clear() {
- Arrays.fill(keys, noEntryKey);
- free = capacity;
- }
-
- public boolean contains(CharSequence key) {
- return keyIndex(key) < 0;
- }
-
- public int keyIndex(CharSequence key) {
- int index = Chars.lowerCaseHashCode(key) & mask;
-
- if (keys[index] == noEntryKey) {
- return index;
- }
-
- if (Chars.equalsIgnoreCase(key, keys[index])) {
- return -index - 1;
- }
-
- return probe(key, index);
- }
-
- public int size() {
- return capacity - free;
- }
-
- private int probe(CharSequence key, int index) {
- do {
- index = (index + 1) & mask;
- if (keys[index] == noEntryKey) {
- return index;
- }
- if (Chars.equalsIgnoreCase(key, keys[index])) {
- return -index - 1;
- }
- } while (true);
- }
-
-}
\ No newline at end of file
diff --git a/core/src/main/java/io/questdb/client/std/Base64Helper.java b/core/src/main/java/io/questdb/client/std/Base64Helper.java
deleted file mode 100644
index 27d6772..0000000
--- a/core/src/main/java/io/questdb/client/std/Base64Helper.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*******************************************************************************
- * ___ _ ____ ____
- * / _ \ _ _ ___ ___| |_| _ \| __ )
- * | | | | | | |/ _ \/ __| __| | | | _ \
- * | |_| | |_| | __/\__ \ |_| |_| | |_) |
- * \__\_\\__,_|\___||___/\__|____/|____/
- *
- * Copyright (c) 2014-2019 Appsicle
- * Copyright (c) 2019-2026 QuestDB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- ******************************************************************************/
-
-// Written by Gil Tene of Azul Systems, and released to the public domain,
-// as explained at http://creativecommons.org/publicdomain/zero/1.0/
-//
-// @author Gil Tene
-
-package io.questdb.client.std;
-
-import java.lang.reflect.Method;
-
-/**
- * Base64Helper exists to bridge inconsistencies in Java SE support of Base64 encoding and decoding.
- * Earlier Java SE platforms (up to and including Java SE 8) supported base64 encode/decode via the
- * javax.xml.bind.DatatypeConverter class, which was deprecated and eventually removed in Java SE 9.
- * Later Java SE platforms (Java SE 8 and later) support base64 encode/decode via the
- * java.util.Base64 class (first introduced in Java SE 8, and not available on e.g. Java SE 6 or 7).
- *
- * This makes it "hard" to write a single piece of source code that deals with base64 encodings and
- * will compile and run on e.g. Java SE 7 AND Java SE 9. And such common source is a common need for
- * libraries. This class is intended to encapsulate this "hard"-ness and hide the ugly pretzle-twising
- * needed under the covers.
- *
- * Base64Helper provides a common API that works across Java SE 6..9 (and beyond hopefully), and
- * uses late binding (Reflection) internally to avoid javac-compile-time dependencies on a specific
- * Java SE version (e.g. beyond 7 or before 9).
- */
-public class Base64Helper {
-
- private static Method decodeMethod;
- // encoderObj and decoderObj are used in non-static method forms, and
- // irrelevant for static method forms:
- private static Object decoderObj;
- private static Method encodeMethod;
- private static Object encoderObj;
-
- /**
- * Converts a Base64 encoded String to a byte array
- *
- * @param base64input A base64-encoded input String
- * @return a byte array containing the binary representation equivalent of the Base64 encoded input
- */
- public static byte[] parseBase64Binary(String base64input) {
- try {
- return (byte[]) decodeMethod.invoke(decoderObj, base64input);
- } catch (Throwable e) {
- throw new UnsupportedOperationException("Failed to use platform's base64 decode method");
- }
- }
-
- /**
- * Converts an array of bytes into a Base64 string.
- *
- * @param binaryArray A binary encoded input array
- * @return a String containing the Base64 encoded equivalent of the binary input
- */
- static String printBase64Binary(byte[] binaryArray) {
- try {
- return (String) encodeMethod.invoke(encoderObj, binaryArray);
- } catch (Throwable e) {
- throw new UnsupportedOperationException("Failed to use platform's base64 encode method");
- }
- }
-
- static {
- try {
- Class javaUtilBase64Class = Class.forName("java.util.Base64");
-
- Method getDecoderMethod = javaUtilBase64Class.getMethod("getDecoder");
- decoderObj = getDecoderMethod.invoke(null);
- decodeMethod = decoderObj.getClass().getMethod("decode", String.class);
-
- Method getEncoderMethod = javaUtilBase64Class.getMethod("getEncoder");
- encoderObj = getEncoderMethod.invoke(null);
- encodeMethod = encoderObj.getClass().getMethod("encodeToString", byte[].class);
- } catch (Throwable e) {
- decodeMethod = null;
- encodeMethod = null;
- }
-
- if (encodeMethod == null) {
- decoderObj = null;
- encoderObj = null;
- try {
- Class javaxXmlBindDatatypeConverterClass = Class.forName("javax.xml.bind.DatatypeConverter");
- decodeMethod = javaxXmlBindDatatypeConverterClass.getMethod("parseBase64Binary", String.class);
- encodeMethod = javaxXmlBindDatatypeConverterClass.getMethod("printBase64Binary", byte[].class);
- } catch (Throwable e) {
- decodeMethod = null;
- encodeMethod = null;
- }
- }
- }
-}
diff --git a/core/src/main/java/io/questdb/client/std/BiIntFunction.java b/core/src/main/java/io/questdb/client/std/BiIntFunction.java
deleted file mode 100644
index 5de884f..0000000
--- a/core/src/main/java/io/questdb/client/std/BiIntFunction.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*******************************************************************************
- * ___ _ ____ ____
- * / _ \ _ _ ___ ___| |_| _ \| __ )
- * | | | | | | |/ _ \/ __| __| | | | _ \
- * | |_| | |_| | __/\__ \ |_| |_| | |_) |
- * \__\_\\__,_|\___||___/\__|____/|____/
- *
- * Copyright (c) 2014-2019 Appsicle
- * Copyright (c) 2019-2026 QuestDB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- ******************************************************************************/
-
-package io.questdb.client.std;
-
-@FunctionalInterface
-public interface BiIntFunction Retrieval operations (including {@code get}) generally do not
- * block, so may overlap with update operations (including {@code put}
- * and {@code remove}). Retrievals reflect the results of the most
- * recently completed update operations holding upon their
- * onset. (More formally, an update operation for a given key bears a
- * happens-before relation with any (non-null) retrieval for
- * that key reporting the updated value.) For aggregate operations
- * such as {@code putAll} and {@code clear}, concurrent retrievals may
- * reflect insertion or removal of only some entries. Similarly,
- * Iterators, Spliterators and Enumerations return elements reflecting the
- * state of the hash table at some point at or since the creation of the
- * iterator/enumeration. They do not throw {@link
- * java.util.ConcurrentModificationException ConcurrentModificationException}.
- * However, iterators are designed to be used by only one thread at a time.
- * Bear in mind that the results of aggregate status methods including
- * {@code size}, {@code isEmpty}, and {@code containsValue} are typically
- * useful only when a map is not undergoing concurrent updates in other threads.
- * Otherwise the results of these methods reflect transient states
- * that may be adequate for monitoring or estimation purposes, but not
- * for program control.
- * The table is dynamically expanded when there are too many
- * collisions (i.e., keys that have distinct hash codes but fall into
- * the same slot modulo the table size), with the expected average
- * effect of maintaining roughly two bins per mapping (corresponding
- * to a 0.75 load factor threshold for resizing). There may be much
- * variance around this average as mappings are added and removed, but
- * overall, this maintains a commonly accepted time/space tradeoff for
- * hash tables. However, resizing this or any other kind of hash
- * table may be a relatively slow operation. When possible, it is a
- * good idea to provide a size estimate as an optional {@code
- * initialCapacity} constructor argument. An additional optional
- * {@code loadFactor} constructor argument provides a further means of
- * customizing initial table capacity by specifying the table density
- * to be used in calculating the amount of space to allocate for the
- * given number of elements. Also, for compatibility with previous
- * versions of this class, constructors may optionally specify an
- * expected {@code concurrencyLevel} as an additional hint for
- * internal sizing. Note that using many keys with exactly the same
- * {@code hashCode()} is a sure way to slow down performance of any
- * hash table. To ameliorate impact, when keys are {@link Comparable},
- * this class may use comparison order among keys to help break ties.
- * A {@link Set} projection of a ConcurrentHashMap may be created
- * (using {@link #newKeySet()} or {@link #newKeySet(int)}), or viewed
- * (using {@link #keySet(Object)} when only keys are of interest, and the
- * mapped values are (perhaps transiently) not used or all take the
- * same mapping value.
- * This class and its views and iterators implement all of the
- * optional methods of the {@link Map} and {@link Iterator}
- * interfaces.
- * Like {@link Hashtable} but unlike {@link HashMap}, this class
- * does not allow {@code null} to be used as a key or value.
- * ConcurrentHashMaps support a set of sequential and parallel bulk
- * operations that are designed
- * to be safely, and often sensibly, applied even with maps that are
- * being concurrently updated by other threads; for example, when
- * computing a snapshot summary of the values in a shared registry.
- * There are three kinds of operation, each with four forms, accepting
- * functions with Keys, Values, Entries, and (Key, Value) arguments
- * and/or return values. Because the elements of a ConcurrentHashMap
- * are not ordered in any particular way, and may be processed in
- * different orders in different parallel executions, the correctness
- * of supplied functions should not depend on any ordering, or on any
- * other objects or values that may transiently change while
- * computation is in progress; and except for forEach actions, should
- * ideally be side-effect-free. Bulk operations on {@link java.util.Map.Entry}
- * objects do not support method {@code setValue}.
- * The concurrency properties of bulk operations follow
- * from those of ConcurrentHashMap: Any non-null result returned
- * from {@code get(key)} and related access methods bears a
- * happens-before relation with the associated insertion or
- * update. The result of any bulk operation reflects the
- * composition of these per-element relations (but is not
- * necessarily atomic with respect to the map as a whole unless it
- * is somehow known to be quiescent). Conversely, because keys
- * and values in the map are never null, null serves as a reliable
- * atomic indicator of the current lack of any result. To
- * maintain this property, null serves as an implicit basis for
- * all non-scalar reduction operations. For the double, long, and
- * int versions, the basis should be one that, when combined with
- * any other value, returns that other value (more formally, it
- * should be the identity element for the reduction). Most common
- * reductions have these properties; for example, computing a sum
- * with basis 0 or a minimum with basis MAX_VALUE.
- * Search and transformation functions provided as arguments
- * should similarly return null to indicate the lack of any result
- * (in which case it is not used). In the case of mapped
- * reductions, this also enables transformations to serve as
- * filters, returning null (or, in the case of primitive
- * specializations, the identity basis) if the element should not
- * be combined. You can create compound transformations and
- * filterings by composing them yourself under this "null means
- * there is nothing there now" rule before using them in search or
- * reduce operations.
- * Methods accepting and/or returning Entry arguments maintain
- * key-value associations. They may be useful for example when
- * finding the key for the greatest value. Note that "plain" Entry
- * arguments can be supplied using {@code new
- * AbstractMap.SimpleEntry(k,v)}.
- * Bulk operations may complete abruptly, throwing an
- * exception encountered in the application of a supplied
- * function. Bear in mind when handling such exceptions that other
- * concurrently executing functions could also have thrown
- * exceptions, or would have done so if the first exception had
- * not occurred.
- * Speedups for parallel compared to sequential forms are common
- * but not guaranteed. Parallel operations involving brief functions
- * on small maps may execute more slowly than sequential forms if the
- * underlying work to parallelize the computation is more expensive
- * than the computation itself. Similarly, parallelization may not
- * lead to much actual parallelism if all processors are busy
- * performing unrelated tasks.
- * All arguments to all task methods must be non-null.
- * This class is a member of the
- *
- * Java Collections Framework.
- *
- * @param The view's iterators and spliterators are
- * weakly consistent.
- *
- * @return the set view
- */
- @NotNull
- public Set More formally, if this map contains a mapping from a key
- * {@code k} to a value {@code v} such that {@code key.equals(k)},
- * then this method returns {@code v}; otherwise it returns
- * {@code null}. (There can be at most one such mapping.)
- *
- * @param key map key value
- * @return value to which specified key is mapped
- * @throws NullPointerException if the specified key is null
- */
- @Override
- public V get(Object key) {
- if (key instanceof CharSequence) {
- return get((CharSequence) key);
- }
- return null;
- }
-
- public V get(CharSequence key) {
- Node The view's iterators and spliterators are
- * weakly consistent.
- *
- *
- * @return the set view
- */
- @NotNull
- public KeySetView The value can be retrieved by calling the {@code get} method
- * with a key that is equal to the original key.
- *
- * @param key key with which the specified value is to be associated
- * @param value value to be associated with the specified key
- * @return the previous value associated with {@code key}, or
- * {@code null} if there was no mapping for {@code key}
- * @throws NullPointerException if the specified key or value is null
- */
- public V put(CharSequence key, V value) {
- return putVal(key, value, false);
- }
-
- /**
- * Copies all of the mappings from the specified map to this one.
- * These mappings replace any mappings that this map had for any of the
- * keys currently in the specified map.
- *
- * @param m mappings to be stored in this map
- */
- public void putAll(@NotNull Map m) {
- tryPresize(m.size());
- for (Map.Entry e : m.entrySet())
- putVal(e.getKey(), e.getValue(), false);
- }
-
- /**
- * {@inheritDoc}
- *
- * @return the previous value associated with the specified key,
- * or {@code null} if there was no mapping for the key
- * @throws NullPointerException if the specified key or value is null
- */
- public V putIfAbsent(@NotNull CharSequence key, V value) {
- return putVal(key, value, true);
- }
-
- /**
- * {@inheritDoc}
- *
- * @throws NullPointerException if the specified key is null
- */
- @SuppressWarnings("unchecked")
- public boolean remove(@NotNull Object key, Object value) {
- return value != null && replaceNode((CharSequence) key, null, (V) value) != null;
- }
- // Hashtable legacy methods
-
- /**
- * Removes the key (and its corresponding value) from this map.
- * This method does nothing if the key is not in the map.
- *
- * @param key the key that needs to be removed
- * @return the previous value associated with {@code key}, or
- * {@code null} if there was no mapping for {@code key}
- * @throws NullPointerException if the specified key is null
- */
- public V remove(CharSequence key) {
- return replaceNode(key, null, null);
- }
-
- // ConcurrentHashMap-only methods
-
- /**
- * {@inheritDoc}
- *
- * @throws NullPointerException if any of the arguments are null
- */
- public boolean replace(@NotNull CharSequence key, @NotNull V oldValue, @NotNull V newValue) {
- return replaceNode(key, newValue, oldValue) != null;
- }
-
- /**
- * {@inheritDoc}
- *
- * @return the previous value associated with the specified key,
- * or {@code null} if there was no mapping for the key
- * @throws NullPointerException if the specified key or value is null
- */
- public V replace(@NotNull CharSequence key, @NotNull V value) {
- return replaceNode(key, value, null);
- }
-
- /**
- * {@inheritDoc}
- */
- public int size() {
- long n = sumCount();
- return ((n < 0L) ? 0 :
- (n > (long) Integer.MAX_VALUE) ? Integer.MAX_VALUE :
- (int) n);
- }
-
- /**
- * Returns a string representation of this map. The string
- * representation consists of a list of key-value mappings (in no
- * particular order) enclosed in braces ("{@code {}}"). Adjacent
- * mappings are separated by the characters {@code ", "} (comma
- * and space). Each key-value mapping is rendered as the key
- * followed by an equals sign ("{@code =}") followed by the
- * associated value.
- *
- * @return a string representation of this map
- */
- public String toString() {
- Traverser The view's iterators and spliterators are
- * weakly consistent.
- *
- * @return the collection view
- */
- @NotNull
- public Collection The returned iterator is
- * weakly consistent.
- *
- * @return an iterator over the elements in this collection
- */
- @NotNull
- public abstract Iterator
+ * In async mode, rows are batched and sent asynchronously with flow control.
+ * This provides higher throughput at the cost of more complex error handling.
+ *
+ * This is only used when communicating over WebSocket transport.
+ *
+ * Default is synchronous mode (false).
+ *
+ * @param enabled whether to enable async mode
+ * @return this instance for method chaining
+ */
+ public LineSenderBuilder asyncMode(boolean enabled) {
+ this.asyncMode = enabled;
+ return this;
+ }
+
+ /**
+ * Set the maximum number of bytes per batch before auto-flushing.
+ *
+ * This is only used when communicating over WebSocket transport.
+ *
+ * Default value is 1MB.
+ *
+ * @param bytes maximum bytes per batch
+ * @return this instance for method chaining
+ */
+ public LineSenderBuilder autoFlushBytes(int bytes) {
+ if (this.autoFlushBytes != PARAMETER_NOT_SET_EXPLICITLY) {
+ throw new LineSenderException("auto flush bytes was already configured")
+ .put("[bytes=").put(this.autoFlushBytes).put("]");
+ }
+ if (bytes < 0) {
+ throw new LineSenderException("auto flush bytes cannot be negative")
+ .put("[bytes=").put(bytes).put("]");
+ }
+ this.autoFlushBytes = bytes;
+ return this;
+ }
+
/**
* Set the interval in milliseconds at which the Sender automatically flushes its buffer.
*
@@ -791,6 +855,40 @@ public Sender build() {
username, password, maxNameLength, actualMaxRetriesNanos, maxBackoffMillis, actualMinRequestThroughput, actualAutoFlushIntervalMillis, protocolVersion);
}
+ if (protocol == PROTOCOL_WEBSOCKET) {
+ if (hosts.size() != 1 || ports.size() != 1) {
+ throw new LineSenderException("only a single address (host:port) is supported for WebSocket transport");
+ }
+
+ int actualAutoFlushRows = autoFlushRows == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_WS_AUTO_FLUSH_ROWS : autoFlushRows;
+ int actualAutoFlushBytes = autoFlushBytes == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_WS_AUTO_FLUSH_BYTES : autoFlushBytes;
+ long actualAutoFlushIntervalNanos = autoFlushIntervalMillis == PARAMETER_NOT_SET_EXPLICITLY
+ ? DEFAULT_WS_AUTO_FLUSH_INTERVAL_NANOS
+ : TimeUnit.MILLISECONDS.toNanos(autoFlushIntervalMillis);
+ int actualInFlightWindowSize = inFlightWindowSize == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_IN_FLIGHT_WINDOW_SIZE : inFlightWindowSize;
+
+ if (asyncMode) {
+ return QwpWebSocketSender.connectAsync(
+ hosts.getQuick(0),
+ ports.getQuick(0),
+ tlsEnabled,
+ actualAutoFlushRows,
+ actualAutoFlushBytes,
+ actualAutoFlushIntervalNanos,
+ actualInFlightWindowSize
+ );
+ } else {
+ return QwpWebSocketSender.connect(
+ hosts.getQuick(0),
+ ports.getQuick(0),
+ tlsEnabled,
+ actualAutoFlushRows,
+ actualAutoFlushBytes,
+ actualAutoFlushIntervalNanos
+ );
+ }
+ }
+
assert protocol == PROTOCOL_TCP;
if (hosts.size() != 1 || ports.size() != 1) {
@@ -1048,6 +1146,29 @@ public LineSenderBuilder httpUsernamePassword(String username, String password)
return this;
}
+ /**
+ * Set the maximum number of batches that can be in-flight awaiting server acknowledgment.
+ *
+ * This is only used when communicating over WebSocket transport with async mode enabled.
+ *
+ * Default value is 8.
+ *
+ * @param size maximum number of in-flight batches
+ * @return this instance for method chaining
+ */
+ public LineSenderBuilder inFlightWindowSize(int size) {
+ if (this.inFlightWindowSize != PARAMETER_NOT_SET_EXPLICITLY) {
+ throw new LineSenderException("in-flight window size was already configured")
+ .put("[size=").put(this.inFlightWindowSize).put("]");
+ }
+ if (size < 1) {
+ throw new LineSenderException("in-flight window size must be positive")
+ .put("[size=").put(size).put("]");
+ }
+ this.inFlightWindowSize = size;
+ return this;
+ }
+
/**
* Configures the maximum backoff time between retry attempts when the Sender encounters recoverable errors.
*
@@ -1275,7 +1396,13 @@ private void configureDefaults() {
maximumBufferCapacity = protocol == PROTOCOL_HTTP ? DEFAULT_MAXIMUM_BUFFER_CAPACITY : bufferCapacity;
}
if (ports.size() == 0) {
- ports.add(protocol == PROTOCOL_HTTP ? DEFAULT_HTTP_PORT : DEFAULT_TCP_PORT);
+ if (protocol == PROTOCOL_HTTP) {
+ ports.add(DEFAULT_HTTP_PORT);
+ } else if (protocol == PROTOCOL_WEBSOCKET) {
+ ports.add(DEFAULT_WEBSOCKET_PORT);
+ } else {
+ ports.add(DEFAULT_TCP_PORT);
+ }
}
if (tlsValidationMode == null) {
tlsValidationMode = TlsValidationMode.DEFAULT;
@@ -1287,7 +1414,7 @@ private void configureDefaults() {
if (maxNameLength == PARAMETER_NOT_SET_EXPLICITLY) {
maxNameLength = DEFAULT_MAX_NAME_LEN;
}
- if (maxBackoffMillis == PARAMETER_NOT_SET_EXPLICITLY) {
+ if (maxBackoffMillis == PARAMETER_NOT_SET_EXPLICITLY && protocol == PROTOCOL_HTTP) {
maxBackoffMillis = DEFAULT_MAX_BACKOFF_MILLIS;
}
}
@@ -1334,8 +1461,16 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) {
} else if (Chars.equals("tcps", sink)) {
tcp();
tlsEnabled = true;
+ } else if (Chars.equals("ws", sink)) {
+ if (tlsEnabled) {
+ throw new LineSenderException("cannot use ws protocol when TLS is enabled. use wss instead");
+ }
+ websocket();
+ } else if (Chars.equals("wss", sink)) {
+ websocket();
+ tlsEnabled = true;
} else {
- throw new LineSenderException("invalid schema [schema=").put(sink).put(", supported-schemas=[http, https, tcp, tcps]]");
+ throw new LineSenderException("invalid schema [schema=").put(sink).put(", supported-schemas=[http, https, tcp, tcps, ws, wss]]");
}
String tcpToken = null;
@@ -1357,7 +1492,9 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) {
address(sink);
if (ports.size() == hosts.size() - 1) {
// not set
- port(protocol == PROTOCOL_TCP ? DEFAULT_TCP_PORT : DEFAULT_HTTP_PORT);
+ port(protocol == PROTOCOL_TCP ? DEFAULT_TCP_PORT
+ : protocol == PROTOCOL_WEBSOCKET ? DEFAULT_WEBSOCKET_PORT
+ : DEFAULT_HTTP_PORT);
}
} else if (Chars.equals("user", sink)) {
// deprecated key: user, new key: username
@@ -1414,7 +1551,7 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) {
} else if (protocol == PROTOCOL_HTTP) {
httpToken(sink.toString());
} else {
- throw new AssertionError();
+ throw new LineSenderException("token is not supported for WebSocket protocol");
}
} else if (Chars.equals("retry_timeout", sink)) {
pos = getValue(configurationString, pos, sink, "retry_timeout");
@@ -1617,12 +1754,52 @@ private void validateParameters() {
if (autoFlushIntervalMillis != PARAMETER_NOT_SET_EXPLICITLY) {
throw new LineSenderException("auto flush interval is not supported for TCP protocol");
}
+ } else if (protocol == PROTOCOL_WEBSOCKET) {
+ if (privateKey != null) {
+ throw new LineSenderException("TCP authentication is not supported for WebSocket protocol");
+ }
+ if (httpToken != null || username != null || password != null) {
+ // TODO: WebSocket auth not yet implemented
+ throw new LineSenderException("Authentication is not yet supported for WebSocket protocol");
+ }
+ if (inFlightWindowSize != PARAMETER_NOT_SET_EXPLICITLY && !asyncMode) {
+ throw new LineSenderException("in-flight window size requires async mode");
+ }
+ if (httpPath != null) {
+ throw new LineSenderException("HTTP path is not supported for WebSocket protocol");
+ }
+ if (httpTimeout != PARAMETER_NOT_SET_EXPLICITLY) {
+ throw new LineSenderException("HTTP timeout is not supported for WebSocket protocol");
+ }
+ if (retryTimeoutMillis != PARAMETER_NOT_SET_EXPLICITLY) {
+ throw new LineSenderException("retry timeout is not supported for WebSocket protocol");
+ }
+ if (minRequestThroughput != PARAMETER_NOT_SET_EXPLICITLY) {
+ throw new LineSenderException("minimum request throughput is not supported for WebSocket protocol");
+ }
+ if (maxBackoffMillis != PARAMETER_NOT_SET_EXPLICITLY) {
+ throw new LineSenderException("max backoff is not supported for WebSocket protocol");
+ }
+ if (protocolVersion != PARAMETER_NOT_SET_EXPLICITLY) {
+ throw new LineSenderException("protocol version is not supported for WebSocket protocol");
+ }
+ if (autoFlushIntervalMillis == Integer.MAX_VALUE) {
+ throw new LineSenderException("disabling auto-flush is not supported for WebSocket protocol");
+ }
} else {
throw new LineSenderException("unsupported protocol ")
.put("[protocol=").put(protocol).put("]");
}
}
+ private void websocket() {
+ if (protocol != PARAMETER_NOT_SET_EXPLICITLY) {
+ throw new LineSenderException("protocol was already configured ")
+ .put("[protocol=").put(protocol).put("]");
+ }
+ protocol = PROTOCOL_WEBSOCKET;
+ }
+
public class AdvancedTlsSettings {
/**
* Configure a custom truststore. This is only needed when using {@link #enableTls()} when your default
diff --git a/core/src/main/java/io/questdb/client/cairo/ColumnType.java b/core/src/main/java/io/questdb/client/cairo/ColumnType.java
index b00046f..275650b 100644
--- a/core/src/main/java/io/questdb/client/cairo/ColumnType.java
+++ b/core/src/main/java/io/questdb/client/cairo/ColumnType.java
@@ -214,19 +214,6 @@ public static int encodeArrayType(int elemType, int nDims, boolean checkSupporte
| ARRAY;
}
- /**
- * Encodes an array type with weak dimensionality. The dimensionality is still
- * encoded but marked as tentative and can be updated based on actual data.
- * This is useful for PostgreSQL wire protocol where type information doesn't
- * include array dimensions.
- *
+ *
+ *
+ * WebSocketSendBuffer buf = client.getSendBuffer();
+ * buf.beginBinaryFrame();
+ * buf.putLong(data);
+ * WebSocketSendBuffer.FrameInfo frame = buf.endBinaryFrame();
+ * client.sendFrame(frame, timeout);
+ * buf.reset();
+ *
+ */
+ public WebSocketSendBuffer getSendBuffer() {
+ return sendBuffer;
+ }
+
+ /**
+ * Returns whether the WebSocket is connected and upgraded.
+ */
+ public boolean isConnected() {
+ return upgraded && !closed && !socket.isClosed();
+ }
+
+ /**
+ * Receives and processes WebSocket frames.
+ *
+ * @param handler frame handler callback
+ * @param timeout timeout in milliseconds
+ * @return true if a frame was received, false on timeout
+ */
+ public boolean receiveFrame(WebSocketFrameHandler handler, int timeout) {
+ checkConnected();
+
+ // First, try to parse any data already in buffer
+ Boolean result = tryParseFrame(handler);
+ if (result != null) {
+ return result;
+ }
+
+ // Need more data
+ long startTime = System.nanoTime();
+ while (true) {
+ int remainingTimeout = remainingTime(timeout, startTime);
+ if (remainingTimeout <= 0) {
+ return false; // Timeout
+ }
+
+ // Ensure buffer has space
+ if (recvPos >= recvBufSize - 1024) {
+ growRecvBuffer();
+ }
+
+ int bytesRead = recvOrTimeout(recvBufPtr + recvPos, recvBufSize - recvPos, remainingTimeout);
+ if (bytesRead <= 0) {
+ return false; // Timeout
+ }
+ recvPos += bytesRead;
+
+ result = tryParseFrame(handler);
+ if (result != null) {
+ return result;
+ }
+ }
+ }
+
+ /**
+ * Sends binary data as a WebSocket binary frame.
+ *
+ * @param dataPtr pointer to data
+ * @param length data length
+ * @param timeout timeout in milliseconds
+ */
+ public void sendBinary(long dataPtr, int length, int timeout) {
+ checkConnected();
+ sendBuffer.reset();
+ sendBuffer.beginFrame();
+ sendBuffer.putBlockOfBytes(dataPtr, length);
+ WebSocketSendBuffer.FrameInfo frame = sendBuffer.endBinaryFrame();
+ doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout);
+ sendBuffer.reset();
+ }
+
+ /**
+ * Sends binary data with default timeout.
+ */
+ public void sendBinary(long dataPtr, int length) {
+ sendBinary(dataPtr, length, defaultTimeout);
+ }
+
+ /**
+ * Sends a close frame.
+ */
+ public void sendCloseFrame(int code, String reason, int timeout) {
+ controlFrameBuffer.reset();
+ WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writeCloseFrame(code, reason);
+ try {
+ doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, timeout);
+ } finally {
+ controlFrameBuffer.reset();
+ }
+ }
+
+ /**
+ * Sends a ping frame.
+ */
+ public void sendPing(int timeout) {
+ checkConnected();
+ controlFrameBuffer.reset();
+ WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writePingFrame();
+ doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, timeout);
+ controlFrameBuffer.reset();
+ }
+
+ /**
+ * Non-blocking attempt to receive a WebSocket frame.
+ * Returns immediately if no complete frame is available.
+ *
+ * @param handler frame handler callback
+ * @return true if a frame was received, false if no data available
+ */
+ public boolean tryReceiveFrame(WebSocketFrameHandler handler) {
+ checkConnected();
+
+ // First, try to parse any data already in buffer
+ Boolean result = tryParseFrame(handler);
+ if (result != null) {
+ return result;
+ }
+
+ // Try one non-blocking recv
+ if (recvPos >= recvBufSize - 1024) {
+ growRecvBuffer();
+ }
+
+ int n = socket.recv(recvBufPtr + recvPos, recvBufSize - recvPos);
+ if (n < 0) {
+ throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']');
+ }
+ if (n == 0) {
+ return false; // No data available
+ }
+ recvPos += n;
+
+ // Try to parse again
+ result = tryParseFrame(handler);
+ return result != null && result;
+ }
+
+ /**
+ * Performs WebSocket upgrade handshake.
+ *
+ * @param path the WebSocket endpoint path (e.g., "/ws")
+ * @param timeout timeout in milliseconds
+ */
+ public void upgrade(CharSequence path, int timeout) {
+ if (closed) {
+ throw new HttpClientException("WebSocket client is closed");
+ }
+ if (socket.isClosed()) {
+ throw new HttpClientException("Not connected");
+ }
+ if (upgraded) {
+ return; // Already upgraded
+ }
+
+ // Generate random key
+ byte[] keyBytes = new byte[16];
+ for (int i = 0; i < 16; i++) {
+ keyBytes[i] = (byte) rnd.nextInt();
+ }
+ handshakeKey = Base64.getEncoder().encodeToString(keyBytes);
+
+ // Build upgrade request
+ sendBuffer.reset();
+ sendBuffer.putAscii("GET ");
+ sendBuffer.putAscii(path);
+ sendBuffer.putAscii(" HTTP/1.1\r\n");
+ sendBuffer.putAscii("Host: ");
+ sendBuffer.putAscii(host);
+ if ((socket.supportsTls() && port != 443) || (!socket.supportsTls() && port != 80)) {
+ sendBuffer.putAscii(":");
+ sendBuffer.putAscii(Integer.toString(port));
+ }
+ sendBuffer.putAscii("\r\n");
+ sendBuffer.putAscii("Upgrade: websocket\r\n");
+ sendBuffer.putAscii("Connection: Upgrade\r\n");
+ sendBuffer.putAscii("Sec-WebSocket-Key: ");
+ sendBuffer.putAscii(handshakeKey);
+ sendBuffer.putAscii("\r\n");
+ sendBuffer.putAscii("Sec-WebSocket-Version: 13\r\n");
+ sendBuffer.putAscii("\r\n");
+
+ // Send request
+ long startTime = System.nanoTime();
+ doSend(sendBuffer.getBufferPtr(), sendBuffer.getWritePos(), timeout);
+
+ // Read response
+ int remainingTimeout = getRemainingTimeOrThrow(timeout, startTime);
+ readUpgradeResponse(remainingTimeout);
+
+ upgraded = true;
+ sendBuffer.reset();
+ LOG.debug("WebSocket upgraded [path={}]", path);
+ }
+
+ /**
+ * Performs upgrade with default timeout.
+ */
+ public void upgrade(CharSequence path) {
+ upgrade(path, defaultTimeout);
+ }
+
+ private static String computeAcceptKey(String key) {
+ MessageDigest sha1 = SHA1_DIGEST.get();
+ sha1.reset();
+ for (int i = 0, n = key.length(); i < n; i++) {
+ sha1.update((byte) key.charAt(i));
+ }
+ sha1.update(WEBSOCKET_GUID_BYTES);
+ return Base64.getEncoder().encodeToString(sha1.digest());
+ }
+
+ private static boolean containsHeaderValue(String response, String headerName, String expectedValue, boolean ignoreValueCase) {
+ int headerLen = headerName.length();
+ int responseLen = response.length();
+ for (int i = 0; i <= responseLen - headerLen; i++) {
+ if (response.regionMatches(true, i, headerName, 0, headerLen)) {
+ int valueStart = i + headerLen;
+ int lineEnd = response.indexOf('\r', valueStart);
+ if (lineEnd < 0) {
+ lineEnd = responseLen;
+ }
+ String actualValue = response.substring(valueStart, lineEnd).trim();
+ return ignoreValueCase
+ ? actualValue.equalsIgnoreCase(expectedValue)
+ : actualValue.equals(expectedValue);
+ }
+ }
+ return false;
+ }
+
+ private static int remainingTime(int timeoutMillis, long startTimeNanos) {
+ return timeoutMillis - (int) NANOSECONDS.toMillis(System.nanoTime() - startTimeNanos);
+ }
+
+ private void appendToFragmentBuffer(long payloadPtr, int payloadLen) {
+ if (payloadLen == 0) {
+ return;
+ }
+ int required = fragmentBufPos + payloadLen;
+ if (required > maxRecvBufSize) {
+ throw new HttpClientException("WebSocket fragment buffer size exceeded maximum [required=")
+ .put(required)
+ .put(", max=")
+ .put(maxRecvBufSize)
+ .put(']');
+ }
+ if (fragmentBufPtr == 0) {
+ fragmentBufSize = Math.max(required, DEFAULT_RECV_BUFFER_SIZE);
+ fragmentBufPtr = Unsafe.malloc(fragmentBufSize, MemoryTag.NATIVE_DEFAULT);
+ } else if (required > fragmentBufSize) {
+ int newSize = (int) Math.min(Math.max((long) fragmentBufSize * 2, required), maxRecvBufSize);
+ fragmentBufPtr = Unsafe.realloc(fragmentBufPtr, fragmentBufSize, newSize, MemoryTag.NATIVE_DEFAULT);
+ fragmentBufSize = newSize;
+ }
+ Vect.memmove(fragmentBufPtr + fragmentBufPos, payloadPtr, payloadLen);
+ fragmentBufPos += payloadLen;
+ }
+
+ private void checkConnected() {
+ if (closed) {
+ throw new HttpClientException("WebSocket client is closed");
+ }
+ if (!upgraded) {
+ throw new HttpClientException("WebSocket not connected or upgraded");
+ }
+ }
+
+ private void compactRecvBuffer() {
+ if (recvReadPos > 0) {
+ int remaining = recvPos - recvReadPos;
+ if (remaining > 0) {
+ Vect.memmove(recvBufPtr, recvBufPtr + recvReadPos, remaining);
+ }
+ recvPos = remaining;
+ recvReadPos = 0;
+ }
+ }
+
+ private int dieIfNegative(int byteCount) {
+ if (byteCount < 0) {
+ throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']');
+ }
+ return byteCount;
+ }
+
+ private void doConnect(CharSequence host, int port, int timeout) {
+ int fd = nf.socketTcp(true);
+ if (fd < 0) {
+ throw new HttpClientException("could not allocate a file descriptor [errno=").errno(nf.errno()).put(']');
+ }
+
+ if (nf.setTcpNoDelay(fd, true) < 0) {
+ LOG.info("could not disable Nagle's algorithm [fd={}, errno={}]", fd, nf.errno());
+ }
+
+ socket.of(fd);
+ nf.configureKeepAlive(fd);
+
+ long addrInfo = nf.getAddrInfo(host, port);
+ if (addrInfo == -1) {
+ disconnect();
+ throw new HttpClientException("could not resolve host [host=").put(host).put(']');
+ }
+
+ if (nf.connectAddrInfo(fd, addrInfo) != 0) {
+ int errno = nf.errno();
+ nf.freeAddrInfo(addrInfo);
+ disconnect();
+ throw new HttpClientException("could not connect [host=").put(host)
+ .put(", port=").put(port)
+ .put(", errno=").put(errno).put(']');
+ }
+ nf.freeAddrInfo(addrInfo);
+
+ if (nf.configureNonBlocking(fd) < 0) {
+ int errno = nf.errno();
+ disconnect();
+ throw new HttpClientException("could not configure non-blocking [fd=").put(fd)
+ .put(", errno=").put(errno).put(']');
+ }
+
+ if (socket.supportsTls()) {
+ try {
+ socket.startTlsSession(host);
+ } catch (TlsSessionInitFailedException e) {
+ int errno = nf.errno();
+ disconnect();
+ throw new HttpClientException("could not start TLS session [fd=").put(fd)
+ .put(", error=").put(e.getFlyweightMessage())
+ .put(", errno=").put(errno).put(']');
+ }
+ }
+
+ setupIoWait();
+ LOG.debug("Connected to [host={}, port={}]", host, port);
+ }
+
+ private void doSend(long ptr, int len, int timeout) {
+ long startTime = System.nanoTime();
+ while (len > 0) {
+ int remainingTimeout = getRemainingTimeOrThrow(timeout, startTime);
+ ioWait(remainingTimeout, IOOperation.WRITE);
+ int sent = dieIfNegative(socket.send(ptr, len));
+ while (socket.wantsTlsWrite()) {
+ remainingTimeout = getRemainingTimeOrThrow(timeout, startTime);
+ ioWait(remainingTimeout, IOOperation.WRITE);
+ dieIfNegative(socket.tlsIO(Socket.WRITE_FLAG));
+ }
+ if (sent > 0) {
+ ptr += sent;
+ len -= sent;
+ }
+ }
+ }
+
+ private int findHeaderEnd() {
+ // Look for \r\n\r\n
+ for (int i = 0; i < recvPos - 3; i++) {
+ if (Unsafe.getUnsafe().getByte(recvBufPtr + i) == '\r' &&
+ Unsafe.getUnsafe().getByte(recvBufPtr + i + 1) == '\n' &&
+ Unsafe.getUnsafe().getByte(recvBufPtr + i + 2) == '\r' &&
+ Unsafe.getUnsafe().getByte(recvBufPtr + i + 3) == '\n') {
+ return i + 4;
+ }
+ }
+ return -1;
+ }
+
+ private int getRemainingTimeOrThrow(int timeoutMillis, long startTimeNanos) {
+ int remaining = remainingTime(timeoutMillis, startTimeNanos);
+ if (remaining <= 0) {
+ throw new HttpClientException("timed out [errno=").errno(nf.errno()).put(']').flagAsTimeout();
+ }
+ return remaining;
+ }
+
+ private void growRecvBuffer() {
+ int newSize = (int) Math.min((long) recvBufSize * 2, maxRecvBufSize);
+ if (newSize >= maxRecvBufSize) {
+ if (recvBufSize >= maxRecvBufSize) {
+ throw new HttpClientException("WebSocket receive buffer size exceeded maximum [current=")
+ .put(recvBufSize)
+ .put(", max=")
+ .put(maxRecvBufSize)
+ .put(']');
+ }
+ newSize = maxRecvBufSize;
+ }
+ recvBufPtr = Unsafe.realloc(recvBufPtr, recvBufSize, newSize, MemoryTag.NATIVE_DEFAULT);
+ recvBufSize = newSize;
+ }
+
+ private void readUpgradeResponse(int timeout) {
+ // Read HTTP response into receive buffer
+ long startTime = System.nanoTime();
+
+ while (true) {
+ int remainingTimeout = getRemainingTimeOrThrow(timeout, startTime);
+ int bytesRead = recvOrDie(recvBufPtr + recvPos, recvBufSize - recvPos, remainingTimeout);
+ if (bytesRead > 0) {
+ recvPos += bytesRead;
+ }
+
+ // Check for end of headers (\r\n\r\n)
+ int headerEnd = findHeaderEnd();
+ if (headerEnd > 0) {
+ validateUpgradeResponse(headerEnd);
+ // Compact buffer - move remaining data to start
+ int remaining = recvPos - headerEnd;
+ if (remaining > 0) {
+ Vect.memmove(recvBufPtr, recvBufPtr + headerEnd, remaining);
+ }
+ recvPos = remaining;
+ recvReadPos = 0;
+ return;
+ }
+
+ if (recvPos >= recvBufSize) {
+ throw new HttpClientException("HTTP response too large");
+ }
+ }
+ }
+
+ private int recvOrDie(long ptr, int len, int timeout) {
+ long startTime = System.nanoTime();
+ int n = dieIfNegative(socket.recv(ptr, len));
+ if (n == 0) {
+ ioWait(getRemainingTimeOrThrow(timeout, startTime), IOOperation.READ);
+ n = dieIfNegative(socket.recv(ptr, len));
+ }
+ return n;
+ }
+
+ private int recvOrTimeout(long ptr, int len, int timeout) {
+ long startTime = System.nanoTime();
+ int n = socket.recv(ptr, len);
+ if (n < 0) {
+ throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']');
+ }
+ if (n == 0) {
+ try {
+ ioWait(timeout, IOOperation.READ);
+ } catch (HttpClientException e) {
+ if (!e.isTimeout()) {
+ throw e;
+ }
+ return 0;
+ }
+ n = socket.recv(ptr, len);
+ if (n < 0) {
+ throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']');
+ }
+ }
+ return n;
+ }
+
+ private void resetFragmentState() {
+ fragmentOpcode = -1;
+ fragmentBufPos = 0;
+ }
+
+ private void sendCloseFrameEcho(int code) {
+ try {
+ controlFrameBuffer.reset();
+ WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writeCloseFrame(code, null);
+ doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, 1000);
+ controlFrameBuffer.reset();
+ } catch (Exception e) {
+ LOG.error("Failed to echo close frame: {}", e.getMessage());
+ }
+ }
+
+ private void sendPongFrame(long payloadPtr, int payloadLen) {
+ try {
+ controlFrameBuffer.reset();
+ WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writePongFrame(payloadPtr, payloadLen);
+ doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, 1000);
+ controlFrameBuffer.reset();
+ } catch (Exception e) {
+ LOG.error("Failed to send pong: {}", e.getMessage());
+ }
+ }
+
+ private Boolean tryParseFrame(WebSocketFrameHandler handler) {
+ if (recvPos <= recvReadPos) {
+ return null; // No data
+ }
+
+ frameParser.reset();
+ int consumed = frameParser.parse(recvBufPtr + recvReadPos, recvBufPtr + recvPos);
+
+ if (frameParser.getState() == WebSocketFrameParser.STATE_NEED_MORE ||
+ frameParser.getState() == WebSocketFrameParser.STATE_NEED_PAYLOAD) {
+ return null; // Need more data
+ }
+
+ if (frameParser.getState() == WebSocketFrameParser.STATE_ERROR) {
+ throw new HttpClientException("WebSocket frame parse error: ")
+ .put(WebSocketCloseCode.describe(frameParser.getErrorCode()));
+ }
+
+ if (frameParser.getState() == WebSocketFrameParser.STATE_COMPLETE) {
+ long payloadPtr = recvBufPtr + recvReadPos + frameParser.getHeaderSize();
+ long payloadLength = frameParser.getPayloadLength();
+ if (payloadLength > Integer.MAX_VALUE) {
+ throw new HttpClientException("WebSocket frame payload too large [length=")
+ .put(payloadLength).put(']');
+ }
+ int payloadLen = (int) payloadLength;
+
+ // Unmask if needed (server frames should not be masked)
+ if (frameParser.isMasked()) {
+ frameParser.unmaskPayload(payloadPtr, payloadLen);
+ }
+
+ // Handle frame by opcode
+ int opcode = frameParser.getOpcode();
+ switch (opcode) {
+ case WebSocketOpcode.PING:
+ // Auto-respond with pong
+ sendPongFrame(payloadPtr, payloadLen);
+ if (handler != null) {
+ handler.onPing(payloadPtr, payloadLen);
+ }
+ break;
+ case WebSocketOpcode.PONG:
+ if (handler != null) {
+ handler.onPong(payloadPtr, payloadLen);
+ }
+ break;
+ case WebSocketOpcode.CLOSE:
+ int closeCode = 0;
+ String reason = null;
+ if (payloadLen >= 2) {
+ closeCode = ((Unsafe.getUnsafe().getByte(payloadPtr) & 0xFF) << 8)
+ | (Unsafe.getUnsafe().getByte(payloadPtr + 1) & 0xFF);
+ if (payloadLen > 2) {
+ byte[] reasonBytes = new byte[payloadLen - 2];
+ for (int i = 0; i < reasonBytes.length; i++) {
+ reasonBytes[i] = Unsafe.getUnsafe().getByte(payloadPtr + 2 + i);
+ }
+ reason = new String(reasonBytes, StandardCharsets.UTF_8);
+ }
+ }
+ // RFC 6455 Section 5.5.1: echo a close frame back before
+ // marking the connection as no longer upgraded
+ sendCloseFrameEcho(closeCode);
+ upgraded = false;
+ if (handler != null) {
+ handler.onClose(closeCode, reason);
+ }
+ break;
+ case WebSocketOpcode.BINARY:
+ case WebSocketOpcode.TEXT:
+ if (frameParser.isFin()) {
+ if (fragmentOpcode != -1) {
+ throw new HttpClientException("WebSocket protocol error: new data frame during fragmented message");
+ }
+ if (handler != null) {
+ if (opcode == WebSocketOpcode.BINARY) {
+ handler.onBinaryMessage(payloadPtr, payloadLen);
+ } else {
+ handler.onTextMessage(payloadPtr, payloadLen);
+ }
+ }
+ } else {
+ if (fragmentOpcode != -1) {
+ throw new HttpClientException("WebSocket protocol error: new data frame during fragmented message");
+ }
+ fragmentOpcode = opcode;
+ appendToFragmentBuffer(payloadPtr, payloadLen);
+ }
+ break;
+ case WebSocketOpcode.CONTINUATION:
+ if (fragmentOpcode == -1) {
+ throw new HttpClientException("WebSocket protocol error: continuation frame without initial fragment");
+ }
+ appendToFragmentBuffer(payloadPtr, payloadLen);
+ if (frameParser.isFin()) {
+ if (handler != null) {
+ if (fragmentOpcode == WebSocketOpcode.BINARY) {
+ handler.onBinaryMessage(fragmentBufPtr, fragmentBufPos);
+ } else {
+ handler.onTextMessage(fragmentBufPtr, fragmentBufPos);
+ }
+ }
+ resetFragmentState();
+ }
+ break;
+ }
+
+ // Advance read position
+ recvReadPos += consumed;
+
+ // Compact buffer if needed
+ compactRecvBuffer();
+
+ return true;
+ }
+
+ return false;
+ }
+
+ private void validateUpgradeResponse(int headerEnd) {
+ // Extract response as string for parsing
+ byte[] responseBytes = new byte[headerEnd];
+ for (int i = 0; i < headerEnd; i++) {
+ responseBytes[i] = Unsafe.getUnsafe().getByte(recvBufPtr + i);
+ }
+ String response = new String(responseBytes, StandardCharsets.US_ASCII);
+
+ // Check status line
+ if (!response.startsWith("HTTP/1.1 101")) {
+ String statusLine = response.split("\r\n")[0];
+ throw new HttpClientException("WebSocket upgrade failed: ").put(statusLine);
+ }
+
+ // Verify Upgrade: websocket (case-insensitive value per RFC 6455 Section 4.1)
+ if (!containsHeaderValue(response, "Upgrade:", "websocket", true)) {
+ throw new HttpClientException("Missing or invalid Upgrade header in WebSocket response");
+ }
+
+ // Verify Connection: Upgrade (case-insensitive value per RFC 6455 Section 4.1)
+ if (!containsHeaderValue(response, "Connection:", "Upgrade", true)) {
+ throw new HttpClientException("Missing or invalid Connection header in WebSocket response");
+ }
+
+ // Verify Sec-WebSocket-Accept (exact value match per RFC 6455 Section 4.1)
+ String expectedAccept = computeAcceptKey(handshakeKey);
+ if (!containsHeaderValue(response, "Sec-WebSocket-Accept:", expectedAccept, false)) {
+ throw new HttpClientException("Invalid Sec-WebSocket-Accept header");
+ }
+ }
+
+ protected void dieWaiting(int n) {
+ if (n == 1) {
+ return;
+ }
+ if (n == 0) {
+ throw new HttpClientException("timed out [errno=").put(nf.errno()).put(']').flagAsTimeout();
+ }
+ throw new HttpClientException("queue error [errno=").put(nf.errno()).put(']');
+ }
+
+ /**
+ * Waits for I/O readiness using platform-specific mechanism.
+ *
+ * @param timeout timeout in milliseconds
+ * @param op I/O operation (READ or WRITE)
+ */
+ protected abstract void ioWait(int timeout, int op);
+
+ /**
+ * Sets up platform-specific I/O wait mechanism after connection.
+ */
+ protected abstract void setupIoWait();
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java
new file mode 100644
index 0000000..c6b36d2
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java
@@ -0,0 +1,136 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.http.client;
+
+import io.questdb.client.ClientTlsConfiguration;
+import io.questdb.client.DefaultHttpClientConfiguration;
+import io.questdb.client.HttpClientConfiguration;
+import io.questdb.client.network.JavaTlsClientSocketFactory;
+import io.questdb.client.network.PlainSocketFactory;
+import io.questdb.client.network.SocketFactory;
+import io.questdb.client.std.Os;
+
+/**
+ * Factory for creating platform-specific {@link WebSocketClient} instances.
+ *
+ * // Plain text connection
+ * WebSocketClient client = WebSocketClientFactory.newPlainTextInstance();
+ *
+ * // TLS connection
+ * WebSocketClient client = WebSocketClientFactory.newTlsInstance(config, tlsConfig);
+ *
+ * // Connect and upgrade
+ * client.connect("localhost", 9000);
+ * client.upgrade("/ws");
+ *
+ * // Send data
+ * WebSocketSendBuffer buf = client.getSendBuffer();
+ * buf.beginBinaryFrame();
+ * buf.putLong(data);
+ * WebSocketSendBuffer.FrameInfo frame = buf.endBinaryFrame();
+ * client.sendFrame(frame);
+ * buf.reset();
+ *
+ * // Receive data
+ * client.receiveFrame(handler);
+ *
+ * client.close();
+ *
+ */
+public class WebSocketClientFactory {
+
+ // Utility class -- no instantiation
+ private WebSocketClientFactory() {
+ }
+
+ /**
+ * Creates a new WebSocket client with insecure TLS (no certificate validation).
+ *
+ * buffer.beginBinaryFrame();
+ * // Write payload using ArrayBufferAppender methods
+ * buffer.putLong(value);
+ * buffer.putBlockOfBytes(ptr, len);
+ * // Finish frame and get send info
+ * FrameInfo frame = buffer.endBinaryFrame();
+ * // Send frame using socket
+ * socket.send(buffer.getBufferPtr() + frame.offset, frame.length);
+ * buffer.reset();
+ *
+ * Retryability
- * The {@link #isRetryable()} method provides a best-effort indication of whether the error
- * might be resolved by retrying at the application level. This is particularly important
- * because this exception is only thrown after the sender has exhausted its own internal
- * retry attempts. The retryability flag helps applications decide whether to implement
- * additional retry logic with longer delays or different strategies.
- *
- * @see io.questdb.client.Sender
* @see io.questdb.client.Sender#flush()
* @see io.questdb.client.Sender#reset()
*/
@@ -115,4 +109,4 @@ public LineSenderException putAsPrintable(CharSequence nonPrintable) {
message.putAsPrintable(nonPrintable);
return this;
}
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java
new file mode 100644
index 0000000..b8c1dfe
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java
@@ -0,0 +1,145 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client;
+
+import io.questdb.client.std.CharSequenceIntHashMap;
+import io.questdb.client.std.ObjList;
+
+/**
+ * Global symbol dictionary that maps symbol strings to sequential integer IDs.
+ *
+ *
+ * Assumptions that keep it simple and lock-free:
+ *
+ *
+ * With these constraints we can rely on volatile reads/writes (no CAS) and still
+ * offer blocking waits for space/empty without protecting the counters with locks.
+ */
+public class InFlightWindow {
+
+ public static final long DEFAULT_TIMEOUT_MS = 30_000;
+ public static final int DEFAULT_WINDOW_SIZE = 8;
+ private static final Logger LOG = LoggerFactory.getLogger(InFlightWindow.class);
+ private static final long PARK_NANOS = 100_000; // 100 microseconds
+ // Spin parameters
+ private static final int SPIN_TRIES = 100;
+ private static final VarHandle TOTAL_ACKED;
+ private static final VarHandle TOTAL_FAILED;
+
+ static {
+ try {
+ MethodHandles.Lookup lookup = MethodHandles.lookup();
+ TOTAL_ACKED = lookup.findVarHandle(InFlightWindow.class, "totalAcked", long.class);
+ TOTAL_FAILED = lookup.findVarHandle(InFlightWindow.class, "totalFailed", long.class);
+ } catch (ReflectiveOperationException e) {
+ throw new ExceptionInInitializerError(e);
+ }
+ }
+ // Error state
+ private final AtomicReference
+ * Buffer States:
+ * ┌─────────────┐ seal() ┌─────────────┐ markSending() ┌─────────────┐
+ * │ FILLING │──────────────►│ SEALED │──────────────────►│ SENDING │
+ * │ (user owns) │ │ (in queue) │ │ (I/O owns) │
+ * └─────────────┘ └─────────────┘ └──────┬──────┘
+ * ▲ │
+ * │ markRecycled() │
+ * └───────────────────────────────────────────────────────────────┘
+ * (after send complete)
+ *
+ *
+ *
+ *
+ *
+ *
+ *
+ *
+ * try (QwpWebSocketSender sender = QwpWebSocketSender.connect("localhost", 9000)) {
+ * for (int i = 0; i < 100_000; i++) {
+ * sender.table("metrics")
+ * .symbol("host", "server-" + (i % 10))
+ * .doubleColumn("cpu", Math.random() * 100)
+ * .atNow();
+ * // Rows are batched and sent asynchronously!
+ * }
+ * // flush() waits for all pending batches to be sent
+ * sender.flush();
+ * }
+ *
+ *
+ * // Setup (once)
+ * QwpTableBuffer tableBuffer = sender.getTableBuffer("q");
+ * QwpTableBuffer.ColumnBuffer colSymbol = tableBuffer.getOrCreateColumn("s", TYPE_SYMBOL, true);
+ * QwpTableBuffer.ColumnBuffer colBid = tableBuffer.getOrCreateColumn("b", TYPE_DOUBLE, false);
+ *
+ * // Hot path (per row)
+ * colSymbol.addSymbolWithGlobalId(symbol, sender.getOrAddGlobalSymbol(symbol));
+ * colBid.addDouble(bid);
+ * tableBuffer.nextRow();
+ * sender.incrementPendingRowCount();
+ *
+ */
+public class QwpWebSocketSender implements Sender {
+
+ public static final int DEFAULT_AUTO_FLUSH_BYTES = 1024 * 1024; // 1MB
+ public static final long DEFAULT_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms
+ public static final int DEFAULT_AUTO_FLUSH_ROWS = 500;
+ public static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = InFlightWindow.DEFAULT_WINDOW_SIZE; // 8
+ private static final int DEFAULT_BUFFER_SIZE = 8192;
+ private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB
+ private static final Logger LOG = LoggerFactory.getLogger(QwpWebSocketSender.class);
+ private static final String WRITE_PATH = "/write/v4";
+ private final AckFrameHandler ackHandler = new AckFrameHandler(this);
+ private final WebSocketResponse ackResponse = new WebSocketResponse();
+ private final int autoFlushBytes;
+ private final long autoFlushIntervalNanos;
+ // Auto-flush configuration
+ private final int autoFlushRows;
+ private final Decimal256 currentDecimal256 = new Decimal256();
+ // Encoder for ILP v4 messages
+ private final QwpWebSocketEncoder encoder;
+ // Global symbol dictionary for delta encoding
+ private final GlobalSymbolDictionary globalSymbolDictionary;
+ private final String host;
+ // Flow control configuration
+ private final int inFlightWindowSize;
+ private final int port;
+ // Track schema hashes that have been sent to the server (for schema reference mode)
+ // First time we send a schema: full schema. Subsequent times: 8-byte hash reference.
+ // Combined key = schemaHash XOR (tableNameHash << 32) to include table name in lookup.
+ private final LongHashSet sentSchemaHashes = new LongHashSet();
+ private final CharSequenceObjHashMap
+ * +--------+----------+------------------+
+ * | status | sequence | error (if any) |
+ * | 1 byte | 8 bytes | 2 bytes + UTF-8 |
+ * +--------+----------+------------------+
+ *
+ *
+ *
+ *
+ *
+ *
+ * @param ptr response buffer pointer
+ * @param length response frame payload length
+ * @return true if payload structure is valid
+ */
+ public static boolean isStructurallyValid(long ptr, int length) {
+ if (length < MIN_RESPONSE_SIZE) {
+ return false;
+ }
+
+ byte status = Unsafe.getUnsafe().getByte(ptr);
+ if (status == STATUS_OK) {
+ return length == MIN_RESPONSE_SIZE;
+ }
+
+ if (length < MIN_ERROR_RESPONSE_SIZE) {
+ return false;
+ }
+
+ int msgLen = Unsafe.getUnsafe().getShort(ptr + MIN_RESPONSE_SIZE) & 0xFFFF;
+ return length == MIN_ERROR_RESPONSE_SIZE + msgLen;
+ }
+
+ /**
+ * Creates a success response.
+ */
+ public static WebSocketResponse success(long sequence) {
+ WebSocketResponse response = new WebSocketResponse();
+ response.status = STATUS_OK;
+ response.sequence = sequence;
+ return response;
+ }
+
+ /**
+ * Returns the error message, or null for success responses.
+ */
+ public String getErrorMessage() {
+ return errorMessage;
+ }
+
+ /**
+ * Returns the sequence number.
+ */
+ public long getSequence() {
+ return sequence;
+ }
+
+ /**
+ * Returns a human-readable status name.
+ */
+ public String getStatusName() {
+ return switch (status) {
+ case STATUS_OK -> "OK";
+ case STATUS_PARSE_ERROR -> "PARSE_ERROR";
+ case STATUS_SCHEMA_ERROR -> "SCHEMA_ERROR";
+ case STATUS_WRITE_ERROR -> "WRITE_ERROR";
+ case STATUS_SECURITY_ERROR -> "SECURITY_ERROR";
+ case STATUS_INTERNAL_ERROR -> "INTERNAL_ERROR";
+ default -> "UNKNOWN(" + (status & 0xFF) + ")";
+ };
+ }
+
+ /**
+ * Returns true if this is a success response.
+ */
+ public boolean isSuccess() {
+ return status == STATUS_OK;
+ }
+
+ /**
+ * Reads a response from native memory.
+ *
+ * @param ptr source address
+ * @param length available bytes
+ * @return true if successfully parsed, false if not enough data
+ */
+ public boolean readFrom(long ptr, int length) {
+ if (length < MIN_RESPONSE_SIZE) {
+ return false;
+ }
+
+ int offset = 0;
+
+ // Status (1 byte)
+ status = Unsafe.getUnsafe().getByte(ptr + offset);
+ offset += 1;
+
+ // Sequence (8 bytes, little-endian)
+ sequence = Unsafe.getUnsafe().getLong(ptr + offset);
+ offset += 8;
+
+ // Error message (if status != OK and more data available)
+ if (status != STATUS_OK && length > offset + 2) {
+ int msgLen = Unsafe.getUnsafe().getShort(ptr + offset) & 0xFFFF;
+ offset += 2;
+
+ if (length >= offset + msgLen && msgLen > 0) {
+ byte[] msgBytes = new byte[msgLen];
+ for (int i = 0; i < msgLen; i++) {
+ msgBytes[i] = Unsafe.getUnsafe().getByte(ptr + offset + i);
+ }
+ errorMessage = new String(msgBytes, StandardCharsets.UTF_8);
+ } else {
+ errorMessage = null;
+ }
+ } else {
+ errorMessage = null;
+ }
+
+ return true;
+ }
+
+ /**
+ * Calculates the serialized size of this response.
+ */
+ public int serializedSize() {
+ int size = MIN_RESPONSE_SIZE;
+ if (errorMessage != null && !errorMessage.isEmpty()) {
+ byte[] msgBytes = errorMessage.getBytes(StandardCharsets.UTF_8);
+ int msgLen = Math.min(msgBytes.length, MAX_ERROR_MESSAGE_LENGTH);
+ size += 2 + msgLen; // 2 bytes for length prefix
+ }
+ return size;
+ }
+
+ @Override
+ public String toString() {
+ if (isSuccess()) {
+ return "WebSocketResponse{status=OK, seq=" + sequence + "}";
+ } else {
+ return "WebSocketResponse{status=" + getStatusName() + ", seq=" + sequence +
+ ", error=" + errorMessage + "}";
+ }
+ }
+
+ /**
+ * Writes this response to native memory.
+ *
+ * @param ptr destination address
+ * @return number of bytes written
+ */
+ public int writeTo(long ptr) {
+ int offset = 0;
+
+ // Status (1 byte)
+ Unsafe.getUnsafe().putByte(ptr + offset, status);
+ offset += 1;
+
+ // Sequence (8 bytes, little-endian)
+ Unsafe.getUnsafe().putLong(ptr + offset, sequence);
+ offset += 8;
+
+ // Error message (if any)
+ if (status != STATUS_OK && errorMessage != null && !errorMessage.isEmpty()) {
+ byte[] msgBytes = errorMessage.getBytes(StandardCharsets.UTF_8);
+ int msgLen = Math.min(msgBytes.length, MAX_ERROR_MESSAGE_LENGTH);
+
+ // Length prefix (2 bytes, little-endian)
+ Unsafe.getUnsafe().putShort(ptr + offset, (short) msgLen);
+ offset += 2;
+
+ // Message bytes
+ for (int i = 0; i < msgLen; i++) {
+ Unsafe.getUnsafe().putByte(ptr + offset + i, msgBytes[i]);
+ }
+ offset += msgLen;
+ }
+
+ return offset;
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java
new file mode 100644
index 0000000..b155bab
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java
@@ -0,0 +1,637 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client;
+
+import io.questdb.client.cutlass.http.client.WebSocketClient;
+import io.questdb.client.cutlass.http.client.WebSocketFrameHandler;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.std.QuietCloseable;
+import org.jetbrains.annotations.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Asynchronous I/O handler for WebSocket microbatch transmission.
+ *
+ *
+ * The single-slot design matches the double-buffering scheme: at most one
+ * sealed buffer is pending while the other is being filled.
+ * Using a single thread eliminates concurrency issues with the WebSocket channel.
+ *
+ *
+ *
+ *
+ */
+public class WebSocketSendQueue implements QuietCloseable {
+
+ public static final long DEFAULT_ENQUEUE_TIMEOUT_MS = 30_000;
+ public static final long DEFAULT_SHUTDOWN_TIMEOUT_MS = 10_000;
+ private static final Logger LOG = LoggerFactory.getLogger(WebSocketSendQueue.class);
+ // The WebSocket client for I/O (single-threaded access only)
+ private final WebSocketClient client;
+ // Configuration
+ private final long enqueueTimeoutMs;
+ // Optional InFlightWindow for tracking sent batches awaiting ACK
+ @Nullable
+ private final InFlightWindow inFlightWindow;
+
+ // The I/O thread for async send/receive
+ private final Thread ioThread;
+ // Counter for batches currently being processed by the I/O thread
+ // This tracks batches that have been dequeued but not yet fully sent
+ private final AtomicInteger processingCount = new AtomicInteger(0);
+ // Lock for all coordination between user thread and I/O thread.
+ // Used for: queue poll + processingCount increment atomicity,
+ // flush() waiting, I/O thread waiting when idle.
+ private final Object processingLock = new Object();
+ // Response parsing
+ private final WebSocketResponse response = new WebSocketResponse();
+ private final ResponseHandler responseHandler = new ResponseHandler();
+ // Synchronization for flush/close
+ private final CountDownLatch shutdownLatch;
+ private final long shutdownTimeoutMs;
+ // Statistics - receiving
+ private final AtomicLong totalAcks = new AtomicLong(0);
+ // Statistics - sending
+ private final AtomicLong totalBatchesSent = new AtomicLong(0);
+ private final AtomicLong totalBytesSent = new AtomicLong(0);
+ private final AtomicLong totalErrors = new AtomicLong(0);
+ // Error handling
+ private volatile Throwable lastError;
+ // Batch sequence counter (must match server's messageSequence)
+ private long nextBatchSequence = 0;
+ // Single pending buffer slot (double-buffering means at most 1 item in queue)
+ // Zero allocation - just a volatile reference handoff
+ private volatile MicrobatchBuffer pendingBuffer;
+ // Running state
+ private volatile boolean running;
+ private volatile boolean shuttingDown;
+
+ /**
+ * Creates a new send queue with default configuration.
+ *
+ * @param client the WebSocket client for I/O
+ */
+ public WebSocketSendQueue(WebSocketClient client) {
+ this(client, null, DEFAULT_ENQUEUE_TIMEOUT_MS, DEFAULT_SHUTDOWN_TIMEOUT_MS);
+ }
+
+ /**
+ * Creates a new send queue with InFlightWindow for tracking sent batches.
+ *
+ * @param client the WebSocket client for I/O
+ * @param inFlightWindow the window to track sent batches awaiting ACK (may be null)
+ */
+ public WebSocketSendQueue(WebSocketClient client, @Nullable InFlightWindow inFlightWindow) {
+ this(client, inFlightWindow, DEFAULT_ENQUEUE_TIMEOUT_MS, DEFAULT_SHUTDOWN_TIMEOUT_MS);
+ }
+
+ /**
+ * Creates a new send queue with custom configuration.
+ *
+ * @param client the WebSocket client for I/O
+ * @param inFlightWindow the window to track sent batches awaiting ACK (may be null)
+ * @param enqueueTimeoutMs timeout for enqueue operations (ms)
+ * @param shutdownTimeoutMs timeout for graceful shutdown (ms)
+ */
+ public WebSocketSendQueue(WebSocketClient client, @Nullable InFlightWindow inFlightWindow,
+ long enqueueTimeoutMs, long shutdownTimeoutMs) {
+ if (client == null) {
+ throw new IllegalArgumentException("client cannot be null");
+ }
+
+ this.client = client;
+ this.inFlightWindow = inFlightWindow;
+ this.enqueueTimeoutMs = enqueueTimeoutMs;
+ this.shutdownTimeoutMs = shutdownTimeoutMs;
+ this.running = true;
+ this.shuttingDown = false;
+ this.shutdownLatch = new CountDownLatch(1);
+
+ // Start the I/O thread (handles both sending and receiving)
+ this.ioThread = new Thread(this::ioLoop, "questdb-websocket-io");
+ this.ioThread.setDaemon(true);
+ this.ioThread.start();
+
+ LOG.info("WebSocket I/O thread started");
+ }
+
+ /**
+ * Closes the send queue gracefully.
+ *
+ *
+ */
+ private void ioLoop() {
+ LOG.info("I/O loop started");
+
+ try {
+ while (running || !isPendingEmpty()) {
+ MicrobatchBuffer batch = null;
+ boolean hasInFlight = (inFlightWindow != null && inFlightWindow.getInFlightCount() > 0);
+ IoState state = computeState(hasInFlight);
+
+ switch (state) {
+ case IDLE:
+ // Nothing to do - wait for work under lock
+ synchronized (processingLock) {
+ // Re-check under lock to avoid missed wakeup
+ if (isPendingEmpty() && running) {
+ try {
+ processingLock.wait(100);
+ } catch (InterruptedException e) {
+ if (!running) return;
+ }
+ }
+ }
+ break;
+
+ case ACTIVE:
+ case DRAINING:
+ // Try to receive any pending ACKs (non-blocking)
+ if (hasInFlight && client.isConnected()) {
+ tryReceiveAcks();
+ }
+
+ // Try to dequeue and send a batch
+ boolean hasWindowSpace = (inFlightWindow == null || inFlightWindow.hasWindowSpace());
+ if (hasWindowSpace) {
+ // Atomically: poll queue + increment processingCount
+ synchronized (processingLock) {
+ batch = pollPending();
+ if (batch != null) {
+ processingCount.incrementAndGet();
+ }
+ }
+
+ if (batch != null) {
+ try {
+ safeSendBatch(batch);
+ } finally {
+ // Atomically: decrement + notify flush()
+ synchronized (processingLock) {
+ processingCount.decrementAndGet();
+ processingLock.notifyAll();
+ }
+ }
+ }
+ }
+
+ // In DRAINING state with no work, short wait to avoid busy loop
+ if (state == IoState.DRAINING && batch == null) {
+ synchronized (processingLock) {
+ try {
+ processingLock.wait(10);
+ } catch (InterruptedException e) {
+ if (!running) return;
+ }
+ }
+ }
+ break;
+ }
+ }
+ } finally {
+ shutdownLatch.countDown();
+ LOG.info("I/O loop stopped [totalAcks={}, totalErrors={}]", totalAcks.get(), totalErrors.get());
+ }
+ }
+
+ private boolean isPendingEmpty() {
+ return pendingBuffer == null;
+ }
+
+ private boolean offerPending(MicrobatchBuffer buffer) {
+ if (pendingBuffer != null) {
+ return false; // slot occupied
+ }
+ pendingBuffer = buffer;
+ return true;
+ }
+
+ private MicrobatchBuffer pollPending() {
+ MicrobatchBuffer buffer = pendingBuffer;
+ if (buffer != null) {
+ pendingBuffer = null;
+ }
+ return buffer;
+ }
+
+ /**
+ * Sends a batch with error handling. Does NOT manage processingCount.
+ */
+ private void safeSendBatch(MicrobatchBuffer batch) {
+ try {
+ sendBatch(batch);
+ } catch (Throwable t) {
+ LOG.error("Error sending batch [id={}]{}", batch.getBatchId(), "", t);
+ failTransport(new LineSenderException("Error sending batch " + batch.getBatchId() + ": " + t.getMessage(), t));
+ // Mark as recycled even on error to allow cleanup
+ if (batch.isSealed()) {
+ batch.markSending();
+ }
+ if (batch.isSending()) {
+ batch.markRecycled();
+ }
+ }
+ }
+
+ /**
+ * Sends a single batch over the WebSocket channel.
+ */
+ private void sendBatch(MicrobatchBuffer batch) {
+ // Transition state: SEALED -> SENDING
+ batch.markSending();
+
+ // Use our own sequence counter (must match server's messageSequence)
+ long batchSequence = nextBatchSequence++;
+ int bytes = batch.getBufferPos();
+ int rows = batch.getRowCount();
+
+ LOG.debug("Sending batch [seq={}, bytes={}, rows={}, bufferId={}]", batchSequence, bytes, rows, batch.getBatchId());
+
+ // Add to in-flight window BEFORE sending (so we're ready for ACK)
+ // Use non-blocking tryAddInFlight since we already checked window space in ioLoop
+ if (inFlightWindow != null) {
+ LOG.debug("Adding to in-flight window [seq={}, inFlight={}, max={}]", batchSequence, inFlightWindow.getInFlightCount(), inFlightWindow.getMaxWindowSize());
+ if (!inFlightWindow.tryAddInFlight(batchSequence)) {
+ // Should not happen since we checked hasWindowSpace before polling
+ throw new LineSenderException("In-flight window unexpectedly full");
+ }
+ LOG.debug("Added to in-flight window [seq={}]", batchSequence);
+ }
+
+ // Send over WebSocket
+ LOG.debug("Calling sendBinary [seq={}]", batchSequence);
+ client.sendBinary(batch.getBufferPtr(), bytes);
+ LOG.debug("sendBinary returned [seq={}]", batchSequence);
+
+ // Update statistics
+ totalBatchesSent.incrementAndGet();
+ totalBytesSent.addAndGet(bytes);
+
+ // Transition state: SENDING -> RECYCLED
+ batch.markRecycled();
+
+ LOG.debug("Batch sent and recycled [seq={}, bufferId={}]", batchSequence, batch.getBatchId());
+ }
+
+ /**
+ * Tries to receive ACKs from the server (non-blocking).
+ */
+ private void tryReceiveAcks() {
+ try {
+ client.tryReceiveFrame(responseHandler);
+ } catch (Exception e) {
+ if (running) {
+ LOG.error("Error receiving response: {}", e.getMessage());
+ failTransport(new LineSenderException("Error receiving response: " + e.getMessage(), e));
+ }
+ }
+ }
+
+ /**
+ * I/O loop states for the state machine.
+ *
+ *
+ */
+ private enum IoState {
+ IDLE, ACTIVE, DRAINING
+ }
+
+ /**
+ * Handler for received WebSocket frames (ACKs from server).
+ */
+ private class ResponseHandler implements WebSocketFrameHandler {
+
+ @Override
+ public void onBinaryMessage(long payloadPtr, int payloadLen) {
+ if (!WebSocketResponse.isStructurallyValid(payloadPtr, payloadLen)) {
+ LineSenderException error = new LineSenderException(
+ "Invalid ACK response payload [length=" + payloadLen + ']'
+ );
+ LOG.error("Invalid ACK response payload [length={}]", payloadLen);
+ failTransport(error);
+ return;
+ }
+
+ // Parse response from binary payload
+ if (!response.readFrom(payloadPtr, payloadLen)) {
+ LineSenderException error = new LineSenderException("Failed to parse ACK response");
+ LOG.error("Failed to parse response");
+ failTransport(error);
+ return;
+ }
+
+ long sequence = response.getSequence();
+
+ if (response.isSuccess()) {
+ // Cumulative ACK - acknowledge all batches up to this sequence
+ if (inFlightWindow != null) {
+ int acked = inFlightWindow.acknowledgeUpTo(sequence);
+ if (acked > 0) {
+ totalAcks.addAndGet(acked);
+ LOG.debug("Cumulative ACK received [upTo={}, acked={}]", sequence, acked);
+ } else {
+ LOG.debug("ACK for already-acknowledged sequences [upTo={}]", sequence);
+ }
+ }
+ } else {
+ // Error - fail the batch
+ String errorMessage = response.getErrorMessage();
+ LOG.error("Error response [seq={}, status={}, error={}]", sequence, response.getStatusName(), errorMessage);
+
+ if (inFlightWindow != null) {
+ LineSenderException error = new LineSenderException(
+ "Server error for batch " + sequence + ": " +
+ response.getStatusName() + " - " + errorMessage);
+ inFlightWindow.fail(sequence, error);
+ }
+ totalErrors.incrementAndGet();
+ }
+ }
+
+ @Override
+ public void onClose(int code, String reason) {
+ LOG.info("WebSocket closed by server [code={}, reason={}]", code, reason);
+ failTransport(new LineSenderException("WebSocket closed by server [code=" + code + ", reason=" + reason + ']'));
+ }
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java
new file mode 100644
index 0000000..a30cf3c
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java
@@ -0,0 +1,200 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.protocol;
+
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.QuietCloseable;
+import io.questdb.client.std.Unsafe;
+
+/**
+ * Lightweight append-only off-heap buffer for columnar data storage.
+ *
+ * QwpBitWriter writer = new QwpBitWriter();
+ * writer.reset(address, capacity);
+ * writer.writeBits(value, numBits);
+ * writer.writeBits(value2, numBits2);
+ * writer.flush(); // must call before reading output
+ * long bytesWritten = writer.getPosition() - address;
+ *
+ */
+public class QwpBitWriter {
+
+ // Buffer for accumulating bits before writing
+ private long bitBuffer;
+ // Number of bits currently in the buffer (0-63)
+ private int bitsInBuffer;
+ private long currentAddress;
+ private long endAddress;
+ private long startAddress;
+
+ /**
+ * Creates a new bit writer. Call {@link #reset} before use.
+ */
+ public QwpBitWriter() {
+ }
+
+ /**
+ * Aligns the writer to the next byte boundary by padding with zeros.
+ * If already byte-aligned, this is a no-op.
+ */
+ public void alignToByte() {
+ if (bitsInBuffer > 0) {
+ flush();
+ }
+ }
+
+ /**
+ * Finishes writing and returns the number of bytes written since reset.
+ *
+ * DoD = (t[n] - t[n-1]) - (t[n-1] - t[n-2])
+ *
+ * if DoD == 0: write '0' (1 bit)
+ * elif DoD in [-64, 63]: write '10' + 7-bit (9 bits)
+ * elif DoD in [-256, 255]: write '110' + 9-bit (12 bits)
+ * elif DoD in [-2048, 2047]: write '1110' + 12-bit (16 bits)
+ * else: write '1111' + 32-bit (36 bits)
+ *
+ *
+ *
+ *
+ * @param deltaOfDelta the delta-of-delta value to encode
+ */
+ public void encodeDoD(long deltaOfDelta) {
+ int bucket = getBucket(deltaOfDelta);
+ switch (bucket) {
+ case 0 -> bitWriter.writeBit(0);
+ case 1 -> {
+ bitWriter.writeBits(0b01, 2);
+ bitWriter.writeSigned(deltaOfDelta, 7);
+ }
+ case 2 -> {
+ bitWriter.writeBits(0b011, 3);
+ bitWriter.writeSigned(deltaOfDelta, 9);
+ }
+ case 3 -> {
+ bitWriter.writeBits(0b0111, 4);
+ bitWriter.writeSigned(deltaOfDelta, 12);
+ }
+ default -> {
+ bitWriter.writeBits(0b1111, 4);
+ bitWriter.writeSigned(deltaOfDelta, 32);
+ }
+ }
+ }
+
+ /**
+ * Encodes timestamps from off-heap memory using Gorilla compression.
+ *
+ * - First timestamp: int64 (8 bytes, little-endian)
+ * - Second timestamp: int64 (8 bytes, little-endian)
+ * - Remaining timestamps: bit-packed delta-of-delta
+ *
+ *
- *
- *
- *
- *