From 61cb29f59dd0d09f3de23defaf68ad191b9e22b4 Mon Sep 17 00:00:00 2001 From: "Aaron M. Ucko" Date: Thu, 9 May 2024 16:36:31 -0400 Subject: [PATCH 1/9] Improve support for TDS 5 (Sybase) bulk transfers. * Read defaults, send them back explicitly as needed, and clean up their storage afterwards. * tds5_bcp_add_variable_columns: ** Send spaces in lieu of fully empty SYB(VAR)CHAR data. ** Send correct textpos values in all cases. ** When writing the adjustment table, avoid a potential Sybase "row format error" by eliding the (effective) column count when it would have repeated the largest adjustment table entry. * tds_deinit_bcpinfo: Free and reset current_row for transfers in. Avoid potentially trying to interpret fragments of outgoing TDS data as BLOB pointers. Signed-off-by: Aaron M. Ucko --- include/freetds/tds.h | 4 ++ src/tds/bulk.c | 122 +++++++++++++++++++++++++++++++++++++++--- src/tds/mem.c | 14 +++++ src/tds/token.c | 4 +- 4 files changed, 137 insertions(+), 7 deletions(-) diff --git a/include/freetds/tds.h b/include/freetds/tds.h index df55f8824f..50864ecb41 100644 --- a/include/freetds/tds.h +++ b/include/freetds/tds.h @@ -1676,6 +1676,10 @@ typedef struct tds5_colinfo TDS_TINYINT status; TDS_SMALLINT offset; TDS_INT length; + TDS_TINYINT dflt; + + TDS_INT dflt_size; + TDS_UCHAR* dflt_value; } TDS5COLINFO; struct tds_bcpinfo diff --git a/src/tds/bulk.c b/src/tds/bulk.c index 938915e4ab..b208963c10 100644 --- a/src/tds/bulk.c +++ b/src/tds/bulk.c @@ -74,6 +74,9 @@ static int tds5_bcp_add_variable_columns(TDSBCPINFO *bcpinfo, tds_bcp_get_col_da int offset, TDS_UCHAR *rowbuffer, int start, int *pncols); static void tds_bcp_row_free(TDSRESULTINFO* result, unsigned char *row); static TDSRET tds5_process_insert_bulk_reply(TDSSOCKET * tds, TDSBCPINFO *bcpinfo); +static TDSRET tds5_get_col_data_or_dflt(tds_bcp_get_col_data get_col_data, + TDSBCPINFO *bulk, TDSCOLUMN *bcpcol, + int offset, int colnum); /** * Initialize BCP information. @@ -439,7 +442,29 @@ tds5_send_record(TDSSOCKET *tds, TDSBCPINFO *bcpinfo, for (i = 0; i < bcpinfo->bindinfo->num_cols; i++) { TDSCOLUMN *bindcol = bcpinfo->bindinfo->columns[i]; if (is_blob_type(bindcol->on_server.column_type)) { - TDS_PROPAGATE(get_col_data(bcpinfo, bindcol, offset)); + /* Elide trailing NULLs */ + if (bindcol->bcp_column_data->is_null) { + int j; + for (j = i + 1; + j < bcpinfo->bindinfo->num_cols; ++j) { + TDSCOLUMN *bindcol2 + = bcpinfo->bindinfo + ->columns[j]; + if (is_blob_type(bindcol2->column_type) + && !(bindcol2->bcp_column_data + ->is_null)) { + break; + } + } + if (j == bcpinfo->bindinfo->num_cols) { + i = j; + break; + } + } + + TDS_PROPAGATE(tds5_get_col_data_or_dflt + (get_col_data, bcpinfo, bindcol, offset, + i)); /* unknown but zero */ tds_put_smallint(tds, 0); TDS_PUT_BYTE(tds, bindcol->on_server.column_type); @@ -538,7 +563,8 @@ tds5_bcp_add_fixed_columns(TDSBCPINFO *bcpinfo, tds_bcp_get_col_data get_col_dat tdsdump_log(TDS_DBG_FUNC, "tds5_bcp_add_fixed_columns column %d (%s) is a fixed column\n", i + 1, tds_dstr_cstr(&bcpcol->column_name)); - if (TDS_FAILED(get_col_data(bcpinfo, bcpcol, offset))) { + if (TDS_FAILED(tds5_get_col_data_or_dflt(get_col_data, bcpinfo, + bcpcol, offset, i))) { tdsdump_log(TDS_DBG_INFO1, "get_col_data (column %d) failed\n", i + 1); return -1; } @@ -647,7 +673,8 @@ tds5_bcp_add_variable_columns(TDSBCPINFO *bcpinfo, tds_bcp_get_col_data get_col_ tdsdump_log(TDS_DBG_FUNC, "%4d %8d %8d %8d\n", i, ncols, row_pos, cpbytes); - if (TDS_FAILED(get_col_data(bcpinfo, bcpcol, offset))) + if (TDS_FAILED(tds5_get_col_data_or_dflt(get_col_data, bcpinfo, + bcpcol, offset, i))) return -1; /* If it's a NOT NULL column, and we have no data, throw an error. @@ -668,12 +695,19 @@ tds5_bcp_add_variable_columns(TDSBCPINFO *bcpinfo, tds_bcp_get_col_data get_col_ TDS_NUMERIC *num = (TDS_NUMERIC *) bcpcol->bcp_column_data->data; cpbytes = tds_numeric_bytes_per_prec[num->precision]; memcpy(&rowbuffer[row_pos], num->array, cpbytes); + } else if ((bcpcol->column_type == SYBVARCHAR + || bcpcol->column_type == SYBCHAR) + && bcpcol->bcp_column_data->datalen == 0) { + cpbytes = 1; + rowbuffer[row_pos] = ' '; } else { cpbytes = bcpcol->bcp_column_data->datalen > bcpcol->column_size ? bcpcol->column_size : bcpcol->bcp_column_data->datalen; memcpy(&rowbuffer[row_pos], bcpcol->bcp_column_data->data, cpbytes); tds5_swap_data(bcpcol, &rowbuffer[row_pos]); } + } else if (is_blob_type(bcpcol->column_type)) { + bcpcol->column_textpos = row_pos; } row_pos += cpbytes; @@ -710,7 +744,8 @@ tds5_bcp_add_variable_columns(TDSBCPINFO *bcpinfo, tds_bcp_get_col_data get_col_ tdsdump_log(TDS_DBG_FUNC, "ncols=%u poff=%p [%u]\n", ncols, poff, offsets[ncols]); - *poff++ = ncols + 1; + if (offsets[ncols] / 256 == offsets[ncols-1] / 256) + *poff++ = ncols + 1; /* this is some kind of run-length-prefix encoding */ while (pfx_top) { unsigned int n_pfx = 1; @@ -912,6 +947,7 @@ enum { BULKCOL_length, BULKCOL_status, BULKCOL_offset, + BULKCOL_dflt, /* number of columns needed */ BULKCOL_COUNT, @@ -933,6 +969,9 @@ tds5_bulk_insert_column(const char *name) BULKCOL(colcnt); BULKCOL(colid); break; + case 'd': + BULKCOL(dflt); + break; case 't': BULKCOL(type); break; @@ -950,6 +989,27 @@ tds5_bulk_insert_column(const char *name) return -1; } +static void +tds5_read_bulk_defaults(TDSRESULTINFO *res_info, TDSBCPINFO *bcpinfo) +{ + int i; + TDS5COLINFO *syb_info = bcpinfo->sybase_colinfo; + for (i = 0; i < res_info->num_cols; ++i, ++syb_info) { + TDSCOLUMN *col = res_info->columns[i]; + TDS_UCHAR* src = col->column_data; + TDS_INT len = col->column_cur_size; + if (is_blob_type(col->column_type)) { + src = (unsigned char*) ((TDSBLOB*)src)->textvalue; + } + while ( !syb_info->dflt ) { + ++syb_info; + } + syb_info->dflt_size = len; + tds_realloc((void**)&syb_info->dflt_value, len); + memcpy(syb_info->dflt_value, src, len); + } +} + static TDSRET tds5_process_insert_bulk_reply(TDSSOCKET * tds, TDSBCPINFO *bcpinfo) { @@ -965,6 +1025,7 @@ tds5_process_insert_bulk_reply(TDSSOCKET * tds, TDSBCPINFO *bcpinfo) int cols_pos[BULKCOL_COUNT]; int cols_values[BULKCOL_COUNT]; TDS5COLINFO *colinfo; + int num_defs = 0; CHECK_TDS_EXTRA(tds); @@ -991,9 +1052,14 @@ tds5_process_insert_bulk_reply(TDSSOCKET * tds, TDSBCPINFO *bcpinfo) case TDS_ROW_RESULT: /* get the results */ col_flags = 0; - if (!row_match) - continue; res_info = tds->current_results; + if (!row_match) { +#if ENABLE_EXTRA_CHECKS + assert(res_info->num_cols == num_defs); +#endif + tds5_read_bulk_defaults(res_info, bcpinfo); + continue; + } if (!res_info) continue; for (icol = 0; icol < BULKCOL_COUNT; ++icol) { @@ -1035,6 +1101,10 @@ tds5_process_insert_bulk_reply(TDSSOCKET * tds, TDSBCPINFO *bcpinfo) colinfo->status = cols_values[BULKCOL_status]; colinfo->offset = cols_values[BULKCOL_offset]; colinfo->length = cols_values[BULKCOL_length]; + colinfo->dflt = cols_values[BULKCOL_dflt]; + if (colinfo->dflt) { + ++num_defs; + } tdsdump_log(TDS_DBG_INFO1, "gotten row information %d type %d length %d status %d offset %d\n", cols_values[BULKCOL_colid], colinfo->type, @@ -1057,6 +1127,46 @@ tds5_process_insert_bulk_reply(TDSSOCKET * tds, TDSBCPINFO *bcpinfo) return ret; } +static TDSRET tds5_get_col_data_or_dflt(tds_bcp_get_col_data get_col_data, + TDSBCPINFO *bulk, TDSCOLUMN *bcpcol, + int offset, int colnum) +{ + TDSRET ret; + BCPCOLDATA *coldata; + if (bcpcol->column_lenbind == NULL) { + bcpcol->column_lenbind = (TDS_INT *)&bcpcol->column_bindlen; + } + ret = get_col_data(bulk, bcpcol, offset); + coldata = bcpcol->bcp_column_data; + if (bcpcol->column_varaddr == NULL && coldata->datalen == 0 + && bulk->sybase_colinfo != NULL + && ( !is_blob_type(bcpcol->column_type) + || bcpcol->column_lenbind == NULL + || bcpcol->column_lenbind[offset] == 0)) { + const TDS5COLINFO *syb_info = &bulk->sybase_colinfo[colnum]; + const TDS_SMALLINT *nullind = bcpcol->column_nullbind; + if ((nullind != NULL && nullind[offset] == -1) + || !syb_info->dflt) { + if ( !bcpcol->column_nullable ) { + return TDS_FAIL; + } + coldata->datalen = 0; + coldata->is_null = true; + } else { + if (syb_info->dflt_size > 4096) { + tds_realloc((void**)&coldata->data, + syb_info->dflt_size); + } + memcpy(coldata->data, syb_info->dflt_value, + syb_info->dflt_size); + coldata->datalen = syb_info->dflt_size; + coldata->is_null = false; + } + return TDS_SUCCESS; + } + return ret; +} + /** * Free row data allocated in the result set. */ diff --git a/src/tds/mem.c b/src/tds/mem.c index a79d6ffa87..de1eebbb4a 100644 --- a/src/tds/mem.c +++ b/src/tds/mem.c @@ -1868,11 +1868,25 @@ tds_alloc_bcpinfo(void) void tds_deinit_bcpinfo(TDSBCPINFO *bcpinfo) { + /* + * Historically done for all TDS 5.0 transfers, but the protocol + * version isn't available here, or even in blk_done anymore. + */ + if (bcpinfo->direction == TDS_BCP_IN && bcpinfo->bindinfo != NULL + && bcpinfo->bindinfo->current_row != NULL) { + TDS_ZERO_FREE(bcpinfo->bindinfo->current_row); + } tds_dstr_free(&bcpinfo->hint); tds_dstr_free(&bcpinfo->tablename); TDS_ZERO_FREE(bcpinfo->insert_stmt); tds_free_results(bcpinfo->bindinfo); bcpinfo->bindinfo = NULL; + if (bcpinfo->sybase_colinfo != NULL) { + int i; + for (i = 0; i < bcpinfo->sybase_count; ++i) { + free(bcpinfo->sybase_colinfo[i].dflt_value); + } + } TDS_ZERO_FREE(bcpinfo->sybase_colinfo); bcpinfo->sybase_count = 0; } diff --git a/src/tds/token.c b/src/tds/token.c index c915208c72..75413226c6 100644 --- a/src/tds/token.c +++ b/src/tds/token.c @@ -154,7 +154,9 @@ tds_process_default_tokens(TDSSOCKET * tds, int marker) case TDS_RETURNSTATUS_TOKEN: ret_status = tds_get_int(tds); marker = tds_peek(tds); - if (marker != TDS_PARAM_TOKEN && marker != TDS_DONEPROC_TOKEN && marker != TDS_DONE_TOKEN) + if (marker != TDS_PARAM_TOKEN && marker != TDS_DONEPROC_TOKEN + && marker != TDS_DONE_TOKEN + && marker != TDS5_PARAMFMT_TOKEN) break; tds->has_status = true; tds->ret_status = ret_status; From c3f9ec72a2b22397584f4be642f05d44a9d9389a Mon Sep 17 00:00:00 2001 From: "Aaron M. Ucko" Date: Fri, 10 May 2024 10:23:49 -0400 Subject: [PATCH 2/9] Fix DATE and DATETIME2 bulk insertion via TDS 7.0 and 7.1. Under these protocol versions, MS SQL Server describes these columns as having NVARCHAR(n) types rather than traditional DATETIME types, due to DATETIME's range limitations. (Likewise for TIME, which however still needs to come in as a preconverted UTF-16 string to compensate for combined API and protocol limitations.) - ctlib/blk.c: Perform character conversion on strings converted from fixed-width types such as CS_DATETIME_TYPE when a comparison of column-size limits appears to indicate an encoding difference. - tds/config.c: Adjust STD_DATETIME_FMT to match MS's (ISO-ish) tastes; in particular, ensure that the entire date portion fits within the first 10 characters. Signed-off-by: Aaron M. Ucko --- src/ctlib/blk.c | 17 +++++++++++++++++ src/tds/config.c | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/ctlib/blk.c b/src/ctlib/blk.c index 6c8e70aba7..fc03364d3d 100644 --- a/src/ctlib/blk.c +++ b/src/ctlib/blk.c @@ -644,6 +644,7 @@ _blk_get_col_data(TDSBCPINFO *bulk, TDSCOLUMN *bindcol, int offset) CS_INT *datalen = NULL; CS_BLKDESC *blkdesc = (CS_BLKDESC *) bulk; CS_CONTEXT *ctx = CONN(blkdesc)->ctx; + BCPCOLDATA *coldata = bindcol->bcp_column_data; tdsdump_log(TDS_DBG_FUNC, "_blk_get_col_data(%p, %p, %d)\n", bulk, bindcol, offset); @@ -712,6 +713,7 @@ _blk_get_col_data(TDSBCPINFO *bulk, TDSCOLUMN *bindcol, int offset) CS_DATAFMT_COMMON srcfmt, destfmt; CS_INT desttype; TDS_SERVER_TYPE tds_desttype = TDS_INVALID_TYPE; + TDSSOCKET * tds = CONN(blkdesc)->tds_socket; srcfmt.datatype = srctype; srcfmt.maxlength = srclen; @@ -738,6 +740,21 @@ _blk_get_col_data(TDSBCPINFO *bulk, TDSCOLUMN *bindcol, int offset) srctype, desttype); return TDS_FAIL; } + if (destfmt.maxlength != bindcol->column_size + && destfmt.datatype == CS_CHAR_TYPE + && is_fixed_type(_ct_get_server_type(tds, srctype))) { + size_t out_len; + TDS_UCHAR *buf + = (TDS_UCHAR *) tds_convert_string + (tds, bindcol->char_conv, + (char *) coldata->data, destlen, + &out_len); + if (buf != NULL && buf != coldata->data) { + free(coldata->data); + coldata->data = buf; + destlen = (CS_INT) out_len; + } + } } bindcol->bcp_column_data->datalen = destlen; diff --git a/src/tds/config.c b/src/tds/config.c index 80924b31d4..bf7e41c7b7 100644 --- a/src/tds/config.c +++ b/src/tds/config.c @@ -91,7 +91,7 @@ static tds_dir_char *interf_file = NULL; #define TDS_ISSPACE(c) isspace((unsigned char ) (c)) -const char STD_DATETIME_FMT[] = "%b %e %Y %I:%M%p"; +const char STD_DATETIME_FMT[] = "%Y-%m-%d %H:%M:%S.%z"; #if !defined(_WIN32) && !defined(DOS32X) static const char pid_config_logpath[] = "/tmp/tdsconfig.log.%d"; From c37db4b36c7bca0b8983d2be5e4e57126b9f453d Mon Sep 17 00:00:00 2001 From: "Aaron M. Ucko" Date: Mon, 13 May 2024 11:36:51 -0400 Subject: [PATCH 3/9] ctlib: Optimize blob conversion for bulk transfers in. * Extend _cs_convert to take an additional handle argument (allowed to be NULL) to provide for the possibility of replacing the caller's original output buffer with the one tdsconvert allocated itself (as it always does for blobs), rather than copying the blob data. * blk.c (_blk_get_col_data): Provide a non-NULL handle, both for efficiency's sake and because tds_alloc_bcp_column_data imposes a 4 KiB cap with the expectation that callers will take this approach. (dblib already does.) Signed-off-by: Aaron M. Ucko --- include/ctlib.h | 6 +++-- src/ctlib/blk.c | 6 +++-- src/ctlib/cs.c | 68 ++++++++++++++++++++++++++++++++++++++++--------- src/ctlib/ct.c | 3 ++- 4 files changed, 66 insertions(+), 17 deletions(-) diff --git a/include/ctlib.h b/include/ctlib.h index edba6f492c..49ca765ae0 100644 --- a/include/ctlib.h +++ b/include/ctlib.h @@ -397,8 +397,10 @@ CS_INT _ct_get_string_length(const char *buf, CS_INT buflen); int _cs_convert_not_client(CS_CONTEXT *ctx, const TDSCOLUMN *curcol, CONV_RESULT *convert_buffer, unsigned char **p_src); -CS_RETCODE _cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, CS_VOID * srcdata, - const CS_DATAFMT_COMMON * destfmt, CS_VOID * destdata, CS_INT * resultlen, TDS_SERVER_TYPE desttype); +CS_RETCODE _cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, + CS_VOID * srcdata, const CS_DATAFMT_COMMON * destfmt, + CS_VOID * destdata, CS_INT * resultlen, + TDS_SERVER_TYPE desttype, CS_VOID ** handle); bool _ct_is_large_identifiers_version(CS_INT version); const CS_DATAFMT_COMMON * _ct_datafmt_common(CS_CONTEXT * ctx, const CS_DATAFMT * datafmt); const CS_DATAFMT_LARGE *_ct_datafmt_conv_in(CS_CONTEXT * ctx, const CS_DATAFMT * datafmt, CS_DATAFMT_LARGE * fmtbuf); diff --git a/src/ctlib/blk.c b/src/ctlib/blk.c index fc03364d3d..af581d666f 100644 --- a/src/ctlib/blk.c +++ b/src/ctlib/blk.c @@ -733,8 +733,10 @@ _blk_get_col_data(TDSBCPINFO *bulk, TDSCOLUMN *bindcol, int offset) destfmt.format = CS_FMT_UNUSED; /* if convert return FAIL mark error but process other columns */ - result = _cs_convert(ctx, &srcfmt, (CS_VOID *) src, - &destfmt, (CS_VOID *) bindcol->bcp_column_data->data, &destlen, tds_desttype); + result = _cs_convert(ctx, &srcfmt, (CS_VOID *) src, &destfmt, + (CS_VOID *) coldata->data, &destlen, + tds_desttype, + (CS_VOID **) &coldata->data); if (result != CS_SUCCEED) { tdsdump_log(TDS_DBG_ERROR, "conversion from srctype %d to desttype %d failed\n", srctype, desttype); diff --git a/src/ctlib/cs.c b/src/ctlib/cs.c index 0cd4f64177..5d6a96c86b 100644 --- a/src/ctlib/cs.c +++ b/src/ctlib/cs.c @@ -504,8 +504,10 @@ cs_config(CS_CONTEXT * ctx, CS_INT action, CS_INT property, CS_VOID * buffer, CS } CS_RETCODE -_cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, CS_VOID * srcdata, - const CS_DATAFMT_COMMON * destfmt, CS_VOID * destdata, CS_INT * resultlen, TDS_SERVER_TYPE desttype) +_cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, + CS_VOID * srcdata, const CS_DATAFMT_COMMON * destfmt, + CS_VOID * destdata, CS_INT * resultlen, TDS_SERVER_TYPE desttype, + CS_VOID ** handle) { TDS_SERVER_TYPE src_type; int src_len, destlen, len; @@ -515,8 +517,10 @@ _cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, CS_VOID * srcdat CS_INT dummy, datatype; CS_VARCHAR *destvc = NULL; - tdsdump_log(TDS_DBG_FUNC, "cs_convert(%p, %p, %p, %p, %p, %p, %d)\n", - ctx, srcfmt, srcdata, destfmt, destdata, resultlen, desttype); + tdsdump_log(TDS_DBG_FUNC, + "cs_convert(%p, %p, %p, %p, %p, %p, %d, %p)\n", + ctx, srcfmt, srcdata, destfmt, destdata, resultlen, + desttype, handle); /* If destfmt is NULL we have a problem */ if (destfmt == NULL) { @@ -588,6 +592,14 @@ _cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, CS_VOID * srcdat if (src_type == desttype) { int minlen = src_len < destlen? src_len : destlen; + if (handle != NULL && destvc == NULL) { + int type_len = tds_get_size_by_type(src_type); + if (type_len > minlen) { + minlen = type_len; + } + tds_realloc(handle, minlen + 1); + dest = (unsigned char *) *handle; + } tdsdump_log(TDS_DBG_FUNC, "cs_convert() srctype == desttype\n"); switch (desttype) { @@ -781,8 +793,14 @@ _cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, CS_VOID * srcdat ret = CS_FAIL; len = destlen; } - memcpy(dest, cres.ib, len); - free(cres.ib); + if (handle == NULL) { + memcpy(dest, cres.ib, len); + free(cres.ib); + } else { + free(*handle); + *handle = cres.ib; + destlen = len; + } *resultlen = len; if (destvc) { destvc->len = len; @@ -851,7 +869,13 @@ _cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, CS_VOID * srcdat tdsdump_log(TDS_DBG_FUNC, "not enough room for data + a null terminator - error\n"); ret = CS_FAIL; /* not enough room for data + a null terminator - error */ } else { - memcpy(dest, cres.c, len); + if (handle == NULL) { + memcpy(dest, cres.c, len); + } else { + free(*handle); + *handle = cres.c; + dest = *handle; + } dest[len] = 0; *resultlen = len + 1; } @@ -860,7 +884,13 @@ _cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, CS_VOID * srcdat case CS_FMT_PADBLANK: tdsdump_log(TDS_DBG_FUNC, "cs_convert() FMT_PADBLANK\n"); /* strcpy here can lead to a small buffer overflow */ - memcpy(dest, cres.c, len); + if (handle == NULL) { + memcpy(dest, cres.c, len); + } else { + free(*handle); + *handle = cres.c; + destlen = len; + } memset(dest + len, ' ', destlen - len); *resultlen = destlen; break; @@ -868,13 +898,24 @@ _cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, CS_VOID * srcdat case CS_FMT_PADNULL: tdsdump_log(TDS_DBG_FUNC, "cs_convert() FMT_PADNULL\n"); /* strcpy here can lead to a small buffer overflow */ - memcpy(dest, cres.c, len); + if (handle == NULL) { + memcpy(dest, cres.c, len); + } else { + free(*handle); + *handle = cres.c; + destlen = len; + } memset(dest + len, '\0', destlen - len); *resultlen = destlen; break; case CS_FMT_UNUSED: tdsdump_log(TDS_DBG_FUNC, "cs_convert() FMT_UNUSED\n"); - memcpy(dest, cres.c, len); + if (handle == NULL) { + memcpy(dest, cres.c, len); + } else { + free(*handle); + *handle = cres.c; + } *resultlen = len; break; default: @@ -885,7 +926,9 @@ _cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, CS_VOID * srcdat destvc->len = len; *resultlen = sizeof(*destvc); } - free(cres.c); + if (handle == NULL || *handle != cres.c) { + free(cres.c); + } break; default: ret = CS_FAIL; @@ -899,7 +942,8 @@ CS_RETCODE cs_convert(CS_CONTEXT * ctx, CS_DATAFMT * srcfmt, CS_VOID * srcdata, CS_DATAFMT * destfmt, CS_VOID * destdata, CS_INT * resultlen) { return _cs_convert(ctx, _ct_datafmt_common(ctx, srcfmt), srcdata, - _ct_datafmt_common(ctx, destfmt), destdata, resultlen, TDS_INVALID_TYPE); + _ct_datafmt_common(ctx, destfmt), destdata, + resultlen, TDS_INVALID_TYPE, NULL); } CS_RETCODE diff --git a/src/ctlib/ct.c b/src/ctlib/ct.c index 2a736fe303..55fc6127cb 100644 --- a/src/ctlib/ct.c +++ b/src/ctlib/ct.c @@ -1985,7 +1985,8 @@ _ct_bind_data(CS_CONTEXT *ctx, TDSRESULTINFO * resinfo, TDSRESULTINFO *bindinfo, destfmt.format = bindcol->column_bindfmt; /* if convert return FAIL mark error but process other columns */ - ret = _cs_convert(ctx, &srcfmt, src, &destfmt, dest, pdatalen, TDS_INVALID_TYPE); + ret = _cs_convert(ctx, &srcfmt, src, &destfmt, dest, pdatalen, + TDS_INVALID_TYPE, NULL); if (ret != CS_SUCCEED) { tdsdump_log(TDS_DBG_FUNC, "cs_convert-result = %d\n", ret); result = 1; From 3a2509cc1ae818c2ad8cf2dbbd9e90fb57eebca2 Mon Sep 17 00:00:00 2001 From: "Aaron M. Ucko" Date: Mon, 13 May 2024 11:52:20 -0400 Subject: [PATCH 4/9] blk.c: Handle CS_BLK_CANCEL in blk_done. To that end, factor a static _blk_clean_desc function out of blk_done's CS_BLK_ALL case. Signed-off-by: Aaron M. Ucko --- src/ctlib/blk.c | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/src/ctlib/blk.c b/src/ctlib/blk.c index af581d666f..0b24d78262 100644 --- a/src/ctlib/blk.c +++ b/src/ctlib/blk.c @@ -42,6 +42,7 @@ static void _blk_null_error(TDSBCPINFO *bcpinfo, int index, int offset); static TDSRET _blk_get_col_data(TDSBCPINFO *bulk, TDSCOLUMN *bcpcol, int offset); static CS_RETCODE _blk_rowxfer_in(CS_BLKDESC * blkdesc, CS_INT rows_to_xfer, CS_INT * rows_xferred); static CS_RETCODE _blk_rowxfer_out(CS_BLKDESC * blkdesc, CS_INT rows_to_xfer, CS_INT * rows_xferred); +static void _blk_clean_desc(CS_BLKDESC * blkdesc); #define CONN(bulk) ((CS_CONNECTION *) (bulk)->bcpinfo.parent) @@ -282,19 +283,44 @@ blk_done(CS_BLKDESC * blkdesc, CS_INT type, CS_INT * outrow) *outrow = rows_copied; /* free allocated storage in blkdesc & initialise flags, etc. */ - tds_deinit_bcpinfo(&blkdesc->bcpinfo); - - blkdesc->bcpinfo.direction = 0; - blkdesc->bcpinfo.bind_count = CS_UNUSED; - blkdesc->bcpinfo.xfer_init = false; - + _blk_clean_desc(blkdesc); break; + case CS_BLK_CANCEL: + /* discard staged query data */ + tds->out_pos = sizeof(TDS_HEADER); + /* can't transition directly from SENDING to PENDING */ + tds_set_state(tds, TDS_WRITING); + tds_set_state(tds, TDS_PENDING); + + tds_send_cancel(tds); + + if (TDS_FAILED(tds_process_cancel(tds))) { + _ctclient_msg(NULL, CONN(blkdesc), "blk_done", + 2, 5, 1, 140, ""); + return CS_FAIL; + } + + if (outrow) + *outrow = 0; + + /* free allocated storage in blkdesc & initialise flags, etc. */ + _blk_clean_desc(blkdesc); + break; } return CS_SUCCEED; } +static void _blk_clean_desc (CS_BLKDESC * blkdesc) +{ + tds_deinit_bcpinfo(&blkdesc->bcpinfo); + + blkdesc->bcpinfo.direction = 0; + blkdesc->bcpinfo.bind_count = CS_UNUSED; + blkdesc->bcpinfo.xfer_init = false; +} + CS_RETCODE blk_drop(CS_BLKDESC * blkdesc) { From 18d89436436a44476e8dc21df43ba2b0fc4a7d55 Mon Sep 17 00:00:00 2001 From: "Aaron M. Ucko" Date: Mon, 13 May 2024 14:24:07 -0400 Subject: [PATCH 5/9] Implement inbound blk_textxfer, adjusting libtds as needed. * tds.h (struct tds_bcpinfo): Add more fields to enable the necessary bookkeeping. * ctlib/blk.c: Implement blk_textfer; adjust other functions accordingly. * tds/{bulk,data}.c: Allow for deferred, and possibly piecemeal, blob submission; in particular, make tds_bcp_send_record resumable. Signed-off-by: Aaron M. Ucko --- include/freetds/tds.h | 3 + src/ctlib/blk.c | 81 +++++++++++++++++++----- src/tds/bulk.c | 142 +++++++++++++++++++++++++++++++++++------- src/tds/data.c | 9 ++- 4 files changed, 194 insertions(+), 41 deletions(-) diff --git a/include/freetds/tds.h b/include/freetds/tds.h index 50864ecb41..c0212f7756 100644 --- a/include/freetds/tds.h +++ b/include/freetds/tds.h @@ -1695,6 +1695,9 @@ struct tds_bcpinfo TDSRESULTINFO *bindinfo; TDS5COLINFO *sybase_colinfo; TDS_INT sybase_count; + TDS_INT text_sent; + TDS_INT next_col; + TDS_INT blob_cols; }; TDSRET tds_bcp_init(TDSSOCKET *tds, TDSBCPINFO *bcpinfo); diff --git a/src/ctlib/blk.c b/src/ctlib/blk.c index 0b24d78262..619a36e6a5 100644 --- a/src/ctlib/blk.c +++ b/src/ctlib/blk.c @@ -319,6 +319,9 @@ static void _blk_clean_desc (CS_BLKDESC * blkdesc) blkdesc->bcpinfo.direction = 0; blkdesc->bcpinfo.bind_count = CS_UNUSED; blkdesc->bcpinfo.xfer_init = false; + blkdesc->bcpinfo.text_sent = 0; + blkdesc->bcpinfo.next_col = 0; + blkdesc->bcpinfo.blob_cols = 0; } CS_RETCODE @@ -523,10 +526,39 @@ blk_srvinit(SRV_PROC * srvproc, CS_BLKDESC * blkdescp) CS_RETCODE blk_textxfer(CS_BLKDESC * blkdesc, CS_BYTE * buffer, CS_INT buflen, CS_INT * outlen) { + TDSSOCKET *tds; + TDSCOLUMN *bindcol; + tdsdump_log(TDS_DBG_FUNC, "blk_textxfer(%p, %p, %d, %p)\n", blkdesc, buffer, buflen, outlen); - tdsdump_log(TDS_DBG_FUNC, "UNIMPLEMENTED blk_textxfer()\n"); - return CS_FAIL; + if (blkdesc == NULL || buffer == NULL + || blkdesc->bcpinfo.direction != CS_BLK_IN) { + return CS_FAIL; + } + + tds = CONN(blkdesc)->tds_socket; + + bindcol = blkdesc->bcpinfo.bindinfo->columns + [blkdesc->bcpinfo.next_col-1]; + + if (bindcol->column_varaddr != NULL) { + return CS_FAIL; + } + + bindcol->column_cur_size = buflen; + bindcol->column_lenbind = &bindcol->column_cur_size; + bindcol->column_varaddr = (TDS_CHAR*) buffer; + + if (TDS_FAILED(tds_bcp_send_record(tds, &blkdesc->bcpinfo, + _blk_get_col_data, _blk_null_error, + 0))) { + return CS_FAIL; + } else if (blkdesc->bcpinfo.next_col == 0) { + return CS_END_DATA; /* all done */ + } else { + bindcol->column_varaddr = NULL; + return CS_SUCCEED; /* still need more data */ + } } static CS_RETCODE @@ -623,6 +655,8 @@ _blk_rowxfer_in(CS_BLKDESC * blkdesc, CS_INT rows_to_xfer, CS_INT * rows_xferred if (!blkdesc->bcpinfo.xfer_init) { + blkdesc->bcpinfo.xfer_init = true; + /* * first call the start_copy function, which will * retrieve details of the database table columns @@ -630,16 +664,19 @@ _blk_rowxfer_in(CS_BLKDESC * blkdesc, CS_INT rows_to_xfer, CS_INT * rows_xferred if (TDS_FAILED(tds_bcp_start_copy_in(tds, &blkdesc->bcpinfo))) { _ctclient_msg(NULL, CONN(blkdesc), "blk_rowxfer", 2, 5, 1, 140, ""); + + blkdesc->bcpinfo.xfer_init = false; return CS_FAIL; } - - blkdesc->bcpinfo.xfer_init = true; } for (each_row = 0; each_row < rows_to_xfer; each_row++ ) { if (tds_bcp_send_record(tds, &blkdesc->bcpinfo, _blk_get_col_data, _blk_null_error, each_row) == TDS_SUCCESS) { - /* FIXME */ + if (blkdesc->bcpinfo.next_col > 0) + return CS_BLK_HAS_TEXT; + } else { + return CS_FAIL; } } @@ -674,6 +711,15 @@ _blk_get_col_data(TDSBCPINFO *bulk, TDSCOLUMN *bindcol, int offset) tdsdump_log(TDS_DBG_FUNC, "_blk_get_col_data(%p, %p, %d)\n", bulk, bindcol, offset); + if (bindcol->column_nullbind) { + nullind = bindcol->column_nullbind; + nullind += offset; + } + if (bindcol->column_lenbind) { + datalen = bindcol->column_lenbind; + datalen += offset; + } + /* * Retrieve the initial bound column_varaddress * and increment it if offset specified @@ -681,21 +727,24 @@ _blk_get_col_data(TDSBCPINFO *bulk, TDSCOLUMN *bindcol, int offset) src = (unsigned char *) bindcol->column_varaddr; if (!src) { + if (nullind && *nullind == -1) { + null_column = true; + } + + bindcol->bcp_column_data->datalen = *datalen; + bindcol->bcp_column_data->is_null = null_column; + + if (is_blob_col(bindcol) + && bindcol->column_varaddr == NULL) { + /* Data will come piecemeal, via blk_textxfer. */ + return TDS_NO_MORE_RESULTS; + } + tdsdump_log(TDS_DBG_ERROR, "error source field not addressable\n"); return TDS_FAIL; } src += offset * bindcol->column_bindlen; - - if (bindcol->column_nullbind) { - nullind = bindcol->column_nullbind; - nullind += offset; - } - if (bindcol->column_lenbind) { - datalen = bindcol->column_lenbind; - datalen += offset; - } - srctype = bindcol->column_bindtype; /* passes to cs_convert */ tdsdump_log(TDS_DBG_INFO1, "blk_get_col_data srctype = %d\n", srctype); @@ -735,7 +784,7 @@ _blk_get_col_data(TDSBCPINFO *bulk, TDSCOLUMN *bindcol, int offset) null_column = true; } - if (!null_column) { + if (!null_column && !is_blob_type(bindcol->column_type)) { CS_DATAFMT_COMMON srcfmt, destfmt; CS_INT desttype; TDS_SERVER_TYPE tds_desttype = TDS_INVALID_TYPE; diff --git a/src/tds/bulk.c b/src/tds/bulk.c index b208963c10..1e233cb85f 100644 --- a/src/tds/bulk.c +++ b/src/tds/bulk.c @@ -51,6 +51,10 @@ #ifndef MAX #define MAX(a,b) ( (a) > (b) ? (a) : (b) ) #endif + +#ifndef MIN +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#endif /** \endcond */ /** @@ -77,6 +81,7 @@ static TDSRET tds5_process_insert_bulk_reply(TDSSOCKET * tds, TDSBCPINFO *bcpinf static TDSRET tds5_get_col_data_or_dflt(tds_bcp_get_col_data get_col_data, TDSBCPINFO *bulk, TDSCOLUMN *bcpcol, int offset, int colnum); +static int tds_bcp_is_bound(TDSBCPINFO *bcpinfo, TDSCOLUMN *colinfo); /** * Initialize BCP information. @@ -170,6 +175,9 @@ tds_bcp_init(TDSSOCKET *tds, TDSBCPINFO *bcpinfo) curcol->bcp_column_data = tds_alloc_bcp_column_data(sizeof(TDS_NUMERIC)); ((TDS_NUMERIC *) curcol->bcp_column_data->data)->precision = curcol->column_prec; ((TDS_NUMERIC *) curcol->bcp_column_data->data)->scale = curcol->column_scale; + } else if (bcpinfo->bind_count != 0 /* ctlib */ + && is_blob_col(curcol)) { + curcol->bcp_column_data = tds_alloc_bcp_column_data(0); } else { curcol->bcp_column_data = tds_alloc_bcp_column_data(MAX(curcol->column_size,curcol->on_server.column_size)); @@ -287,7 +295,8 @@ tds_bcp_start_insert_stmt(TDSSOCKET * tds, TDSBCPINFO * bcpinfo) for (i = 0; i < bcpinfo->bindinfo->num_cols; i++) { bcpcol = bcpinfo->bindinfo->columns[i]; - if (bcpcol->column_timestamp) + if (bcpcol->column_timestamp + || !tds_bcp_is_bound(bcpinfo, bcpcol)) continue; if (!bcpinfo->identity_insert_on && bcpcol->column_identity) continue; @@ -331,18 +340,21 @@ tds_bcp_start_insert_stmt(TDSSOCKET * tds, TDSBCPINFO * bcpinfo) static TDSRET tds7_send_record(TDSSOCKET *tds, TDSBCPINFO *bcpinfo, - tds_bcp_get_col_data get_col_data, tds_bcp_null_error null_error, int offset) + tds_bcp_get_col_data get_col_data, + tds_bcp_null_error null_error, int offset, int start_col) { int i; - tds_put_byte(tds, TDS_ROW_TOKEN); /* 0xd1 */ - for (i = 0; i < bcpinfo->bindinfo->num_cols; i++) { + if (start_col == 0) + tds_put_byte(tds, TDS_ROW_TOKEN); /* 0xd1 */ + for (i = start_col; i < bcpinfo->bindinfo->num_cols; i++) { TDS_INT save_size; unsigned char *save_data; TDSBLOB blob; TDSCOLUMN *bindcol; TDSRET rc; + bool has_text = false; bindcol = bcpinfo->bindinfo->columns[i]; @@ -353,7 +365,8 @@ tds7_send_record(TDSSOCKET *tds, TDSBCPINFO *bcpinfo, if ((!bcpinfo->identity_insert_on && bindcol->column_identity) || bindcol->column_timestamp || - bindcol->column_computed) { + bindcol->column_computed || + !tds_bcp_is_bound(bcpinfo, bindcol)) { continue; } @@ -361,6 +374,8 @@ tds7_send_record(TDSSOCKET *tds, TDSBCPINFO *bcpinfo, if (TDS_FAILED(rc)) { tdsdump_log(TDS_DBG_INFO1, "get_col_data (column %d) failed\n", i + 1); return rc; + } else if (rc == TDS_NO_MORE_RESULTS) { + has_text = true; } tdsdump_log(TDS_DBG_INFO1, "gotten column %d length %d null %d\n", i + 1, bindcol->bcp_column_data->datalen, bindcol->bcp_column_data->is_null); @@ -375,6 +390,9 @@ tds7_send_record(TDSSOCKET *tds, TDSBCPINFO *bcpinfo, return TDS_FAIL; } bindcol->column_cur_size = -1; + } else if (has_text) { + bindcol->column_cur_size + = bindcol->bcp_column_data->datalen; } else if (is_blob_col(bindcol)) { bindcol->column_cur_size = bindcol->bcp_column_data->datalen; memset(&blob, 0, sizeof(blob)); @@ -388,22 +406,27 @@ tds7_send_record(TDSSOCKET *tds, TDSBCPINFO *bcpinfo, bindcol->column_cur_size = save_size; bindcol->column_data = save_data; - TDS_PROPAGATE(rc); + if (TDS_FAILED(rc)) + return rc; + else if (has_text) { + bcpinfo->next_col = i + 1; + /* bcpinfo->text_sent = 0; */ + return TDS_NO_MORE_RESULTS; + } } return TDS_SUCCESS; } static TDSRET -tds5_send_record(TDSSOCKET *tds, TDSBCPINFO *bcpinfo, - tds_bcp_get_col_data get_col_data, tds_bcp_null_error null_error, int offset) +tds5_send_non_blobs(TDSSOCKET *tds, TDSBCPINFO *bcpinfo, + tds_bcp_get_col_data get_col_data, + tds_bcp_null_error null_error, int offset) { int row_pos; int row_sz_pos; - int blob_cols = 0; int var_cols_written = 0; TDS_INT old_record_size = bcpinfo->bindinfo->row_size; unsigned char *record = bcpinfo->bindinfo->current_row; - int i; memset(record, '\0', old_record_size); /* zero the rowbuffer */ @@ -434,14 +457,27 @@ tds5_send_record(TDSSOCKET *tds, TDSBCPINFO *bcpinfo, tds_put_smallint(tds, row_pos); tds_put_n(tds, record, row_pos); + return TDS_SUCCESS; +} +static TDSRET +tds5_send_record(TDSSOCKET *tds, TDSBCPINFO *bcpinfo, + tds_bcp_get_col_data get_col_data, + tds_bcp_null_error null_error, int offset, int start_col) +{ + int i; + if (start_col == 0) { + TDS_PROPAGATE(tds5_send_non_blobs(tds, bcpinfo, get_col_data, + null_error, offset)); + } /* row is done, now handle any text/image data */ - blob_cols = 0; + bcpinfo->blob_cols = 0; - for (i = 0; i < bcpinfo->bindinfo->num_cols; i++) { + for (i = start_col; i < bcpinfo->bindinfo->num_cols; i++) { TDSCOLUMN *bindcol = bcpinfo->bindinfo->columns[i]; if (is_blob_type(bindcol->on_server.column_type)) { + TDSRET rc; /* Elide trailing NULLs */ if (bindcol->bcp_column_data->is_null) { int j; @@ -462,21 +498,26 @@ tds5_send_record(TDSSOCKET *tds, TDSBCPINFO *bcpinfo, } } - TDS_PROPAGATE(tds5_get_col_data_or_dflt - (get_col_data, bcpinfo, bindcol, offset, - i)); + rc = tds5_get_col_data_or_dflt(get_col_data, bcpinfo, + bindcol, offset, i); + TDS_PROPAGATE(rc); /* unknown but zero */ tds_put_smallint(tds, 0); TDS_PUT_BYTE(tds, bindcol->on_server.column_type); - tds_put_byte(tds, 0xff - blob_cols); + tds_put_byte(tds, 0xff - bcpinfo->blob_cols); /* * offset of txptr we stashed during variable * column processing */ tds_put_smallint(tds, bindcol->column_textpos); tds_put_int(tds, bindcol->bcp_column_data->datalen); + if (rc == TDS_NO_MORE_RESULTS) { + bcpinfo->next_col = i + 1; + /* bcpinfo->text_sent = 0; */ + return rc; + } tds_put_n(tds, bindcol->bcp_column_data->data, bindcol->bcp_column_data->datalen); - blob_cols++; + bcpinfo->blob_cols++; } } @@ -497,6 +538,7 @@ tds_bcp_send_record(TDSSOCKET *tds, TDSBCPINFO *bcpinfo, tds_bcp_get_col_data get_col_data, tds_bcp_null_error null_error, int offset) { TDSRET rc; + int start_col = bcpinfo->next_col; tdsdump_log(TDS_DBG_FUNC, "tds_bcp_send_bcp_record(%p, %p, %p, %p, %d)\n", tds, bcpinfo, get_col_data, null_error, offset); @@ -504,12 +546,43 @@ tds_bcp_send_record(TDSSOCKET *tds, TDSBCPINFO *bcpinfo, if (tds->out_flag != TDS_BULK || tds_set_state(tds, TDS_WRITING) != TDS_WRITING) return TDS_FAIL; + if (start_col > 0) { + TDSCOLUMN *bindcol = bcpinfo->bindinfo->columns[start_col - 1]; + *bindcol->column_lenbind + = MIN((TDS_INT) bindcol->column_bindlen + - bcpinfo->text_sent, + *bindcol->column_lenbind); + if (IS_TDS7_PLUS(tds->conn) + && bindcol->column_varint_size == 8 + && *bindcol->column_lenbind > 0) { + /* Put PLP chunk length. */ + tds_put_int(tds, *bindcol->column_lenbind); + } + tds_put_n(tds, bindcol->column_varaddr, + *bindcol->column_lenbind); + bcpinfo->text_sent += *bindcol->column_lenbind; + if ((TDS_UINT) bcpinfo->text_sent < bindcol->column_bindlen) { + return TDS_SUCCESS; /* That's all for now. */ + } else if (!IS_TDS7_PLUS(tds->conn)) { + bcpinfo->blob_cols++; + } else if (bindcol->column_varint_size == 8) { + tds_put_int(tds, 0); /* Put PLP terminator. */ + } + bcpinfo->next_col = 0; + bcpinfo->text_sent = 0; + } + if (IS_TDS7_PLUS(tds->conn)) - rc = tds7_send_record(tds, bcpinfo, get_col_data, null_error, offset); + rc = tds7_send_record(tds, bcpinfo, get_col_data, null_error, + offset, start_col); else - rc = tds5_send_record(tds, bcpinfo, get_col_data, null_error, offset); + rc = tds5_send_record(tds, bcpinfo, get_col_data, null_error, + offset, start_col); + if (rc == TDS_NO_MORE_RESULTS) + return TDS_SUCCESS; tds_set_state(tds, TDS_SENDING); + bcpinfo->next_col = 0; return rc; } @@ -655,9 +728,10 @@ tds5_bcp_add_variable_columns(TDSBCPINFO *bcpinfo, tds_bcp_get_col_data get_col_ tdsdump_log(TDS_DBG_FUNC, "%4s %8s %8s %8s\n", "col", "ncols", "row_pos", "cpbytes"); - for (i = 0; i < bcpinfo->bindinfo->num_cols; i++) { + for (i = bcpinfo->next_col; i < bcpinfo->bindinfo->num_cols; i++) { unsigned int cpbytes = 0; TDSCOLUMN *bcpcol = bcpinfo->bindinfo->columns[i]; + TDSRET rc; /* * Is this column of "variable" type, i.e. NULLable @@ -673,9 +747,13 @@ tds5_bcp_add_variable_columns(TDSBCPINFO *bcpinfo, tds_bcp_get_col_data get_col_ tdsdump_log(TDS_DBG_FUNC, "%4d %8d %8d %8d\n", i, ncols, row_pos, cpbytes); - if (TDS_FAILED(tds5_get_col_data_or_dflt(get_col_data, bcpinfo, - bcpcol, offset, i))) + rc = tds5_get_col_data_or_dflt(get_col_data, bcpinfo, bcpcol, + offset, i); + if (TDS_FAILED(rc)) { return -1; + } else if (rc == TDS_NO_MORE_RESULTS) { + bcpinfo->next_col = i + 1; + } /* If it's a NOT NULL column, and we have no data, throw an error. * This is the behavior for Sybase, this function is only used for Sybase */ @@ -801,7 +879,8 @@ tds7_bcp_send_colmetadata(TDSSOCKET *tds, TDSBCPINFO *bcpinfo) bcpcol = bcpinfo->bindinfo->columns[i]; if ((!bcpinfo->identity_insert_on && bcpcol->column_identity) || bcpcol->column_timestamp || - bcpcol->column_computed) { + bcpcol->column_computed || + !tds_bcp_is_bound(bcpinfo, bcpcol)) { continue; } num_cols++; @@ -822,7 +901,8 @@ tds7_bcp_send_colmetadata(TDSSOCKET *tds, TDSBCPINFO *bcpinfo) if ((!bcpinfo->identity_insert_on && bcpcol->column_identity) || bcpcol->column_timestamp || - bcpcol->column_computed) { + bcpcol->column_computed || + !tds_bcp_is_bound(bcpinfo, bcpcol)) { continue; } @@ -1471,3 +1551,17 @@ tds_writetext_end(TDSSOCKET *tds) tds_set_state(tds, TDS_PENDING); return TDS_SUCCESS; } + + +static int tds_bcp_is_bound(TDSBCPINFO *bcpinfo, TDSCOLUMN *colinfo) +{ + return (bcpinfo && colinfo && + /* Don't interfere with dblib bulk insertion from files. */ + (bcpinfo->xfer_init == 0 + || colinfo->column_varaddr != NULL + || (colinfo->column_lenbind != NULL + && (*colinfo->column_lenbind != 0 + || (colinfo->column_nullbind != NULL + /* null-value for blk_textxfer ... */ + /* && *colinfo->column_nullbind == -1 */))))); +} diff --git a/src/tds/data.c b/src/tds/data.c index aa1bee69f0..5900cdcdaa 100644 --- a/src/tds/data.c +++ b/src/tds/data.c @@ -956,7 +956,7 @@ tds_generic_put(TDSSOCKET * tds, TDSCOLUMN * curcol, int bcp7) size = tds_fix_column_size(tds, curcol); src = curcol->column_data; - if (is_blob_col(curcol)) { + if (is_blob_col(curcol) && src != NULL) { blob = (TDSBLOB *) src; src = (unsigned char *) blob->textvalue; } @@ -1003,6 +1003,8 @@ tds_generic_put(TDSSOCKET * tds, TDSCOLUMN * curcol, int bcp7) * a bug in different server version that does * not accept a length here */ tds_put_int8(tds, bcp7 ? (TDS_INT8) -2 : (TDS_INT8) colsize); + if (blob == NULL) /* anticipate ctlib blk_textxfer */ + return TDS_SUCCESS; tds_put_int(tds, colsize); break; case 4: /* It's a BLOB... */ @@ -1040,6 +1042,8 @@ tds_generic_put(TDSSOCKET * tds, TDSCOLUMN * curcol, int bcp7) /* put real data */ if (blob) { tds_put_n(tds, s, colsize); + } else if (is_blob_col(curcol)) { + return TDS_SUCCESS; /* anticipate ctlib blk_textxfer */ } else { #ifdef WORDS_BIGENDIAN unsigned char buf[64]; @@ -1103,6 +1107,9 @@ tds_generic_put(TDSSOCKET * tds, TDSCOLUMN * curcol, int bcp7) /* put real data */ if (blob) { tds_put_n(tds, s, colsize); + } else if (is_blob_col(curcol)) { + /* accommodate ctlib blk_textxfer */ + return TDS_SUCCESS; } else { #ifdef WORDS_BIGENDIAN unsigned char buf[64]; From af06c45572c0c30ae8115c1c025921fb5056a060 Mon Sep 17 00:00:00 2001 From: "Aaron M. Ucko" Date: Mon, 13 May 2024 14:53:55 -0400 Subject: [PATCH 6/9] _blk_get_col_data: Bail (successfully) on null columns. Signed-off-by: Aaron M. Ucko --- src/ctlib/blk.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ctlib/blk.c b/src/ctlib/blk.c index 619a36e6a5..28658979b9 100644 --- a/src/ctlib/blk.c +++ b/src/ctlib/blk.c @@ -734,8 +734,10 @@ _blk_get_col_data(TDSBCPINFO *bulk, TDSCOLUMN *bindcol, int offset) bindcol->bcp_column_data->datalen = *datalen; bindcol->bcp_column_data->is_null = null_column; - if (is_blob_col(bindcol) - && bindcol->column_varaddr == NULL) { + if (null_column) { + return TDS_SUCCESS; + } else if (is_blob_col(bindcol) + && bindcol->column_varaddr == NULL) { /* Data will come piecemeal, via blk_textxfer. */ return TDS_NO_MORE_RESULTS; } From e025e3c5d3601397c1df3db18a8822e4e7363318 Mon Sep 17 00:00:00 2001 From: "Aaron M. Ucko" Date: Wed, 3 Jul 2024 12:48:38 -0400 Subject: [PATCH 7/9] Match Sybase ctlib's handling of overlong or malformatted BCP input. Adjust existing failure modes' boundaries and introduce the possibility of proceeding with truncated input with or without a message (with the latter occurring in some cases that require no conversion). As for messages, arrange to emit them via _ctclient_msg rather than _csclient_msg with the same (English) wording and context as Sybase yields, modulo formal punctuation differences and the usual discrepancies in layer and origin numbering. To that end: * Add internal BLK_CONV_* constants that serve both as (new) client message numbers and as possible return values for _cs_convert. * Cover those numbers in _ct_get_user_api_layer_error. * Have _cs_convert take the presence of a non-NULL handle (which only _blk_get_col_data supplies) to indicate that the caller wants bulk-insertion semantics, including in particular the possible use of those return values. * Keep a running count of rows sent for reporting purposes (determining the column number to cite by a linear scan of the bindinfo's columns array). Signed-off-by: Aaron M. Ucko --- include/ctlib.h | 7 +++- include/freetds/tds.h | 1 + src/ctlib/blk.c | 13 ++++++- src/ctlib/cs.c | 84 +++++++++++++++++++++++++++++++++++++++---- src/ctlib/ct.c | 9 +++++ src/tds/bulk.c | 1 + 6 files changed, 107 insertions(+), 8 deletions(-) diff --git a/include/ctlib.h b/include/ctlib.h index 49ca765ae0..09548f77b2 100644 --- a/include/ctlib.h +++ b/include/ctlib.h @@ -377,6 +377,11 @@ typedef union CS_DATAFMT user; } CS_DATAFMT_INTERNAL; +/* Alternate _cs_convert return values, which double as message numbers */ +#define BLK_CONV_OVERFLOW 25 +#define BLK_CONV_SYNTAX_ERROR 26 +#define BLK_CONV_TRUNCATION 42 + /* * internal prototypes */ @@ -396,7 +401,7 @@ int _cs_locale_copy_inplace(CS_LOCALE *new_locale, CS_LOCALE *orig); CS_INT _ct_get_string_length(const char *buf, CS_INT buflen); int _cs_convert_not_client(CS_CONTEXT *ctx, const TDSCOLUMN *curcol, CONV_RESULT *convert_buffer, unsigned char **p_src); - +/* NB: only _blk_get_col_data is expected to pass a non-NULL handle */ CS_RETCODE _cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, CS_VOID * srcdata, const CS_DATAFMT_COMMON * destfmt, CS_VOID * destdata, CS_INT * resultlen, diff --git a/include/freetds/tds.h b/include/freetds/tds.h index c0212f7756..8bb0e557e7 100644 --- a/include/freetds/tds.h +++ b/include/freetds/tds.h @@ -1698,6 +1698,7 @@ struct tds_bcpinfo TDS_INT text_sent; TDS_INT next_col; TDS_INT blob_cols; + TDS_INT rows_sent; }; TDSRET tds_bcp_init(TDSSOCKET *tds, TDSBCPINFO *bcpinfo); diff --git a/src/ctlib/blk.c b/src/ctlib/blk.c index 28658979b9..16668659f0 100644 --- a/src/ctlib/blk.c +++ b/src/ctlib/blk.c @@ -814,7 +814,18 @@ _blk_get_col_data(TDSBCPINFO *bulk, TDSCOLUMN *bindcol, int offset) (CS_VOID *) coldata->data, &destlen, tds_desttype, (CS_VOID **) &coldata->data); - if (result != CS_SUCCEED) { + if (result > CS_SUCCEED) { + const TDSRESULTINFO * info = bulk->bindinfo; + TDS_USMALLINT colnum; + for (colnum = 0; colnum < info->num_cols; ++colnum) { + if (info->columns[colnum] == bindcol) + break; + } + _ctclient_msg(ctx, CONN(blkdesc), "blk_rowxfer", + 2, 7, 1, result, "%d,%hu", + bulk->rows_sent + 1, colnum + 1); + } + if (result != CS_SUCCEED && result != BLK_CONV_TRUNCATION) { tdsdump_log(TDS_DBG_ERROR, "conversion from srctype %d to desttype %d failed\n", srctype, desttype); return TDS_FAIL; diff --git a/src/ctlib/cs.c b/src/ctlib/cs.c index 5d6a96c86b..8f2baec778 100644 --- a/src/ctlib/cs.c +++ b/src/ctlib/cs.c @@ -26,6 +26,7 @@ #include #include +#include #if HAVE_STDLIB_H #include @@ -613,8 +614,16 @@ _cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, if (src_len > destlen) { tdsdump_log(TDS_DBG_FUNC, "error: src_len > destlen\n"); - _csclient_msg(ctx, "cs_convert", 2, 4, 1, 36, ""); - ret = CS_FAIL; + if (handle == NULL) { + _csclient_msg(ctx, "cs_convert", + 2, 4, 1, 36, ""); + ret = CS_FAIL; + } else if (srcfmt->datatype + == CS_VARBINARY_TYPE) { + ret = BLK_CONV_OVERFLOW; + } else { + ret = CS_SUCCEED; + } } else { switch (destfmt->format) { case CS_FMT_PADNULL: @@ -643,6 +652,12 @@ _cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, memcpy(dest, srcdata, minlen); *resultlen = minlen; + if (src_len > destlen && handle != NULL) { + src_len = destlen; + if (srcfmt->datatype == CS_VARCHAR_TYPE) + ret = BLK_CONV_TRUNCATION; + } + if (src_len > destlen) { tdsdump_log(TDS_DBG_FUNC, "error: src_len > destlen\n"); _csclient_msg(ctx, "cs_convert", 2, 4, 1, 36, ""); @@ -754,6 +769,38 @@ _cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, len = tds_convert(ctx->tds_ctx, src_type, srcdata, src_len, desttype, &cres); tdsdump_log(TDS_DBG_FUNC, "cs_convert() tds_convert returned %d\n", len); + if (len == TDS_CONVERT_SYNTAX && handle != NULL + && is_binary_type(desttype)) { + char * s = (char *) srcdata; + int src_len2 = destlen * 2; + if (src_len > 1 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) + src_len2 += 2; + /* Match full input's low order bit to get correct phase. */ + if (src_len & 1) + --src_len2; + if (src_len2 < src_len) { + len = tds_convert(ctx->tds_ctx, src_type, srcdata, + src_len2, desttype, &cres); + } + tdsdump_log(TDS_DBG_FUNC, + "cs_convert(): retry returned %d\n", len); + /* + * Check the last character to avoid interference from + * space and NUL stripping. + */ + if (len == destlen && isxdigit(s[src_len2 - 1])) { + if (srcfmt->datatype == CS_VARCHAR_TYPE) { + free(cres.ib); + return BLK_CONV_OVERFLOW; + } else { + ret = BLK_CONV_TRUNCATION; + } + } else { + if (len >= 0) + free(cres.ib); + return BLK_CONV_SYNTAX_ERROR; + } + } switch (len) { case TDS_CONVERT_NOAVAIL: @@ -789,8 +836,16 @@ _cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, ret = CS_SUCCEED; if (len > destlen) { tdsdump_log(TDS_DBG_FUNC, "error_handler: Data-conversion resulted in overflow\n"); - _csclient_msg(ctx, "cs_convert", 2, 4, 1, 36, ""); - ret = CS_FAIL; + if (handle == NULL) { + _csclient_msg(ctx, "cs_convert", 2, 4, 1, 36, + ""); + ret = CS_FAIL; + } else if (srcfmt->datatype == CS_VARBINARY_TYPE + || srcfmt->datatype == CS_VARCHAR_TYPE) { + ret = BLK_CONV_OVERFLOW; + } else { + ret = BLK_CONV_TRUNCATION; + } len = destlen; } if (handle == NULL) { @@ -857,9 +912,26 @@ _cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, ret = CS_SUCCEED; if (len > destlen) { tdsdump_log(TDS_DBG_FUNC, "Data-conversion resulted in overflow\n"); - _csclient_msg(ctx, "cs_convert", 2, 4, 1, 36, ""); len = destlen; - ret = CS_FAIL; + if (handle == NULL) { + _csclient_msg(ctx, "cs_convert", 2, 4, 1, 36, + ""); + ret = CS_FAIL; + } else if (is_char_type(src_type)) { + if (srcfmt->datatype == CS_VARCHAR_TYPE) + ret = BLK_CONV_TRUNCATION; + } else if (is_binary_type(src_type)) { + if (srcfmt->datatype == CS_VARBINARY_TYPE) { + ret = BLK_CONV_OVERFLOW; + } else { + ret = BLK_CONV_TRUNCATION; + len &= ~1; + } + } else if (is_datetime_type(src_type)) { + ret = BLK_CONV_TRUNCATION; + } else { + ret = BLK_CONV_OVERFLOW; + } } switch (destfmt->format) { diff --git a/src/ctlib/ct.c b/src/ctlib/ct.c index 55fc6127cb..8a02034acd 100644 --- a/src/ctlib/ct.c +++ b/src/ctlib/ct.c @@ -154,6 +154,15 @@ _ct_get_user_api_layer_error(int error) case 15: return "Use direction CS_BLK_IN or CS_BLK_OUT for a bulk copy operation."; break; + case 25: + return "Failed in conversion routine - condition overflow." + " col = %1! row = %2!."; + case 26: + return "Failed in conversion routine - syntax error." + " col = %1! row = %2!."; + case 42: + return "Data truncated while doing local character set" + " conversion. col = %1! row = %2!."; case 51: return "Exactly one of context and connection must be non-NULL."; break; diff --git a/src/tds/bulk.c b/src/tds/bulk.c index 1e233cb85f..116b0f8fcb 100644 --- a/src/tds/bulk.c +++ b/src/tds/bulk.c @@ -583,6 +583,7 @@ tds_bcp_send_record(TDSSOCKET *tds, TDSBCPINFO *bcpinfo, return TDS_SUCCESS; tds_set_state(tds, TDS_SENDING); bcpinfo->next_col = 0; + bcpinfo->rows_sent++; return rc; } From 3f51545dbe242560fa58503f91d520cb342d9522 Mon Sep 17 00:00:00 2001 From: "Aaron M. Ucko" Date: Wed, 28 Aug 2024 13:33:47 -0400 Subject: [PATCH 8/9] BCP into computed columns again when allowing triggers. The destination view may well have an INSTEAD OF INSERT trigger that makes meaningful use of the supplied values. Signed-off-by: Aaron M. Ucko --- include/freetds/tds.h | 1 + src/tds/bulk.c | 51 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/include/freetds/tds.h b/include/freetds/tds.h index 8bb0e557e7..b17bf139e3 100644 --- a/include/freetds/tds.h +++ b/include/freetds/tds.h @@ -1699,6 +1699,7 @@ struct tds_bcpinfo TDS_INT next_col; TDS_INT blob_cols; TDS_INT rows_sent; + bool with_triggers; }; TDSRET tds_bcp_init(TDSSOCKET *tds, TDSBCPINFO *bcpinfo); diff --git a/src/tds/bulk.c b/src/tds/bulk.c index 116b0f8fcb..8ba54eb95e 100644 --- a/src/tds/bulk.c +++ b/src/tds/bulk.c @@ -36,6 +36,8 @@ #include #endif /* HAVE_STDLIB_H */ +#include + #include #include @@ -268,6 +270,26 @@ tds7_build_bulk_insert_stmt(TDSSOCKET * tds, TDSPBCB * clause, TDSCOLUMN * bcpco return TDS_SUCCESS; } +static const char * uc_str(const DSTR * ds) +{ + /* return strupper(strdup(tds_dstr_cstr(s))); */ + const char * s = tds_dstr_cstr(ds); + size_t n = strcspn(s, "abcdefghijklmnopqrstuvwxyz"); + if (s[n] == '\0') { + return s; /* already all uppercase */ + } else { + char * result; + size_t i; + n = tds_dstr_len(ds); + result = malloc(n + 1); + for (i = 0; i < n; i++) { + result[i] = toupper(s[i]); + } + result[n] = '\0'; + return result; + } +} + /** * Prepare the query to be sent to server to request BCP information * \tds @@ -284,6 +306,7 @@ tds_bcp_start_insert_stmt(TDSSOCKET * tds, TDSBCPINFO * bcpinfo) TDSCOLUMN *bcpcol; TDSPBCB colclause; char clause_buffer[4096] = { 0 }; + bool triggers_checked = !bcpinfo->hint; /* Done on demand */ colclause.pb = clause_buffer; colclause.cb = sizeof(clause_buffer); @@ -300,8 +323,23 @@ tds_bcp_start_insert_stmt(TDSSOCKET * tds, TDSBCPINFO * bcpinfo) continue; if (!bcpinfo->identity_insert_on && bcpcol->column_identity) continue; - if (bcpcol->column_computed) - continue; + if (bcpcol->column_computed) { + if ( !triggers_checked ) { + const char * uc_hint + = uc_str(&bcpinfo->hint); + if (strstr(uc_hint, "FIRE_TRIGGERS")) { + bcpinfo->with_triggers = true; + } + if (uc_hint + != tds_dstr_cstr(&bcpinfo->hint)) { + free((char *)uc_hint); + } + triggers_checked = true; + } + if ( !bcpinfo->with_triggers ) { + continue; + } + } tds7_build_bulk_insert_stmt(tds, &colclause, bcpcol, firstcol); firstcol = 0; } @@ -365,7 +403,8 @@ tds7_send_record(TDSSOCKET *tds, TDSBCPINFO *bcpinfo, if ((!bcpinfo->identity_insert_on && bindcol->column_identity) || bindcol->column_timestamp || - bindcol->column_computed || + (bindcol->column_computed + && !bcpinfo->with_triggers) || !tds_bcp_is_bound(bcpinfo, bindcol)) { continue; } @@ -880,7 +919,8 @@ tds7_bcp_send_colmetadata(TDSSOCKET *tds, TDSBCPINFO *bcpinfo) bcpcol = bcpinfo->bindinfo->columns[i]; if ((!bcpinfo->identity_insert_on && bcpcol->column_identity) || bcpcol->column_timestamp || - bcpcol->column_computed || + (bcpcol->column_computed + && !bcpinfo->with_triggers) || !tds_bcp_is_bound(bcpinfo, bcpcol)) { continue; } @@ -902,7 +942,8 @@ tds7_bcp_send_colmetadata(TDSSOCKET *tds, TDSBCPINFO *bcpinfo) if ((!bcpinfo->identity_insert_on && bcpcol->column_identity) || bcpcol->column_timestamp || - bcpcol->column_computed || + (bcpcol->column_computed + && !bcpinfo->with_triggers) || !tds_bcp_is_bound(bcpinfo, bcpcol)) { continue; } From 1c45955f0cb9bdf63809f9e5976d7d86e06bbccd Mon Sep 17 00:00:00 2001 From: "Aaron M. Ucko" Date: Mon, 13 May 2024 14:59:25 -0400 Subject: [PATCH 9/9] Declare and implement blk_sethints. Signed-off-by: Aaron M. Ucko --- include/bkpublic.h | 1 + src/ctlib/blk.c | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/include/bkpublic.h b/include/bkpublic.h index 9097f5f28d..7890824f51 100644 --- a/include/bkpublic.h +++ b/include/bkpublic.h @@ -57,6 +57,7 @@ CS_RETCODE blk_rowxfer(CS_BLKDESC * blkdesc); CS_RETCODE blk_rowxfer_mult(CS_BLKDESC * blkdesc, CS_INT * row_count); CS_RETCODE blk_sendrow(CS_BLKDESC * blkdesc, CS_BLK_ROW * row); CS_RETCODE blk_sendtext(CS_BLKDESC * blkdesc, CS_BLK_ROW * row, CS_BYTE * buffer, CS_INT buflen); +CS_RETCODE blk_sethints(CS_BLKDESC* blkdesc, CS_CHAR* hints, CS_INT hintslen); CS_RETCODE blk_srvinit(SRV_PROC * srvproc, CS_BLKDESC * blkdescp); CS_RETCODE blk_textxfer(CS_BLKDESC * blkdesc, CS_BYTE * buffer, CS_INT buflen, CS_INT * outlen); diff --git a/src/ctlib/blk.c b/src/ctlib/blk.c index 16668659f0..2053f1325d 100644 --- a/src/ctlib/blk.c +++ b/src/ctlib/blk.c @@ -514,6 +514,17 @@ blk_sendtext(CS_BLKDESC * blkdesc, CS_BLK_ROW * row, CS_BYTE * buffer, CS_INT bu return CS_FAIL; } +CS_RETCODE +blk_sethints(CS_BLKDESC* blkdesc, CS_CHAR* hints, CS_INT hintslen) +{ + if (blkdesc != NULL + && tds_dstr_copyn(&blkdesc->bcpinfo.hint, hints, hintslen) != NULL) + return CS_SUCCEED; + else + return CS_FAIL; + +} + CS_RETCODE blk_srvinit(SRV_PROC * srvproc, CS_BLKDESC * blkdescp) {