From 26003acae5e4c896e7100049c56479ea0d9f9108 Mon Sep 17 00:00:00 2001 From: kination Date: Fri, 6 Mar 2026 20:18:10 +0900 Subject: [PATCH] Add test features --- vine-core/tests/arrow_bridge_tests.rs | 1 - vine-core/tests/storage_reader_tests.rs | 90 +++++------------ vine-core/tests/streaming_writer_v2_tests.rs | 27 ++++-- vine-core/tests/vine_batch_writer_tests.rs | 90 ++++++----------- .../tests/vine_streaming_writer_tests.rs | 97 +++++++++---------- .../kination/vine/VineJniIntegrationTest.java | 48 +++++++++ .../kination/vine/VineMetadataReaderTest.java | 79 +++++++++++++++ .../io/kination/vine/VineTypeMappingTest.java | 72 ++++++++++++++ 8 files changed, 317 insertions(+), 187 deletions(-) create mode 100644 vine-trino/src/test/java/io/kination/vine/VineJniIntegrationTest.java create mode 100644 vine-trino/src/test/java/io/kination/vine/VineMetadataReaderTest.java create mode 100644 vine-trino/src/test/java/io/kination/vine/VineTypeMappingTest.java diff --git a/vine-core/tests/arrow_bridge_tests.rs b/vine-core/tests/arrow_bridge_tests.rs index d34c82a..4a406e1 100644 --- a/vine-core/tests/arrow_bridge_tests.rs +++ b/vine-core/tests/arrow_bridge_tests.rs @@ -7,7 +7,6 @@ use std::sync::Arc; #[test] fn test_arrow_ipc_serialization_roundtrip() { - // Create a simple RecordBatch directly without CSV conversion let schema = Schema::new(vec![ Field::new("id", DataType::Int32, false), Field::new("name", DataType::Utf8, true), diff --git a/vine-core/tests/storage_reader_tests.rs b/vine-core/tests/storage_reader_tests.rs index 87d56a4..47c7e63 100644 --- a/vine-core/tests/storage_reader_tests.rs +++ b/vine-core/tests/storage_reader_tests.rs @@ -1,6 +1,7 @@ use vine_core::storage_reader::read_vine_data; use vine_core::metadata::{Metadata, MetadataField}; use vine_core::vortex_exp::write_vortex_file; +use vine_core::arrow_bridge::vortex_to_arrow; use tempfile::tempdir; use std::fs; @@ -24,33 +25,36 @@ fn create_test_metadata() -> Metadata { ) } +/// Helper: count total rows from read results using Arrow conversion +fn total_rows(arrays: &[vortex::ArrayRef]) -> usize { + arrays.iter().map(|a| { + vortex_to_arrow(a, true).expect("Failed to convert").num_rows() + }).sum() +} + #[test] fn test_read_vine_data_single_file() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Create metadata let metadata = create_test_metadata(); let meta_path = base_path.join("vine_meta.json"); metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - // Create date directory let date_dir = base_path.join("2024-01-15"); fs::create_dir(&date_dir).expect("Failed to create date dir"); - // Write test data - let csv_rows = vec!["1,Alice".to_string(), "2,Bob".to_string()]; - let csv_rows_refs: Vec<&str> = csv_rows.iter().map(|s| s.as_str()).collect(); let vtx_path = date_dir.join("data_120000_000000.vtx"); - write_vortex_file(&vtx_path, &metadata, &csv_rows_refs) + write_vortex_file(&vtx_path, &metadata, &["1,Alice", "2,Bob"]) .expect("Failed to write vortex file"); - // Read data let result = read_vine_data(base_path.to_str().unwrap()); + assert_eq!(result.len(), 1); // 1 file = 1 array + assert_eq!(total_rows(&result), 2); - assert_eq!(result.len(), 2); - assert_eq!(result[0], "1,Alice"); - assert_eq!(result[1], "2,Bob"); + // Verify column structure + let batch = vortex_to_arrow(&result[0], true).expect("Failed to convert"); + assert_eq!(batch.num_columns(), 2); } #[test] @@ -58,33 +62,23 @@ fn test_read_vine_data_multiple_files() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Create metadata let metadata = create_test_metadata(); let meta_path = base_path.join("vine_meta.json"); metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - // Create date directory let date_dir = base_path.join("2024-01-15"); fs::create_dir(&date_dir).expect("Failed to create date dir"); - // Write first file - let csv_rows1 = vec!["1,Alice".to_string()]; - let csv_rows1_refs: Vec<&str> = csv_rows1.iter().map(|s| s.as_str()).collect(); let vtx_path1 = date_dir.join("data_120000_000000.vtx"); - write_vortex_file(&vtx_path1, &metadata, &csv_rows1_refs) + write_vortex_file(&vtx_path1, &metadata, &["1,Alice"]) .expect("Failed to write first vortex file"); - // Write second file - let csv_rows2 = vec!["2,Bob".to_string()]; - let csv_rows2_refs: Vec<&str> = csv_rows2.iter().map(|s| s.as_str()).collect(); let vtx_path2 = date_dir.join("data_130000_000000.vtx"); - write_vortex_file(&vtx_path2, &metadata, &csv_rows2_refs) + write_vortex_file(&vtx_path2, &metadata, &["2,Bob"]) .expect("Failed to write second vortex file"); - // Read data let result = read_vine_data(base_path.to_str().unwrap()); - - assert_eq!(result.len(), 2); + assert_eq!(total_rows(&result), 2); } #[test] @@ -92,35 +86,24 @@ fn test_read_vine_data_multiple_dates() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Create metadata let metadata = create_test_metadata(); let meta_path = base_path.join("vine_meta.json"); metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - // Create first date directory let date_dir1 = base_path.join("2024-01-14"); fs::create_dir(&date_dir1).expect("Failed to create first date dir"); - let csv_rows1 = vec!["1,Alice".to_string()]; - let csv_rows1_refs: Vec<&str> = csv_rows1.iter().map(|s| s.as_str()).collect(); let vtx_path1 = date_dir1.join("data_120000_000000.vtx"); - write_vortex_file(&vtx_path1, &metadata, &csv_rows1_refs) + write_vortex_file(&vtx_path1, &metadata, &["1,Alice"]) .expect("Failed to write first vortex file"); - // Create second date directory let date_dir2 = base_path.join("2024-01-15"); fs::create_dir(&date_dir2).expect("Failed to create second date dir"); - let csv_rows2 = vec!["2,Bob".to_string()]; - let csv_rows2_refs: Vec<&str> = csv_rows2.iter().map(|s| s.as_str()).collect(); let vtx_path2 = date_dir2.join("data_120000_000000.vtx"); - write_vortex_file(&vtx_path2, &metadata, &csv_rows2_refs) + write_vortex_file(&vtx_path2, &metadata, &["2,Bob"]) .expect("Failed to write second vortex file"); - // Read data (should be in chronological order) let result = read_vine_data(base_path.to_str().unwrap()); - - assert_eq!(result.len(), 2); - assert_eq!(result[0], "1,Alice"); // 2024-01-14 comes first - assert_eq!(result[1], "2,Bob"); // 2024-01-15 comes second + assert_eq!(total_rows(&result), 2); } #[test] @@ -128,14 +111,11 @@ fn test_read_vine_data_empty_directory() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Create metadata let metadata = create_test_metadata(); let meta_path = base_path.join("vine_meta.json"); metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - // Read data from empty directory let result = read_vine_data(base_path.to_str().unwrap()); - assert!(result.is_empty()); } @@ -144,10 +124,7 @@ fn test_read_vine_data_missing_metadata() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Don't create metadata file let result = read_vine_data(base_path.to_str().unwrap()); - - // Should return empty vector on error assert!(result.is_empty()); } @@ -156,32 +133,22 @@ fn test_read_vine_data_ignores_non_vtx_files() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Create metadata let metadata = create_test_metadata(); let meta_path = base_path.join("vine_meta.json"); metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - // Create date directory let date_dir = base_path.join("2024-01-15"); fs::create_dir(&date_dir).expect("Failed to create date dir"); - // Write vtx file - let csv_rows = vec!["1,Alice".to_string()]; - let csv_rows_refs: Vec<&str> = csv_rows.iter().map(|s| s.as_str()).collect(); let vtx_path = date_dir.join("data_120000_000000.vtx"); - write_vortex_file(&vtx_path, &metadata, &csv_rows_refs) + write_vortex_file(&vtx_path, &metadata, &["1,Alice"]) .expect("Failed to write vortex file"); - // Create non-vtx file let txt_path = date_dir.join("README.txt"); fs::write(&txt_path, "This should be ignored").expect("Failed to write txt file"); - // Read data let result = read_vine_data(base_path.to_str().unwrap()); - - // Should only read the .vtx file - assert_eq!(result.len(), 1); - assert_eq!(result[0], "1,Alice"); + assert_eq!(total_rows(&result), 1); } #[test] @@ -189,28 +156,19 @@ fn test_read_vine_data_ignores_invalid_date_directories() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Create metadata let metadata = create_test_metadata(); let meta_path = base_path.join("vine_meta.json"); metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - // Create valid date directory let valid_date_dir = base_path.join("2024-01-15"); fs::create_dir(&valid_date_dir).expect("Failed to create valid date dir"); - let csv_rows = vec!["1,Alice".to_string()]; - let csv_rows_refs: Vec<&str> = csv_rows.iter().map(|s| s.as_str()).collect(); let vtx_path = valid_date_dir.join("data_120000_000000.vtx"); - write_vortex_file(&vtx_path, &metadata, &csv_rows_refs) + write_vortex_file(&vtx_path, &metadata, &["1,Alice"]) .expect("Failed to write vortex file"); - // Create invalid date directory let invalid_date_dir = base_path.join("not-a-date"); fs::create_dir(&invalid_date_dir).expect("Failed to create invalid date dir"); - // Read data let result = read_vine_data(base_path.to_str().unwrap()); - - // Should only read from valid date directory - assert_eq!(result.len(), 1); - assert_eq!(result[0], "1,Alice"); + assert_eq!(total_rows(&result), 1); } diff --git a/vine-core/tests/streaming_writer_v2_tests.rs b/vine-core/tests/streaming_writer_v2_tests.rs index 73a197e..7b9c69a 100644 --- a/vine-core/tests/streaming_writer_v2_tests.rs +++ b/vine-core/tests/streaming_writer_v2_tests.rs @@ -1,6 +1,7 @@ use vine_core::streaming_writer_v2::StreamingWriterV2; use vine_core::writer_config::WriterConfig; use vine_core::metadata::{Metadata, MetadataField}; +use vine_core::vortex_exp::build_struct_array; use tempfile::tempdir; use chrono::Local; @@ -24,6 +25,11 @@ fn create_test_metadata() -> Metadata { ) } +/// Helper: build VortexArrayRef from comma-separated rows +fn build_test_array(metadata: &Metadata, rows: &[&str]) -> vortex::ArrayRef { + build_struct_array(metadata, rows).expect("Failed to build test array") +} + #[test] fn test_streaming_writer_v2_basic() { let temp_dir = tempdir().expect("Failed to create temp dir"); @@ -37,15 +43,17 @@ fn test_streaming_writer_v2_basic() { .expect("Failed to create writer"); // Write and accumulate - writer.write_batch(&["1,Alice", "2,Bob"]).expect("Write failed"); + let array1 = build_test_array(&metadata, &["1,Alice", "2,Bob"]); + writer.write_batch(&array1).expect("Write failed"); assert_eq!(writer.buffered_rows(), 2); assert_eq!(writer.buffered_chunks(), 1); - writer.write_batch(&["3,Charlie"]).expect("Write failed"); + let array2 = build_test_array(&metadata, &["3,Charlie"]); + writer.write_batch(&array2).expect("Write failed"); assert_eq!(writer.buffered_rows(), 3); assert_eq!(writer.buffered_chunks(), 2); - // Flush - should write to file and return summary + // Flush let summary = writer.flush().expect("Flush failed"); assert!(summary.is_some(), "Should return flush summary"); let summary = summary.unwrap(); @@ -59,7 +67,8 @@ fn test_streaming_writer_v2_basic() { assert!(writer.bytes_written() > 0); // Write more (new file) - writer.write_batch(&["4,Diana"]).expect("Write failed"); + let array3 = build_test_array(&metadata, &["4,Diana"]); + writer.write_batch(&array3).expect("Write failed"); writer.close().expect("Close failed"); // Verify files @@ -85,19 +94,18 @@ fn test_auto_flush() { let metadata = create_test_metadata(); metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - // Create writer with small max_rows_per_file let mut config = WriterConfig::default(); config.max_rows_per_file = 5; let mut writer = StreamingWriterV2::with_config(path.to_path_buf(), config) .expect("Failed to create writer"); - // Write 3 rows (no flush yet) - writer.write_batch(&["1,A", "2,B", "3,C"]).expect("Write failed"); + let array1 = build_test_array(&metadata, &["1,A", "2,B", "3,C"]); + writer.write_batch(&array1).expect("Write failed"); assert_eq!(writer.buffered_rows(), 3); - // Write 3 more rows (3+3 > 5, so flushes first 3 data, then add 3) - writer.write_batch(&["4,D", "5,E", "6,F"]).expect("Write failed"); + let array2 = build_test_array(&metadata, &["4,D", "5,E", "6,F"]); + writer.write_batch(&array2).expect("Write failed"); assert_eq!(writer.buffered_rows(), 3); writer.close().expect("Close failed"); @@ -115,7 +123,6 @@ fn test_empty_flush() { let mut writer = StreamingWriterV2::new(path.to_path_buf()) .expect("Failed to create writer"); - // Flush without writing should return None let summary = writer.flush().expect("Flush should succeed"); assert!(summary.is_none(), "Empty flush should return None"); assert_eq!(writer.bytes_written(), 0); diff --git a/vine-core/tests/vine_batch_writer_tests.rs b/vine-core/tests/vine_batch_writer_tests.rs index f58f9c2..827f6f8 100644 --- a/vine-core/tests/vine_batch_writer_tests.rs +++ b/vine-core/tests/vine_batch_writer_tests.rs @@ -1,6 +1,8 @@ use vine_core::vine_batch_writer::VineBatchWriter; use vine_core::metadata::{Metadata, MetadataField}; use vine_core::storage_reader::read_vine_data; +use vine_core::vortex_exp::build_struct_array; +use vine_core::arrow_bridge::vortex_to_arrow; use tempfile::tempdir; use std::fs; @@ -24,62 +26,44 @@ fn create_test_metadata() -> Metadata { ) } -#[test] -fn test_vine_batch_writer_write() { - let temp_dir = tempdir().expect("Failed to create temp dir"); - let base_path = temp_dir.path(); - - // Create metadata - let metadata = create_test_metadata(); - let meta_path = base_path.join("vine_meta.json"); - metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - - // Write data - let rows = vec!["1,Alice", "2,Bob", "3,Charlie"]; - VineBatchWriter::write(base_path, &rows).expect("Failed to write data"); - - // Verify data was written - let result = read_vine_data(base_path.to_str().unwrap()); - assert_eq!(result.len(), 3); - assert_eq!(result[0], "1,Alice"); - assert_eq!(result[1], "2,Bob"); - assert_eq!(result[2], "3,Charlie"); +/// Helper: build a VortexArrayRef from comma-separated rows using metadata +fn build_test_array(metadata: &Metadata, rows: &[&str]) -> vortex::ArrayRef { + build_struct_array(metadata, rows).expect("Failed to build test array") } #[test] -fn test_vine_batch_writer_write_empty() { +fn test_vine_batch_writer_write() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Create metadata let metadata = create_test_metadata(); let meta_path = base_path.join("vine_meta.json"); metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - // Write empty data - let rows: Vec<&str> = vec![]; - VineBatchWriter::write(base_path, &rows).expect("Failed to write empty data"); + let array = build_test_array(&metadata, &["1,Alice", "2,Bob", "3,Charlie"]); + VineBatchWriter::write(base_path, &array).expect("Failed to write data"); - // Verify file was created (even if empty) - let date_dirs: Vec<_> = fs::read_dir(base_path) - .expect("Failed to read dir") - .filter_map(|e| e.ok()) - .filter(|e| e.path().is_dir()) - .collect(); + // Verify data was written and can be read back + let result = read_vine_data(base_path.to_str().unwrap()); + assert_eq!(result.len(), 1); // 1 array (1 file) - assert!(!date_dirs.is_empty()); + // Convert back to Arrow to verify contents + let batch = vortex_to_arrow(&result[0], true).expect("Failed to convert to Arrow"); + assert_eq!(batch.num_rows(), 3); + assert_eq!(batch.num_columns(), 2); } #[test] -fn test_vine_batch_writer_write_missing_metadata() { +fn test_vine_batch_writer_write_without_metadata() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Don't create metadata file - let rows = vec!["1,Alice"]; - let result = VineBatchWriter::write(base_path, &rows); - - assert!(result.is_err()); + // With direct array writes, metadata file is NOT required on disk + // because the array already carries its schema + let metadata = create_test_metadata(); + let array = build_test_array(&metadata, &["1,Alice"]); + let result = VineBatchWriter::write(base_path, &array); + assert!(result.is_ok(), "Direct array write should succeed without metadata file"); } #[test] @@ -87,14 +71,12 @@ fn test_vine_batch_writer_creates_date_partition() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Create metadata let metadata = create_test_metadata(); let meta_path = base_path.join("vine_meta.json"); metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - // Write data - let rows = vec!["1,Alice"]; - VineBatchWriter::write(base_path, &rows).expect("Failed to write data"); + let array = build_test_array(&metadata, &["1,Alice"]); + VineBatchWriter::write(base_path, &array).expect("Failed to write data"); // Verify date partition directory was created let date_dirs: Vec<_> = fs::read_dir(base_path) @@ -105,11 +87,10 @@ fn test_vine_batch_writer_creates_date_partition() { assert_eq!(date_dirs.len(), 1); - // Verify directory name is a valid date (YYYY-MM-DD format) let dir_name = date_dirs[0].file_name(); let dir_name_str = dir_name.to_str().unwrap(); assert!(dir_name_str.contains('-')); - assert_eq!(dir_name_str.len(), 10); // YYYY-MM-DD is 10 characters + assert_eq!(dir_name_str.len(), 10); // YYYY-MM-DD } #[test] @@ -117,16 +98,13 @@ fn test_vine_batch_writer_creates_vtx_file() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Create metadata let metadata = create_test_metadata(); let meta_path = base_path.join("vine_meta.json"); metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - // Write data - let rows = vec!["1,Alice"]; - VineBatchWriter::write(base_path, &rows).expect("Failed to write data"); + let array = build_test_array(&metadata, &["1,Alice"]); + VineBatchWriter::write(base_path, &array).expect("Failed to write data"); - // Find the created .vtx file let date_dirs: Vec<_> = fs::read_dir(base_path) .expect("Failed to read dir") .filter_map(|e| e.ok()) @@ -148,7 +126,6 @@ fn test_vine_batch_writer_creates_vtx_file() { assert_eq!(vtx_files.len(), 1); - // Verify filename format (data_HHMMSS_microseconds.vtx) let file_name = vtx_files[0].file_name(); let file_name_str = file_name.to_str().unwrap(); assert!(file_name_str.starts_with("data_")); @@ -160,20 +137,17 @@ fn test_vine_batch_writer_multiple_writes() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Create metadata let metadata = create_test_metadata(); let meta_path = base_path.join("vine_meta.json"); metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - // Write first batch - let rows1 = vec!["1,Alice"]; - VineBatchWriter::write(base_path, &rows1).expect("Failed to write first batch"); + let array1 = build_test_array(&metadata, &["1,Alice"]); + VineBatchWriter::write(base_path, &array1).expect("Failed to write first batch"); - // Write second batch - let rows2 = vec!["2,Bob"]; - VineBatchWriter::write(base_path, &rows2).expect("Failed to write second batch"); + let array2 = build_test_array(&metadata, &["2,Bob"]); + VineBatchWriter::write(base_path, &array2).expect("Failed to write second batch"); - // Verify both batches were written let result = read_vine_data(base_path.to_str().unwrap()); + // Each write creates a separate file, so 2 arrays assert_eq!(result.len(), 2); } diff --git a/vine-core/tests/vine_streaming_writer_tests.rs b/vine-core/tests/vine_streaming_writer_tests.rs index 7f75b92..9473f3d 100644 --- a/vine-core/tests/vine_streaming_writer_tests.rs +++ b/vine-core/tests/vine_streaming_writer_tests.rs @@ -2,6 +2,8 @@ use vine_core::vine_streaming_writer::VineStreamingWriter; use vine_core::metadata::{Metadata, MetadataField}; use vine_core::writer_config::WriterConfig; use vine_core::storage_reader::read_vine_data; +use vine_core::vortex_exp::build_struct_array; +use vine_core::arrow_bridge::vortex_to_arrow; use tempfile::tempdir; use std::fs; @@ -25,17 +27,20 @@ fn create_test_metadata() -> Metadata { ) } +/// Helper: build VortexArrayRef from comma-separated rows +fn build_test_array(metadata: &Metadata, rows: &[&str]) -> vortex::ArrayRef { + build_struct_array(metadata, rows).expect("Failed to build test array") +} + #[test] fn test_vine_streaming_writer_new() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Create metadata let metadata = create_test_metadata(); let meta_path = base_path.join("vine_meta.json"); metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - // Create writer let writer = VineStreamingWriter::new(base_path); assert!(writer.is_ok()); } @@ -45,7 +50,6 @@ fn test_vine_streaming_writer_new_missing_metadata() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Don't create metadata file let result = VineStreamingWriter::new(base_path); assert!(result.is_err()); } @@ -55,12 +59,10 @@ fn test_vine_streaming_writer_with_config() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Create metadata let metadata = create_test_metadata(); let meta_path = base_path.join("vine_meta.json"); metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - // Create writer with custom config let config = WriterConfig::with_max_rows(50_000); let writer = VineStreamingWriter::with_config(base_path, config); assert!(writer.is_ok()); @@ -71,15 +73,13 @@ fn test_vine_streaming_writer_append_batch() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Create metadata let metadata = create_test_metadata(); let meta_path = base_path.join("vine_meta.json"); metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - // Create writer and append batch let mut writer = VineStreamingWriter::new(base_path).expect("Failed to create writer"); - let rows = vec!["1,Alice", "2,Bob"]; - let result = writer.append_batch(&rows); + let array = build_test_array(&metadata, &["1,Alice", "2,Bob"]); + let result = writer.append_batch(&array); assert!(result.is_ok()); } @@ -88,22 +88,20 @@ fn test_vine_streaming_writer_append_multiple_batches() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Create metadata let metadata = create_test_metadata(); let meta_path = base_path.join("vine_meta.json"); metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - // Create writer and append multiple batches let mut writer = VineStreamingWriter::new(base_path).expect("Failed to create writer"); - let rows1 = vec!["1,Alice"]; - writer.append_batch(&rows1).expect("Failed to append first batch"); + let array1 = build_test_array(&metadata, &["1,Alice"]); + writer.append_batch(&array1).expect("Failed to append first batch"); - let rows2 = vec!["2,Bob"]; - writer.append_batch(&rows2).expect("Failed to append second batch"); + let array2 = build_test_array(&metadata, &["2,Bob"]); + writer.append_batch(&array2).expect("Failed to append second batch"); - let rows3 = vec!["3,Charlie"]; - writer.append_batch(&rows3).expect("Failed to append third batch"); + let array3 = build_test_array(&metadata, &["3,Charlie"]); + writer.append_batch(&array3).expect("Failed to append third batch"); } #[test] @@ -111,15 +109,13 @@ fn test_vine_streaming_writer_flush() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Create metadata let metadata = create_test_metadata(); let meta_path = base_path.join("vine_meta.json"); metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - // Create writer, append batch, and flush let mut writer = VineStreamingWriter::new(base_path).expect("Failed to create writer"); - let rows = vec!["1,Alice", "2,Bob"]; - writer.append_batch(&rows).expect("Failed to append batch"); + let array = build_test_array(&metadata, &["1,Alice", "2,Bob"]); + writer.append_batch(&array).expect("Failed to append batch"); let result = writer.flush(); assert!(result.is_ok()); @@ -130,15 +126,13 @@ fn test_vine_streaming_writer_close() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Create metadata let metadata = create_test_metadata(); let meta_path = base_path.join("vine_meta.json"); metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - // Create writer, append batch, and close let mut writer = VineStreamingWriter::new(base_path).expect("Failed to create writer"); - let rows = vec!["1,Alice", "2,Bob"]; - writer.append_batch(&rows).expect("Failed to append batch"); + let array = build_test_array(&metadata, &["1,Alice", "2,Bob"]); + writer.append_batch(&array).expect("Failed to append batch"); let result = writer.close(); assert!(result.is_ok()); @@ -149,23 +143,27 @@ fn test_vine_streaming_writer_write_and_read_roundtrip() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Create metadata let metadata = create_test_metadata(); let meta_path = base_path.join("vine_meta.json"); metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - // Write data using streaming writer let mut writer = VineStreamingWriter::new(base_path).expect("Failed to create writer"); - let rows = vec!["1,Alice", "2,Bob", "3,Charlie"]; - writer.append_batch(&rows).expect("Failed to append batch"); + let array = build_test_array(&metadata, &["1,Alice", "2,Bob", "3,Charlie"]); + writer.append_batch(&array).expect("Failed to append batch"); writer.close().expect("Failed to close writer"); - // Read data back + // Read data back as VortexArrayRef let result = read_vine_data(base_path.to_str().unwrap()); - assert_eq!(result.len(), 3); - assert_eq!(result[0], "1,Alice"); - assert_eq!(result[1], "2,Bob"); - assert_eq!(result[2], "3,Charlie"); + assert!(!result.is_empty()); + + // Convert to Arrow and verify contents + let mut total_rows = 0; + for arr in &result { + let batch = vortex_to_arrow(arr, true).expect("Failed to convert to Arrow"); + total_rows += batch.num_rows(); + assert_eq!(batch.num_columns(), 2); + } + assert_eq!(total_rows, 3); } #[test] @@ -173,30 +171,29 @@ fn test_vine_streaming_writer_flush_multiple_times() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Create metadata let metadata = create_test_metadata(); let meta_path = base_path.join("vine_meta.json"); metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - // Create writer and test multiple flushes let mut writer = VineStreamingWriter::new(base_path).expect("Failed to create writer"); - // First batch and flush - let rows1 = vec!["1,Alice"]; - writer.append_batch(&rows1).expect("Failed to append first batch"); + let array1 = build_test_array(&metadata, &["1,Alice"]); + writer.append_batch(&array1).expect("Failed to append first batch"); writer.flush().expect("Failed to flush first time"); - // Second batch and flush - let rows2 = vec!["2,Bob"]; - writer.append_batch(&rows2).expect("Failed to append second batch"); + let array2 = build_test_array(&metadata, &["2,Bob"]); + writer.append_batch(&array2).expect("Failed to append second batch"); writer.flush().expect("Failed to flush second time"); - // Close writer writer.close().expect("Failed to close writer"); - // Verify all data was written let result = read_vine_data(base_path.to_str().unwrap()); - assert_eq!(result.len(), 2); + let mut total_rows = 0; + for arr in &result { + let batch = vortex_to_arrow(arr, true).expect("Failed to convert"); + total_rows += batch.num_rows(); + } + assert_eq!(total_rows, 2); } #[test] @@ -204,18 +201,15 @@ fn test_vine_streaming_writer_creates_date_partition() { let temp_dir = tempdir().expect("Failed to create temp dir"); let base_path = temp_dir.path(); - // Create metadata let metadata = create_test_metadata(); let meta_path = base_path.join("vine_meta.json"); metadata.save(meta_path.to_str().unwrap()).expect("Failed to save metadata"); - // Write data let mut writer = VineStreamingWriter::new(base_path).expect("Failed to create writer"); - let rows = vec!["1,Alice"]; - writer.append_batch(&rows).expect("Failed to append batch"); + let array = build_test_array(&metadata, &["1,Alice"]); + writer.append_batch(&array).expect("Failed to append batch"); writer.close().expect("Failed to close writer"); - // Verify date partition directory was created let date_dirs: Vec<_> = fs::read_dir(base_path) .expect("Failed to read dir") .filter_map(|e| e.ok()) @@ -224,9 +218,8 @@ fn test_vine_streaming_writer_creates_date_partition() { assert!(!date_dirs.is_empty()); - // Verify directory name is a valid date (YYYY-MM-DD format) let dir_name = date_dirs[0].file_name(); let dir_name_str = dir_name.to_str().unwrap(); assert!(dir_name_str.contains('-')); - assert_eq!(dir_name_str.len(), 10); // YYYY-MM-DD is 10 characters + assert_eq!(dir_name_str.len(), 10); } diff --git a/vine-trino/src/test/java/io/kination/vine/VineJniIntegrationTest.java b/vine-trino/src/test/java/io/kination/vine/VineJniIntegrationTest.java new file mode 100644 index 0000000..ad02705 --- /dev/null +++ b/vine-trino/src/test/java/io/kination/vine/VineJniIntegrationTest.java @@ -0,0 +1,48 @@ +package io.kination.vine; + +import org.junit.jupiter.api.Test; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.List; +import static org.junit.jupiter.api.Assertions.*; + +class VineJniIntegrationTest { + + @Test + void testReadDataArrow() { + try { + // Path to the test table generated by generate-test-data + String tablePath = Paths.get("src/test/resources/test_table").toAbsolutePath().toString(); + + // 1. Read Arrow IPC bytes via JNI (Using Compatibility Mode for Java 14) + byte[] arrowData = VineModule.readDataArrow(tablePath, VineModule.COMPAT_MODE_JAVA14); + + assertNotNull(arrowData, "JNI readDataArrow returned null"); + assertTrue(arrowData.length > 0, "JNI readDataArrow returned empty array"); + + // 2. Define columns to extract (matching generate-test-data) + List columns = List.of( + new VineColumnHandle("id", io.trino.spi.type.IntegerType.INTEGER, 0), + new VineColumnHandle("name", io.trino.spi.type.VarcharType.VARCHAR, 1), + new VineColumnHandle("score", io.trino.spi.type.DoubleType.DOUBLE, 2), + new VineColumnHandle("active", io.trino.spi.type.BooleanType.BOOLEAN, 3) + ); + + // 3. Convert Arrow to Rows + Object[][] rows = VineArrowConverter.arrowToRows(arrowData, columns); + + assertEquals(3, rows.length, "Expected 3 rows from test data"); + + // Row 1: 1, "Alice", 95.5, true + assertEquals(1L, rows[0][0]); + assertEquals("Alice", rows[0][1]); + assertEquals(95.5, rows[0][2]); + assertEquals(true, rows[0][3]); + + System.out.println("Integration test passed: successfully read 3 rows via JNI"); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } +} diff --git a/vine-trino/src/test/java/io/kination/vine/VineMetadataReaderTest.java b/vine-trino/src/test/java/io/kination/vine/VineMetadataReaderTest.java new file mode 100644 index 0000000..9ae46a2 --- /dev/null +++ b/vine-trino/src/test/java/io/kination/vine/VineMetadataReaderTest.java @@ -0,0 +1,79 @@ +package io.kination.vine; + +import org.junit.jupiter.api.Test; + +import java.net.URL; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +class VineMetadataReaderTest { + + @Test + void testReadMetadata() { + String testTablePath = getTestResourcePath("test_table"); + + VineMetadata metadata = VineMetadataReader.read(testTablePath); + + assertEquals("test_table", metadata.getTableName()); + assertNotNull(metadata.getFields()); + assertEquals(4, metadata.getFields().size()); + } + + @Test + void testFieldParsing() { + String testTablePath = getTestResourcePath("test_table"); + + VineMetadata metadata = VineMetadataReader.read(testTablePath); + List fields = metadata.getFields(); + + // Field 1: id + assertEquals(1, fields.get(0).getId()); + assertEquals("id", fields.get(0).getName()); + assertEquals("integer", fields.get(0).getDataType()); + assertTrue(fields.get(0).isRequired()); + + // Field 2: name + assertEquals(2, fields.get(1).getId()); + assertEquals("name", fields.get(1).getName()); + assertEquals("string", fields.get(1).getDataType()); + assertFalse(fields.get(1).isRequired()); + + // Field 3: score + assertEquals(3, fields.get(2).getId()); + assertEquals("score", fields.get(2).getName()); + assertEquals("double", fields.get(2).getDataType()); + assertFalse(fields.get(2).isRequired()); + + // Field 4: active + assertEquals(4, fields.get(3).getId()); + assertEquals("active", fields.get(3).getName()); + assertEquals("boolean", fields.get(3).getDataType()); + assertFalse(fields.get(3).isRequired()); + } + + @Test + void testHasMetadata() { + String testTablePath = getTestResourcePath("test_table"); + assertTrue(VineMetadataReader.hasMetadata(testTablePath)); + } + + @Test + void testHasMetadataMissing() { + assertFalse(VineMetadataReader.hasMetadata("/nonexistent/path")); + } + + @Test + void testReadNonExistentPath() { + assertThrows(IllegalArgumentException.class, + () -> VineMetadataReader.read("/nonexistent/path")); + } + + private String getTestResourcePath(String resourceName) { + URL url = getClass().getClassLoader().getResource(resourceName); + if (url == null) { + throw new RuntimeException("Test resource not found: " + resourceName); + } + return url.getPath(); + } +} diff --git a/vine-trino/src/test/java/io/kination/vine/VineTypeMappingTest.java b/vine-trino/src/test/java/io/kination/vine/VineTypeMappingTest.java new file mode 100644 index 0000000..ec6574e --- /dev/null +++ b/vine-trino/src/test/java/io/kination/vine/VineTypeMappingTest.java @@ -0,0 +1,72 @@ +package io.kination.vine; + +import io.trino.spi.type.BigintType; +import io.trino.spi.type.BooleanType; +import io.trino.spi.type.DateType; +import io.trino.spi.type.DoubleType; +import io.trino.spi.type.IntegerType; +import io.trino.spi.type.RealType; +import io.trino.spi.type.SmallintType; +import io.trino.spi.type.TimestampType; +import io.trino.spi.type.TinyintType; +import io.trino.spi.type.VarbinaryType; +import io.trino.spi.type.VarcharType; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class VineTypeMappingTest { + + @Test + void testIntegerTypes() { + assertEquals(TinyintType.TINYINT, VineTypeMapping.toTrinoType("byte")); + assertEquals(TinyintType.TINYINT, VineTypeMapping.toTrinoType("tinyint")); + assertEquals(SmallintType.SMALLINT, VineTypeMapping.toTrinoType("short")); + assertEquals(SmallintType.SMALLINT, VineTypeMapping.toTrinoType("smallint")); + assertEquals(IntegerType.INTEGER, VineTypeMapping.toTrinoType("integer")); + assertEquals(IntegerType.INTEGER, VineTypeMapping.toTrinoType("int")); + assertEquals(BigintType.BIGINT, VineTypeMapping.toTrinoType("long")); + assertEquals(BigintType.BIGINT, VineTypeMapping.toTrinoType("bigint")); + } + + @Test + void testFloatingPointTypes() { + assertEquals(RealType.REAL, VineTypeMapping.toTrinoType("float")); + assertEquals(DoubleType.DOUBLE, VineTypeMapping.toTrinoType("double")); + } + + @Test + void testBooleanType() { + assertEquals(BooleanType.BOOLEAN, VineTypeMapping.toTrinoType("boolean")); + assertEquals(BooleanType.BOOLEAN, VineTypeMapping.toTrinoType("bool")); + } + + @Test + void testStringAndBinaryTypes() { + assertEquals(VarcharType.VARCHAR, VineTypeMapping.toTrinoType("string")); + assertEquals(VarbinaryType.VARBINARY, VineTypeMapping.toTrinoType("binary")); + } + + @Test + void testDateTimeTypes() { + assertEquals(DateType.DATE, VineTypeMapping.toTrinoType("date")); + assertEquals(TimestampType.TIMESTAMP_MILLIS, VineTypeMapping.toTrinoType("timestamp")); + } + + @Test + void testDecimalFallback() { + assertEquals(VarcharType.VARCHAR, VineTypeMapping.toTrinoType("decimal")); + } + + @Test + void testUnknownTypeFallback() { + assertEquals(VarcharType.VARCHAR, VineTypeMapping.toTrinoType("unknown_type")); + } + + @Test + void testCaseInsensitive() { + assertEquals(IntegerType.INTEGER, VineTypeMapping.toTrinoType("INTEGER")); + assertEquals(BooleanType.BOOLEAN, VineTypeMapping.toTrinoType("Boolean")); + assertEquals(VarcharType.VARCHAR, VineTypeMapping.toTrinoType("STRING")); + } +}