metadata/json_index_test.go at main · gigapi/metadata · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
package metadata

import (
	"fmt"
	"github.com/google/uuid"
	"testing"
	"time"
)

func TestJSONSave(t *testing.T) {
	MergeConfigurations = []MergeConfigurationsConf{
		{10, 10 * 1024 * 1024, 1},
	}
	idx, err := NewJSONIndex(
		"_testdata",
		"default",
		"test",
		layers)
	if err != nil {
		panic(err)
	}
	var ents []*IndexEntry
	now := time.Now()
	threeDaysAgo := now.Add(-3 * 24 * time.Hour)

	for ts := threeDaysAgo; ts.Before(now); ts = ts.Add(15 * time.Second) {
		ents = append(ents, &IndexEntry{
			Database: "default",
			Table:    "test",
			MinTime:  ts.UnixNano(),
			MaxTime:  ts.Add(15 * time.Second).UnixNano(),
			Path: fmt.Sprintf("date=%s/hour=%02d/%s.1.parquet",
				ts.UTC().Format("2006-01-02"),
				ts.UTC().Hour(),
				uuid.New().String()),
			SizeBytes: 1000000,
			ChunkTime: time.Now().UnixNano(),
			Layer:     "l1",
		})
	}

	p := idx.Batch(ents, nil)
	_, err = p.Get()
	if err != nil {
		panic(err)
	}
	fmt.Printf("Items saved: %d\n", len(ents))
}

func TestJSONSaveAndRM(t *testing.T) {
	MergeConfigurations = []MergeConfigurationsConf{
		{10, 10 * 1024 * 1024, 1},
	}
	idx, err := NewJSONIndex(
		"_testdata",
		"default",
		"test",
		layers)
	if err != nil {
		panic(err)
	}
	var ents []*IndexEntry
	now := time.Now()
	threeDaysAgo := now.Add(-3 * 24 * time.Hour)

	for ts := threeDaysAgo; ts.Before(now); ts = ts.Add(15 * time.Second) {
		ents = append(ents, &IndexEntry{
			Database: "default",
			Table:    "test",
			MinTime:  ts.UnixNano(),
			MaxTime:  ts.Add(15 * time.Second).UnixNano(),
			Path: fmt.Sprintf("date=%s/hour=%02d/%s.1.parquet",
				ts.UTC().Format("2006-01-02"),
				ts.UTC().Hour(),
				uuid.New().String()),
			SizeBytes: 1000000,
			ChunkTime: time.Now().UnixNano(),
			Layer:     "l1",
		})
	}

	p := idx.Batch(ents, nil)
	_, err = p.Get()
	if err != nil {
		panic(err)
	}
	fmt.Printf("Items saved: %d\n", len(ents))

	p = idx.Batch(nil, ents)
	_, err = p.Get()
	if err != nil {
		panic(err)
	}
}