E2e testing

.github/workflows/go.yml

@@ -24,8 +24,38 @@ jobs:
     - name: Build
       run: go build -v ./...
 
+    - name: Make coverage dirs
+      run: mkdir -p /tmp/unit_coverage /tmp/e2e_coverage
+
     - name: Test
-      run: go test -race -v ./...
+      run: go test -v -race -cover -coverpkg=./... $(go list ./... | grep -v /e2e/test/) -args -test.gocoverdir=/tmp/unit_coverage
+
+    - name: E2E Test
+      run: go test -v -cover -covermode=atomic -coverpkg=./... $(go list ./e2e/test/...) -args -test.gocoverdir=/tmp/e2e_coverage
+
+    - name: Convert coverage reports
+      run: |
+        go tool covdata textfmt -i /tmp/unit_coverage/ -o unit_coverage.txt
+        go tool covdata textfmt -i /tmp/e2e_coverage/ -o e2e_coverage.txt
+
+
+    - name: Upload coverage reports to Codecov (unittests)
+      uses: codecov/codecov-action@v5
+      continue-on-error: true
+      with:
+        flags: unittests
+        files: ./unit_coverage.txt
+        disable_search: true
+        token: ${{ secrets.CODECOV_TOKEN }}
+
+    - name: Upload coverage reports to Codecov (e2e tests)
+      uses: codecov/codecov-action@v5
+      continue-on-error: true
+      with:
+        flags: e2etests
+        files: ./e2e_coverage.txt
+        disable_search: true
+        token: ${{ secrets.CODECOV_TOKEN }}
 
     - name: Goroutine leak detector
       run: go test -c -o tests && for test in $(go test -list . | grep -E "^(Test|Example)"); do ./tests -test.run "^$test\$" &>/dev/null && echo -e "$test passed\n" || echo -e "$test failed\n"; done

cmd/cmd.go

@@ -36,12 +36,14 @@ Authors:
 }
 
 // Run the root command
-func Run() error {
+func Prepare() *cobra.Command {
 	rootCmd.CompletionOptions.DisableDefaultCmd = true
 
 	// Define flags and configuration settings
 	rootCmd.PersistentFlags().String("log-level", "info", "stdout log level (debug, info, warn, error)")
 	rootCmd.PersistentFlags().String("config-file", "", "config file (default is $HOME/zeno-config.yaml)")
+	rootCmd.PersistentFlags().String("log-socket", "", "socket log address (e.g. /tmp/zenolog.sock)")
+	rootCmd.PersistentFlags().String("log-socket-level", "info", "socket log level")
 	rootCmd.PersistentFlags().Bool("no-stdout-log", false, "disable stdout logging.")
 	rootCmd.PersistentFlags().Bool("no-stderr-log", false, "disable stderr logging.")
 	rootCmd.PersistentFlags().Bool("no-color-logs", false, "switch the terminal (stdout and stderr) logging handler from [slogcolor] handler to the standard [slog] handler (help ensure compatibility with logging collectors)")
@@ -53,5 +55,9 @@ func Run() error {
 	getCmd := getCMDs()
 	rootCmd.AddCommand(getCmd)
 
-	return rootCmd.Execute()
+	return rootCmd
+}
+func Run() error {
+	cmd := Prepare()
+	return cmd.Execute()
 }

e2e/e2e.go

@@ -0,0 +1,71 @@
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/internetarchive/Zeno/cmd"
+	"github.com/internetarchive/Zeno/e2e/log"
+	"github.com/spf13/cobra"
+)
+
+func CmdZenoGetURL(socketPath string, urls []string) *cobra.Command {
+	cmd := cmd.Prepare()
+	args := append([]string{"get", "url", "--config-file", "config.toml", "--log-socket-level", "debug", "--log-socket", socketPath}, urls...)
+	fmt.Println("Command arguments:", args)
+	cmd.SetArgs(args)
+	return cmd
+}
+
+func lazyDial(socketPath string, timeout time.Duration) (net.Conn, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	var conn net.Conn
+	var err error
+
+	for {
+		select {
+		case <-ctx.Done():
+			return nil, fmt.Errorf("timeout while waiting for socket %s", socketPath)
+		default:
+			conn, err = net.Dial("unix", socketPath)
+			if err == nil {
+				return conn, nil
+			}
+			time.Sleep(100 * time.Millisecond) // Retry
+		}
+	}
+}
+
+func ConnectSocketThenCopy(t *testing.T, wg *sync.WaitGroup, W *io.PipeWriter, socketPath string) {
+	defer wg.Done()
+	conn, err := lazyDial(socketPath, 5*time.Second)
+	if err != nil {
+		t.Errorf("failed to connect to log socket: %v", err)
+	}
+	defer conn.Close()
+	io.Copy(W, conn)
+	defer W.Close()
+}
+
+func ExecuteCmdZenoGetURL(t *testing.T, wg *sync.WaitGroup, socketPath string, urls []string) {
+	defer wg.Done()
+	cmdErr := CmdZenoGetURL(socketPath, urls).Execute()
+	if cmdErr != nil {
+		t.Errorf("failed to start command: %v", cmdErr)
+	}
+}
+
+func LogRecordProcessorWrapper(t *testing.T, R *io.PipeReader, rm log.RecordMatcher, wg *sync.WaitGroup) {
+	defer wg.Done()
+	err := log.LogRecordProcessor(R, rm.Match)
+	if err != nil {
+		t.Error("failed to listen to logs:", err)
+	}
+}

e2e/log/log.go

@@ -0,0 +1,50 @@
+package log
+
+import (
+	"fmt"
+	"io"
+	"testing"
+
+	"github.com/go-logfmt/logfmt"
+)
+
+type RecordMatcher interface {
+	Match(record map[string]string)
+	Assert(t *testing.T)
+}
+
+func ParseLog(r io.Reader) chan map[string]string {
+	d := logfmt.NewDecoder(r)
+	out := make(chan map[string]string)
+
+	go func() {
+		defer close(out)
+		for d.ScanRecord() {
+			record := make(map[string]string)
+			for d.ScanKeyval() {
+				if _, ok := record[string(d.Key())]; ok {
+					panic(fmt.Sprintf("duplicate key %s in record", d.Key()))
+				}
+				record[string(d.Key())] = string(d.Value())
+			}
+			if !hasKey(record, "level") || !hasKey(record, "time") {
+				fmt.Printf("ignore record without level or time: %v\n", record)
+				continue
+			}
+			out <- record
+		}
+		if d.Err() != nil {
+			panic(d.Err())
+		}
+	}()
+	return out
+}
+
+func LogRecordProcessor(pipeR *io.PipeReader, matcher func(map[string]string)) error {
+	logCh := ParseLog(pipeR)
+	for record := range logCh {
+		matcher(record)
+		fmt.Printf("log record: %v\n", record)
+	}
+	return nil
+}

e2e/log/log_test.go

@@ -0,0 +1,23 @@
+package log
+
+import (
+	"bytes"
+	_ "embed"
+	"testing"
+)
+
+//go:embed testdata/zeno.log.data
+var zenoLog []byte
+
+func TestParseLog(t *testing.T) {
+	logCh := ParseLog(bytes.NewReader(zenoLog))
+	ok := false
+	for record := range logCh {
+		if record["msg"] == "done, logs are flushing and will be closed" {
+			ok = true
+		}
+	}
+	if !ok {
+		t.Error("expected log message not found")
+	}
+}