From 54c2f74d9d60d5e7c869c3f55a857675cd16f67c Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Wed, 27 Aug 2025 13:47:51 +0000 Subject: [PATCH] VIndex for Static CT logs This outputs each DNSName from each cert into the VIndex. In the interests of launching and iterating, this doesn't yet output all levels up to the TLD. Confirmed this works on the Google test log coachandhorses2026h1. This was picked because it's a smaller log (only 161 million entries). Changed general client to support static CT Input Logs verifier keys. It doesn't support static CT enough to dereference the input log pointers though. We'll need a custom client for that. --- go.mod | 16 +- go.sum | 37 ++-- vindex/cmd/client/client.go | 62 +++++-- vindex/cmd/ct/README.md | 54 ++++++ vindex/cmd/ct/main.go | 360 ++++++++++++++++++++++++++++++++++++ vindex/cmd/ct/web.go | 76 ++++++++ 6 files changed, 563 insertions(+), 42 deletions(-) create mode 100644 vindex/cmd/ct/README.md create mode 100644 vindex/cmd/ct/main.go create mode 100644 vindex/cmd/ct/web.go diff --git a/go.mod b/go.mod index 47e864d..996bc52 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/transparency-dev/incubator go 1.25.0 require ( + filippo.io/sunlight v0.8.0 + filippo.io/torchwood v0.9.0 github.com/cockroachdb/pebble v1.1.5 github.com/go-git/go-git/v5 v5.19.1 github.com/google/go-cmp v0.7.0 @@ -10,6 +12,7 @@ require ( github.com/transparency-dev/formats v0.1.0 github.com/transparency-dev/merkle v0.0.2 github.com/transparency-dev/tessera v1.0.3-0.20260318145621-a1e0ccb4adf4 + golang.org/x/crypto v0.50.0 golang.org/x/mod v0.36.0 golang.org/x/sync v0.20.0 k8s.io/klog/v2 v2.140.0 @@ -39,8 +42,8 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect - github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/certificate-transparency-go v1.3.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect @@ -48,13 +51,13 @@ require ( github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.15.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.3.1 // indirect @@ -63,7 +66,6 @@ require ( go.opentelemetry.io/otel v1.42.0 // indirect go.opentelemetry.io/otel/metric v1.42.0 // indirect go.opentelemetry.io/otel/trace v1.42.0 // indirect - golang.org/x/crypto v0.50.0 // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/net v0.53.0 // indirect golang.org/x/sys v0.43.0 // indirect diff --git a/go.sum b/go.sum index 1bfee5a..9c92871 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +filippo.io/sunlight v0.8.0 h1:7ytoUj2KmU5k4ogDSLwEtCoEjjrTZsh+g++UIfTGpM4= +filippo.io/sunlight v0.8.0/go.mod h1:gJ1qFtjHWqj9j4f5M2fnaER6ZFPUkTrRz4/pTamneDg= +filippo.io/torchwood v0.9.0 h1:2W156cI7K3MyxEyNTuS1C9lYEW7y1u7PoHLmvgNsiZc= +filippo.io/torchwood v0.9.0/go.mod h1:zOJguxdmaODUQScAvC80bV6N0SOA9U+bFZG1DwJU6N8= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= @@ -37,8 +41,9 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -66,12 +71,10 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/certificate-transparency-go v1.3.2 h1:9ahSNZF2o7SYMaKaXhAumVEzXB2QaayzII9C8rv7v+A= +github.com/google/certificate-transparency-go v1.3.2/go.mod h1:H5FpMUaGa5Ab2+KCYsxg6sELw3Flkl7pGZzWdBoYLXs= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= @@ -95,8 +98,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= @@ -106,16 +109,17 @@ github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= -github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= @@ -166,7 +170,6 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/vindex/cmd/client/client.go b/vindex/cmd/client/client.go index a0b32ec..8def047 100644 --- a/vindex/cmd/client/client.go +++ b/vindex/cmd/client/client.go @@ -19,25 +19,29 @@ package main import ( "context" + "crypto/x509" + "encoding/base64" "errors" "flag" "fmt" "net/http" "slices" + fnote "github.com/transparency-dev/formats/note" "github.com/transparency-dev/incubator/vindex/client" "golang.org/x/mod/sumdb/note" "k8s.io/klog/v2" ) var ( - vindexBaseURL = flag.String("vindex_base_url", "", "The base URL of the vindex server.") - inLogBaseURL = flag.String("in_log_base_url", "", "The base URL of the input log.") - lookup = flag.String("lookup", "", "The key to look up in the vindex.") - outLogPubKey = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint.") - inLogPubKey = flag.String("in_log_pub_key", "", "The public key to use to verify the input log checkpoint.") - inLogOrigin = flag.String("in_log_origin", "", "Optional: allows the Input Log Origin string to be configured to something other than the public key name.") - minIdx = flag.Uint64("min_idx", 0, "The minimum index to look up in the input log.") + vindexBaseURL = flag.String("vindex_base_url", "", "The base URL of the vindex server.") + inLogBaseURL = flag.String("in_log_base_url", "", "The base URL of the input log.") + lookup = flag.String("lookup", "", "The key to look up in the vindex.") + outLogPubKey = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint. Required.") + inLogPubKey = flag.String("in_log_pub_key", "", "The public key to use to verify the input log checkpoint. Required.") + inLogPubKeyDER = flag.String("in_log_pub_key_der", "", "For CT logs. The public key to use to verify the input log checkpoint. Required, along with in_log_origin.") + inLogOrigin = flag.String("in_log_origin", "", "Required if in_log_pub_key_der is used. Otherwise, allows the Input Log Origin string to be configured to something other than the public key name.") + minIdx = flag.Uint64("min_idx", 0, "The minimum index to look up in the input log.") ) func main() { @@ -104,10 +108,7 @@ func newVIndexClientFromFlags() *client.VIndexClient { if *outLogPubKey == "" { klog.Exitf("out_log_pub_key must be provided") } - inV, err := note.NewVerifier(*inLogPubKey) - if err != nil { - klog.Exitf("failed to construct input log verifier: %v", err) - } + inV := inputLogVerifierFromFlags() outV, err := note.NewVerifier(*outLogPubKey) if err != nil { klog.Exitf("failed to construct output log verifier: %v", err) @@ -123,13 +124,7 @@ func newInputLogClientFromFlags() *client.InputLogClient { if *inLogBaseURL == "" { klog.Exit("in_log_base_url flag must be provided") } - if *inLogPubKey == "" { - klog.Exitf("in_log_pub_key must be provided") - } - v, err := note.NewVerifier(*inLogPubKey) - if err != nil { - klog.Exitf("failed to construct input log verifier: %v", err) - } + v := inputLogVerifierFromFlags() origin := *inLogOrigin if len(origin) == 0 { origin = v.Name() @@ -140,3 +135,34 @@ func newInputLogClientFromFlags() *client.InputLogClient { } return c } + +func inputLogVerifierFromFlags() note.Verifier { + if *inLogPubKey == "" && *inLogPubKeyDER == "" { + klog.Exitf("Must provide exactly one --in_log_pub_key* flag") + } + if *inLogPubKey != "" { + v, err := note.NewVerifier(*inLogPubKey) + if err != nil { + klog.Exitf("failed to construct input log verifier: %v", err) + } + return v + } + derBytes, err := base64.StdEncoding.DecodeString(*inLogPubKeyDER) + if err != nil { + klog.Exitf("Error decoding public key: %s", err) + } + pub, err := x509.ParsePKIXPublicKey(derBytes) + if err != nil { + klog.Exitf("Error parsing public key: %v", err) + } + + verifierKey, err := fnote.RFC6962VerifierString(*inLogOrigin, pub) + if err != nil { + klog.Exitf("Error creating RFC6962 verifier string: %v", err) + } + v, err := fnote.NewVerifier(verifierKey) + if err != nil { + klog.Exitf("Error creating verifier: %v", err) + } + return v +} diff --git a/vindex/cmd/ct/README.md b/vindex/cmd/ct/README.md new file mode 100644 index 0000000..ffddc14 --- /dev/null +++ b/vindex/cmd/ct/README.md @@ -0,0 +1,54 @@ +## Verifiable Index: CT + +This is a demo of pulling the contents of a tile-based CT log into a [Verifiable Index](../../README.md). + +[tlog-tiles]: https://c2sp.org/tlog-tiles +[Tessera]: https://github.com/transparency-dev/tessera + +The CT Input Log is processed, with each entry being indexed on all common names defined in the cert. +This allows the owner of a domain to look up all certs for their domain, in a way that is fully verified. + +> [!NOTE] +> This demo doesn't map all certificates! +> In order to generate a manageable number of key/values, this only indexes +> final certs, and only domain names ending with `.co.uk`. +> https://github.com/transparency-dev/incubator/issues/64 + +## Running + +The static CT Input Log is expected to be available for reading at a URL provided by the `--monitoring_url` flag. +This is the base directory that should contain the checkpoint file. +The Verifiable Index and Output Log are constructed locally, persisted to local disk (in the `--storage_dir` directory), and hosted via a web server. + +```shell +OUTPUT_LOG_PRIVATE_KEY=PRIVATE+KEY+example.com/outputlog+07392c46+ATPJ4crkyUbPeaRffN/4NUof3KV0pQznVIPGOQm3SDEJ \ +go run ./vindex/cmd/ct \ + --storage_dir ~/vindex-ct/ \ + --origin="coachandhorses2026h1.staging.certificate.transparency.goog" \ + --public_key="MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAECHOhXfvYgTcu+Fnl7M7niFj3FgqWlQpXUSWUDw2KAaJXvhGxdJTtmyciN5rWTiDtpeNENVmsUTHFS4XQgeRE0g==" \ + --monitoring_url="https://storage.googleapis.com/coachandhorses2026h1.staging.certificate.transparency.goog" +``` + +Running the above will run a web server hosting the following URLs: + - `/vindex/lookup` - the provisional [vindex lookup API](./api/api.go) + - `/outputlog/` - the [tlog-tiles][] base URL for the output log + +To inspect the log, you can use the woodpecker tool (using the corresponding public key to the private key used above): + +```shell +# To inspect the Output Log +go run github.com/mhutchinson/woodpecker@main --custom_log_type=tiles --custom_log_url=http://localhost:8088/outputlog/ --custom_log_vkey=example.com/outputlog+07392c46+AWyS8y8ZsRmQnTr6Fr2knaa8+t6CPYFh5Ho3wJEr14B8 +``` + +Use left/right cursor to browse, and `q` to quit. + +A domain indexed by the verifiable map can be looked up using the following command: + +```shell +go run ./vindex/cmd/client \ + --vindex_base_url http://localhost:8088/vindex/ \ + --out_log_pub_key=example.com/outputlog+07392c46+AWyS8y8ZsRmQnTr6Fr2knaa8+t6CPYFh5Ho3wJEr14B8 \ + --in_log_pub_key_der=MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAECHOhXfvYgTcu+Fnl7M7niFj3FgqWlQpXUSWUDw2KAaJXvhGxdJTtmyciN5rWTiDtpeNENVmsUTHFS4XQgeRE0g== \ + --in_log_origin=coachandhorses2026h1.staging.certificate.transparency.goog \ + --lookup=google.com +``` diff --git a/vindex/cmd/ct/main.go b/vindex/cmd/ct/main.go new file mode 100644 index 0000000..b331688 --- /dev/null +++ b/vindex/cmd/ct/main.go @@ -0,0 +1,360 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// logandmap is a binary that serves as a demo of how to run a log and a map in the +// same process. +// The log is a Tessera POSIX log, and the map is an in-memory verifiable index. +// A web server is hosted that allows lookups in the map to be performed. +// The log is updated periodically with entries of type LogEntry, and the map keys +// each of the module names from that struct to each of the indices in the log where +// an entry for that module is stored. +package main + +import ( + "context" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "errors" + "flag" + "fmt" + "iter" + "net/http" + "os" + "os/signal" + "path" + "syscall" + "time" + + "filippo.io/sunlight" + "filippo.io/torchwood" + "github.com/gorilla/mux" + "github.com/transparency-dev/formats/log" + fnote "github.com/transparency-dev/formats/note" + "github.com/transparency-dev/incubator/vindex" + "golang.org/x/crypto/cryptobyte" + "golang.org/x/mod/sumdb/note" + "golang.org/x/mod/sumdb/tlog" + "k8s.io/klog/v2" +) + +var ( + inputLogUrl = flag.String("monitoring_url", "", "Base URL of the static CT log to index") + origin = flag.String("origin", "", "Origin of the log to check") + pubKey = flag.String("public_key", "", "The log's public key in base64 encoded DER format") + userAgentInfo = flag.String("user_agent_info", "", "Optional string to append to the user agent (e.g. email address for Sunlight logs)") + persistentCacheDir = flag.String("persistent_cache_dir", "", "Optional location of a directory to cache Input Log tiles") + persistIndex = flag.Bool("persist_index", true, "Set to false to use a memory-based implementation of the verifiable index.") + + outputLogPrivKeyFile = flag.String("output_log_private_key", "", "Location of private key file. If unset, uses the contents of the OUTPUT_LOG_PRIVATE_KEY environment variable.") + storageDir = flag.String("storage_dir", "", "Root directory in which to store the data for the demo. This will create subdirectories for the Input Log, Output Log, and allocate space to store the verifiable map persistence.") + listen = flag.String("listen", ":8088", "Address to set up HTTP server listening on") +) + +const ( + userAgent = "TrustFabric VerifiableIndex" +) + +func main() { + klog.InitFlags(nil) + flag.Parse() + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + if err := run(ctx); err != nil { + klog.Exitf("Run failed: %v", err) + } +} + +func run(ctx context.Context) error { + // Set up storage for the input log, index, and output log. + if *storageDir == "" { + return errors.New("storage_dir must be set") + } + outputLogDir := path.Join(*storageDir, "outputlog") + mapRoot := path.Join(*storageDir, "vindex") + + if err := os.MkdirAll(outputLogDir, 0o755); err != nil { + return fmt.Errorf("failed to create output log directory: %v", err) + } + if err := os.MkdirAll(mapRoot, 0o755); err != nil { + return fmt.Errorf("failed to create vindex directory: %v", err) + } + + outputLog, outputCloser := outputLogOrDie(ctx, outputLogDir) + defer outputCloser() + + inputLog := newStaticCTInputLogFromFlags() + + vi, err := vindex.NewVerifiableIndex(ctx, inputLog, mapFn, outputLog, mapRoot, vindex.Options{PersistIndex: *persistIndex}) + if err != nil { + return fmt.Errorf("failed to create vindex: %v", err) + } + klog.Info("Created verifiable index") + + // Keeps the map synced with the latest published input log state. + go maintainMap(ctx, vi) + + // Run a web server to serve the input log, index, and output log. + go runWebServer(vi, outputLogDir) + <-ctx.Done() + return nil +} + +func cutEntry(tile []byte) (entry []byte, rh tlog.Hash, rest []byte, err error) { + // This implementation is terribly inefficient, parsing the whole entry just + // to re-serialize and throw it away. If this function shows up in profiles, + // let me know and I'll improve it. + e, rest, err := sunlight.ReadTileLeaf(tile) + if err != nil { + return nil, tlog.Hash{}, nil, err + } + rh = tlog.RecordHash(e.MerkleTreeLeaf()) + entry = tile[:len(tile)-len(rest)] + return entry, rh, rest, nil +} + +func newStaticCTInputLogFromFlags() *staticCTInputLog { + ua := userAgent + if *userAgentInfo != "" { + ua = fmt.Sprintf("%s (%s)", userAgent, *userAgentInfo) + } + fetcher, err := torchwood.NewTileFetcher(*inputLogUrl, + torchwood.WithTilePath(sunlight.TilePath), + torchwood.WithUserAgent(ua)) + if err != nil { + klog.Exitf("failed to create client: %v", err) + } + var tileReader torchwood.TileReader = fetcher + if *persistentCacheDir != "" { + tileReader, err = torchwood.NewPermanentCache(fetcher, *persistentCacheDir) + if err != nil { + klog.Exitf("failed to create permanent cache: %v", err) + } + } + client, err := torchwood.NewClient(tileReader, torchwood.WithCutEntry(cutEntry)) + if err != nil { + klog.Exitf("failed to create client: %v", err) + } + return &staticCTInputLog{ + c: client, + f: fetcher, + v: verifierFromFlags(), + } +} + +type staticCTInputLog struct { + c *torchwood.Client + f *torchwood.TileFetcher + v note.Verifier + + lastCheckpoint log.Checkpoint +} + +func (l *staticCTInputLog) Checkpoint(ctx context.Context) (checkpoint []byte, err error) { + return l.f.ReadEndpoint(ctx, "checkpoint") +} + +// Parse unmarshals and verifies a checkpoint obtained from GetCheckpoint. +func (l *staticCTInputLog) Parse(checkpoint []byte) (*log.Checkpoint, error) { + cp, _, _, err := log.ParseCheckpoint(checkpoint, l.v.Name(), l.v) + if err != nil { + return nil, err + } + l.lastCheckpoint = *cp + return cp, err +} + +// Leaves returns all the leaves in the range [start, end), outputting them via +// the returned iterator. +func (l *staticCTInputLog) Leaves(ctx context.Context, start, end uint64) iter.Seq2[[]byte, error] { + tree := tlog.Tree{ + N: int64(end), + Hash: tlog.Hash(l.lastCheckpoint.Hash), + } + return func(yield func([]byte, error) bool) { + for _, entry := range l.c.Entries(ctx, tree, int64(start)) { + e, _, err := sunlight.ReadTileLeaf(entry) + if err != nil { + if !yield(nil, err) { + return + } + } + if !yield(e.MerkleTreeLeaf(), nil) { + return + } + } + if err := l.c.Err(); err != nil { + yield(nil, l.c.Err()) + } + } +} + +// outputLogOrDie returns an output log using a POSIX log in the given directory. +func outputLogOrDie(ctx context.Context, outputLogDir string) (log vindex.OutputLog, closer func()) { + s, v := getOutputLogSignerVerifierOrDie() + + l, c, err := vindex.NewOutputLog(ctx, outputLogDir, s, v, vindex.OutputLogOpts{}) + if err != nil { + klog.Exit(err) + } + return l, c +} + +func verifierFromFlags() note.Verifier { + if *origin == "" { + klog.Exitf("Must provide the --origin flag") + } + if *pubKey == "" { + klog.Exitf("Must provide the --pub_key flag") + } + derBytes, err := base64.StdEncoding.DecodeString(*pubKey) + if err != nil { + klog.Exitf("Error decoding public key: %s", err) + } + pub, err := x509.ParsePKIXPublicKey(derBytes) + if err != nil { + klog.Exitf("Error parsing public key: %v", err) + } + + verifierKey, err := fnote.RFC6962VerifierString(*origin, pub) + if err != nil { + klog.Exitf("Error creating RFC6962 verifier string: %v", err) + } + logSigV, err := fnote.NewVerifier(verifierKey) + if err != nil { + klog.Exitf("Error creating verifier: %v", err) + } + + klog.Infof("Using verifier string: %v", verifierKey) + + return logSigV +} + +// maintainMap reads entries from the log and sync them to the vindex. +func maintainMap(ctx context.Context, vi *vindex.VerifiableIndex) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + for { + if err := vi.Update(ctx); err != nil { + klog.Warning(err) + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func runWebServer(vi *vindex.VerifiableIndex, outLogDir string) { + web := NewServer(vi.Lookup) + + olfs := http.FileServer(http.Dir(outLogDir)) + r := mux.NewRouter() + r.PathPrefix("/outputlog/").Handler(http.StripPrefix("/outputlog/", olfs)) + web.registerHandlers(r) + hServer := &http.Server{ + Addr: *listen, + Handler: r, + } + go func() { + _ = hServer.ListenAndServe() + }() + klog.Infof("Started HTTP server listening on %s", *listen) +} + +// Read output log private key from file or environment variable and generate the +// note Signer and Verifier pair for it. +func getOutputLogSignerVerifierOrDie() (note.Signer, note.Verifier) { + var privKey string + var err error + if len(*outputLogPrivKeyFile) > 0 { + privKey, err = getKeyFile(*outputLogPrivKeyFile) + if err != nil { + klog.Exitf("Unable to get private key: %v", err) + } + } else { + privKey = os.Getenv("OUTPUT_LOG_PRIVATE_KEY") + if len(privKey) == 0 { + klog.Exit("Supply private key file path using --output_log_private_key or set OUTPUT_LOG_PRIVATE_KEY environment variable") + } + } + s, v, err := fnote.NewEd25519SignerVerifier(privKey) + if err != nil { + klog.Exitf("Failed to get signer/verifier: %v", err) + } + return s, v +} + +func getKeyFile(path string) (string, error) { + k, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("failed to read key file: %w", err) + } + return string(k), nil +} + +func mapFn(data []byte) [][sha256.Size]byte { + s := cryptobyte.String(data) + + var version, leafType uint8 + var timestamp uint64 + var certType uint16 + if !s.ReadUint8(&version) || !s.ReadUint8(&leafType) || !s.ReadUint64(×tamp) || !s.ReadUint16(&certType) { + klog.Warningf("Failed to unmarshal headers") + // This should return a sentinel value (e.g. all zero hash) so unprocessable entries can be found + return nil + } + var isPreCert bool + var cert cryptobyte.String + switch certType { + case 0: + // x509 + isPreCert = false + s.ReadUint24LengthPrefixed(&cert) + case 1: + if true { + // Need to support parsing TBS certs + return nil + } + // precert + isPreCert = true + var ikh []byte + s.ReadBytes(&ikh, sha256.Size) + s.ReadUint24LengthPrefixed(&cert) + default: + panic("unknown cert type") + } + + parsedCert, err := x509.ParseCertificate(cert) + if err != nil { + klog.Warningf("failed to parse x509 cert (preCert=%t): %v", isPreCert, err) + // This could return a sentinel value (e.g. all zero hash) so unprocessable entries can be found + return nil + } + if klog.V(2).Enabled() { + klog.V(2).Info(parsedCert.DNSNames) + } + hashes := make([][sha256.Size]byte, 0, len(parsedCert.DNSNames)) + for _, cn := range parsedCert.DNSNames { + // This should output keys for various levels up to the TLD, e.g. + // maps.google.co.uk should have google.co.uk as a secondary key. + h := sha256.Sum256([]byte(cn)) + hashes = append(hashes, h) + } + return hashes +} diff --git a/vindex/cmd/ct/web.go b/vindex/cmd/ct/web.go new file mode 100644 index 0000000..68c8c25 --- /dev/null +++ b/vindex/cmd/ct/web.go @@ -0,0 +1,76 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/sha256" + _ "embed" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + + "github.com/gorilla/mux" + "github.com/transparency-dev/incubator/vindex/api" + "k8s.io/klog/v2" +) + +func NewServer(lookup func(context.Context, [sha256.Size]byte) (api.LookupResponse, error)) Server { + return Server{ + lookup: lookup, + } +} + +type Server struct { + lookup func(context.Context, [sha256.Size]byte) (api.LookupResponse, error) +} + +// handleLookup handles GET requests for looking up map entries. +func (s Server) handleLookup(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + hashStr, ok := vars["hash"] + if !ok { + http.Error(w, "hash parameter not found", http.StatusBadRequest) + return + } + + h, err := hex.DecodeString(hashStr) + if err != nil { + http.Error(w, fmt.Sprintf("invalid hex hash: %v", err), http.StatusBadRequest) + return + } + if l := len(h); l != sha256.Size { + http.Error(w, fmt.Sprintf("hash wrong length (decoded %d bytes)", l), http.StatusBadRequest) + return + } + + klog.V(2).Infof("Received hash from request: '%s'", h) + + resp, err := s.lookup(r.Context(), [sha256.Size]byte(h)) + if err != nil { + http.Error(w, fmt.Sprintf("lookup failed: %v", err), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(resp); err != nil { + klog.Warningf("failed to encode response: %v", err) + } +} + +func (s Server) registerHandlers(r *mux.Router) { + r.HandleFunc("/vindex/lookup/{hash}", s.handleLookup).Methods("GET") +}