Skip to content
Snippets Groups Projects
  1. Mar 20, 2025
  2. Mar 19, 2025
  3. Mar 07, 2025
  4. Feb 07, 2025
    • Heiko Reese's avatar
      feat: search fingerprints and ignore 0-prefixes in search terms · 747a8ec8
      Heiko Reese authored
      # Conflicts:
      #	webroot/index.html
      
      diff --git a/Makefile b/Makefile
      index 8c91b7e..1ebc279 100644
      --- a/Makefile
      +++ b/Makefile
      @@ -3,7 +3,7 @@
       .DEFAULT_GOAL := default
      
       getdeps:
      -	go get git.scc.kit.edu/KIT-CA/websearch/ca-websearch...
      +	go get gitlab.kit.edu/kit/kit-ca/lib/certificatestats/ca-websearch...
      
       default: *.go ca-websearch/*.go
      -	go install -v  -ldflags="-s -w" git.scc.kit.edu/KIT-CA/websearch/ca-websearch
      +	go install -v  -ldflags="-s -w" gitlab.kit.edu/kit/kit-ca/lib/certificatestats/ca-websearch
      diff --git a/debian/control b/debian/control
      index d3b76bb..1eb30e4 100644
      --- a/debian/control
      +++ b/debian/control
      @@ -22,9 +22,9 @@ Build-Depends: debhelper-compat (= 13),
                      golang-github-hreese-go-humanreltime-dev,
                      golang-git.scc.kit.edu-heiko.reese-certificatestats-dev
       Standards-Version: 4.5.0
      -Homepage: https://git.scc.kit.edu/KIT-CA/websearch
      +Homepage: https://gitlab.kit.edu/kit/kit-ca/lib/certificatestats
       Rules-Requires-Root: no
      -XS-Go-Import-Path: git.scc.kit.edu/KIT-CA/websearch
      +XS-Go-Import-Path: gitlab.kit.edu/kit/kit-ca/lib/certificatestats
      
       Package: kit-ca-websearch
       Architecture: any
      diff --git a/debian/kit-ca-websearch-static.install b/debian/kit-ca-websearch-static.install
      index b5ecf8b..3ffa454 100644
      --- a/debian/kit-ca-websearch-static.install
      +++ b/debian/kit-ca-websearch-static.install
      @@ -1 +1 @@
      -usr/share/gocode/src/git.scc.kit.edu/KIT-CA/websearch/webroot/ usr/share/KIT-CA/search
      +usr/share/gocode/src/gitlab.kit.edu/kit/kit-ca/lib/certificatestats/webroot/ usr/share/KIT-CA/search
      diff --git a/debian/rules b/debian/rules
      index 6d08c91..051ce31 100755
      --- a/debian/rules
      +++ b/debian/rules
      @@ -3,7 +3,7 @@
       #DH_VERBOSE = 1
       #export DH_OPTIONS=-v
      
      -export DH_GOLANG_BUILDPKG := git.scc.kit.edu/KIT-CA/websearch/ca-websearch
      +export DH_GOLANG_BUILDPKG := gitlab.kit.edu/kit/kit-ca/lib/certificatestats/ca-websearch
      
       export DH_GOLANG_INSTALL_EXTRA := webroot
      
      diff --git a/go.mod b/go.mod
      index 86bb3f0..c7085f8 100644
      --- a/go.mod
      +++ b/go.mod
      @@ -15,15 +15,11 @@ require (
       	golang.org/x/text v0.13.0
       )
      
      -require (
      -	git.scc.kit.edu/KIT-CA/websearch v0.0.0-20231011192005-88398b4c
      -	git.scc.kit.edu/heiko.reese/CertificateStats v0.0.0-20240104024726-431b94dc953a
      -)
      -
       require (
       	github.com/gorilla/securecookie v1.1.1 // indirect
       	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
       	github.com/hashicorp/golang-lru v1.0.2 // indirect
       	github.com/mattn/go-isatty v0.0.20 // indirect
      +	gitlab.kit.edu/kit/kit-ca/lib/certificatestats v0.0.0-20250207165819-6c3517b36c80 // indirect
       	golang.org/x/sys v0.16.0 // indirect
       )
      diff --git a/go.sum b/go.sum
      index 25c89c7..e754831 100644
      --- a/go.sum
      +++ b/go.sum
      @@ -1,7 +1,3 @@
      -git.scc.kit.edu/KIT-CA/websearch v0.0.0-20231011192005-88398b4c h1:hGl/e1gWmY+vKoNAeLK+5VMwcj7HHUaDFj5OEiWvaUg=
      -git.scc.kit.edu/KIT-CA/websearch v0.0.0-20231011192005-88398b4c/go.mod h1:/9a19NicVTGlMnNoGUSB3LbcFW2zbEuqItQUzUVd4E0=
      -git.scc.kit.edu/heiko.reese/CertificateStats v0.0.0-20240104024726-431b94dc953a h1:Gr52dAxQIEBKw2vKRH8YekzCcq7gx29r9haHR+IdQZI=
      -git.scc.kit.edu/heiko.reese/CertificateStats v0.0.0-20240104024726-431b94dc953a/go.mod h1:IJS5YS382p82clOBzOsCH07gug4PWSEGU8z0i3/NcAI=
       github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
       github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
       github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
      @@ -35,6 +31,10 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
       github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
       github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
       github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
      +gitlab.kit.edu/kit/kit-ca/lib/certificatestats v0.0.0-20231011192005-88398b4c h1:hGl/e1gWmY+vKoNAeLK+5VMwcj7HHUaDFj5OEiWvaUg=
      +gitlab.kit.edu/kit/kit-ca/lib/certificatestats v0.0.0-20231011192005-88398b4c/go.mod h1:/9a19NicVTGlMnNoGUSB3LbcFW2zbEuqItQUzUVd4E0=
      +gitlab.kit.edu/kit/kit-ca/lib/certificatestats v0.0.0-20250207165819-6c3517b36c80 h1:DB8oM+0GoEQ7S6a8syyk0y0wpgX2svXG41IexWdWsHw=
      +gitlab.kit.edu/kit/kit-ca/lib/certificatestats v0.0.0-20250207165819-6c3517b36c80/go.mod h1:M0d24E/8e9FI1qEpPvWQhGr4KiqXS3V6dZ0iYRm+5II=
       golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
       golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
       golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
      diff --git a/searchablecert.go b/searchablecert.go
      index eb2adfd..49c8c08 100644
      --- a/searchablecert.go
      +++ b/searchablecert.go
      @@ -2,12 +2,16 @@ package websearch
      
       import (
       	"bytes"
      +	"crypto/md5"
       	"crypto/rsa"
      +	"crypto/sha1"
      +	"crypto/sha256"
       	"crypto/x509"
       	"crypto/x509/pkix"
       	"encoding/pem"
      -	"git.scc.kit.edu/heiko.reese/CertificateStats"
      +	"fmt"
       	"github.com/hreese/go-humanreltime"
      +	"gitlab.kit.edu/kit/kit-ca/lib/certificatestats"
       	"golang.org/x/text/runes"
       	"golang.org/x/text/transform"
       	"golang.org/x/text/unicode/norm"
      @@ -35,6 +39,7 @@ var (
       		"ul":  true,
       	}
       	publicQuerySplitter = regexp.MustCompile("[[:space:],]+")
      +	ZeroPrefix          = regexp.MustCompile("^(?:0x)?0*([0-9a-fA-F]+)$")
       )
      
       const (
      @@ -187,6 +192,9 @@ type SearchableCert struct {
       	searchablestring   string
       	pubsearchToken     []string
       	rawCertificate     *x509.Certificate
      +	FingerprintSHA1    string
      +	FingerprintSHA256  string
      +	FingerprintMD5     string
       }
      
       func (c *SearchableCert) GetPEM() []byte {
      @@ -213,12 +221,12 @@ func CleanupQueryString(query string) string {
       	query = strings.ToLower(query)
       	// remove leading and trailing white space
       	query = strings.TrimSpace(query)
      -	// DNF-Rules
      -	query = strings.Replace(query, "ä", "ae", -1)
      -	query = strings.Replace(query, "ö", "oe", -1)
      -	query = strings.Replace(query, "ü", "ue", -1)
      -	query = strings.Replace(query, "ß", "ss", -1)
      -	// remove diacrits and thelike
      +	// DFN-Rules (removed for Sectigo and HARICA/TBD)
      +	//query = strings.Replace(query, "ä", "ae", -1)
      +	//query = strings.Replace(query, "ö", "oe", -1)
      +	//query = strings.Replace(query, "ü", "ue", -1)
      +	//query = strings.Replace(query, "ß", "ss", -1)
      +	// remove diacritics and the like
       	unicodeSimplification := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC)
       	result, _, _ := transform.String(unicodeSimplification, query)
       	return result
      @@ -232,6 +240,17 @@ func MakeInternalSearchFilter(query string) SCFilter {
       	cleaned := CleanupQueryString(query)
       	token := tokenizeQuery(cleaned)
      
      +	// remove colons and leading zeros from strings that look like hex numbers
      +	for idx, t := range token {
      +		if strings.Count(t, ":") > 1 {
      +			t = strings.Replace(t, ":", "", -1)
      +			m := ZeroPrefix.FindStringSubmatch(t)
      +			if len(m) > 1 && len(m[1]) > 0 {
      +				token[idx] = m[1]
      +			}
      +		}
      +	}
      +
       	// return zero results on empty query
       	if len(token) == 0 {
       		return NeverMatch
      @@ -440,6 +459,11 @@ func CertToSearchable(c *x509.Certificate) SearchableCert {
       		cert.CAGeneration = &unknown
       	}
      
      +	// add serials
      +	cert.FingerprintSHA1 = fmt.Sprintf("0x%x", sha1.Sum(c.Raw))
      +	cert.FingerprintSHA256 = fmt.Sprintf("0x%x", sha256.Sum256(c.Raw))
      +	cert.FingerprintMD5 = fmt.Sprintf("0x%x", md5.Sum(c.Raw))
      +
       	// build string used for simple search
       	var buffer bytes.Buffer
       	buffer.WriteString(cert.Serial)
      @@ -452,6 +476,13 @@ func CertToSearchable(c *x509.Certificate) SearchableCert {
       	buffer.WriteString(strings.Join(cert.DNSNames, " "))
       	buffer.WriteString(strings.Join(cert.EmailAddresses, " "))
       	buffer.WriteString(strings.Join(cert.IPAddresses, " "))
      +	buffer.WriteString(" ")
      +	buffer.WriteString(cert.FingerprintSHA256)
      +	buffer.WriteString(" ")
      +	buffer.WriteString(cert.FingerprintSHA1)
      +	buffer.WriteString(" ")
      +	buffer.WriteString(cert.FingerprintMD5)
      +
       	cert.searchablestring = strings.ToLower(buffer.String())
      
       	// build array for public search
      @@ -530,6 +561,9 @@ type JSONResult struct {
       	Expired            bool     `json:"expired"`
       	Validity           string   `json:"valid"`
       	Public             string   `json:"public"`
      +	FingerprintSHA1    string   `json:"fingerprintsha1"`
      +	FingerprintSHA256  string   `json:"fingerprintsha256"`
      +	FingerprintMD5     string   `json:"fingerprintmd5"`
       }
      
       func (c *SearchableCert) JSONResult(watchers map[int]*AttributeState) *JSONResult {
      @@ -557,6 +591,9 @@ func (c *SearchableCert) JSONResult(watchers map[int]*AttributeState) *JSONResul
       		Expired:            c.NotAfter.Before(time.Now()),
       		Validity:           ValidityToName(watchers[WatchValid].Get(c.Serial)),
       		Public:             VisibilityToName(watchers[WatchVisibile].Get(c.Serial)),
      +		FingerprintMD5:     c.FingerprintMD5,
      +		FingerprintSHA1:    c.FingerprintSHA1,
      +		FingerprintSHA256:  c.FingerprintSHA256,
       	}
      
       	return r
      diff --git a/testing/Format/main.go b/testing/Format/main.go
      index 650fed0..d8e64be 100644
      --- a/testing/Format/main.go
      +++ b/testing/Format/main.go
      @@ -2,8 +2,8 @@ package main
      
       import (
       	"flag"
      -	. "git.scc.kit.edu/KIT-CA/websearch"
       	"github.com/k0kubun/pp"
      +	. "gitlab.kit.edu/kit/kit-ca/lib/certificatestats"
       )
      
       func main() {
      diff --git a/testing/boltdb/main.go b/testing/boltdb/main.go
      index 60fbf67..3030de3 100644
      --- a/testing/boltdb/main.go
      +++ b/testing/boltdb/main.go
      @@ -5,10 +5,10 @@ import (
       	"encoding/gob"
       	"flag"
       	"fmt"
      -	. "git.scc.kit.edu/heiko.reese/CertificateStats"
       	_ "github.com/k0kubun/pp"
      +	. "gitlab.kit.edu/kit/kit-ca/lib/certificatestats"
       	"log"
      -	//. "git.scc.kit.edu/KIT-CA/websearch"
      +	//. "gitlab.kit.edu/kit/kit-ca/lib/certificatestats"
       	"github.com/boltdb/bolt"
       )
      
      diff --git a/testing/getoids/getoids.go b/testing/getoids/getoids.go
      index e36d729..560dbf3 100644
      --- a/testing/getoids/getoids.go
      +++ b/testing/getoids/getoids.go
      @@ -4,8 +4,8 @@ import (
       	_ "crypto/x509"
       	"crypto/x509/pkix"
       	"flag"
      -	. "git.scc.kit.edu/heiko.reese/CertificateStats"
       	"github.com/k0kubun/pp"
      +	. "gitlab.kit.edu/kit/kit-ca/lib/certificatestats"
       )
      
       func main() {
      diff --git a/testing/gomemdb/main.go b/testing/gomemdb/main.go
      index 5075c26..e228718 100644
      --- a/testing/gomemdb/main.go
      +++ b/testing/gomemdb/main.go
      @@ -5,12 +5,11 @@ import (
       	_ "encoding/gob"
       	"flag"
       	_ "fmt"
      -	. "git.scc.kit.edu/heiko.reese/CertificateStats"
      +	"github.com/hashicorp/go-memdb"
       	_ "github.com/k0kubun/pp"
      +	. "gitlab.kit.edu/kit/kit-ca/lib/certificatestats"
      +	"gitlab.kit.edu/kit/kit-ca/websearch"
       	_ "log"
      -	//. "git.scc.kit.edu/KIT-CA/websearch"
      -	"git.scc.kit.edu/KIT-CA/websearch"
      -	"github.com/hashicorp/go-memdb"
       )
      
       var (
      diff --git a/webroot/index.html b/webroot/index.html
      index d68d7b7..9436d52 100644
      --- a/webroot/index.html
      +++ b/webroot/index.html
      @@ -203,6 +203,9 @@
                                   <div>
                                       <small><em>Serial&nbsp;(hex)</em>:&nbsp;{{entry.hexserial.replace("0x", "").toLocaleUpperCase()}}</small>
                                   </div>
      +                            <div>
      +                                <small><em>Fingerprint</em>:&nbsp;{{entry.fingerprintsha1.replace("0x", "").match(/.{1,2}/g).join(":").toLocaleUpperCase()}}</small>
      +                            </div>
                                   <div v-if="entry.cageneration"><small><em>CA</em>:&nbsp;{{entry.cageneration}}</small></div>
                               </td>
                               <td>
      747a8ec8
  5. Mar 18, 2024
  6. Feb 05, 2024
  7. Feb 02, 2024
  8. Oct 11, 2023
  9. Sep 28, 2023
  10. Sep 22, 2023
  11. Dec 06, 2022
  12. Jan 26, 2022
  13. Jan 03, 2022
  14. Dec 30, 2021
  15. Dec 22, 2021
Loading