certstream

package module
v0.34.5 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 24, 2026 License: MIT Imports: 43 Imported by: 1

README

build coverage goreport Docs

CertStream

Small library wrapping github.com/google/certificate-transparency-go and github.com/FiloSottile/sunlight.

Requires a Postgres database to use.

func grabdata() {
	fulllimiter := bwlimit.NewLimiter()
	fulldialer := fulllimiter.Wrap(nil)
	var filllimiter *bwlimit.Limiter
	var filldialer proxy.ContextDialer

	cfg := certstream.NewConfig()
	cfg.Logger = slog.Default()
	cfg.PgUser = "username"
	cfg.PgPass = "password"
	cfg.PgName = "certstream"
	cfg.PgAddr = "127.0.0.1:5432"
	cfg.HeadDialer = fulldialer
	filllimiter = bwlimit.NewLimiter(int64(1 * 1024 * 1024))
	filldialer = filllimiter.Wrap(fulldialer)
	cfg.TailDialer = filldialer

	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()

	var wg sync.WaitGroup
	cs, err := certstream.Start(ctx, &wg, cfg)
	defer wg.Wait()

	if err != nil {
        panic(err)
	} else {
		for le := range cs.C {
			fmt.Printf("%q %v %v\n", le.Domain, le.Historical, le.Cert().DNSNames)
		}
	}
}

Documentation

Index

Constants

View Source
const SelectBackfillIndex = `SELECT backfill_logindex FROM CERTDB_stream WHERE id = $1;`
View Source
const SelectEstimate = `SELECT reltuples AS estimate FROM pg_class WHERE relname = $1;`
View Source
const SelectFindGap = `SELECT start_gap, end_gap FROM CERTDB_findgap($1, $2, $3);`
View Source
const SelectMaxIndex = `SELECT MAX(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1;`
View Source
const SelectMinIndex = `SELECT MIN(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1;`
View Source
const SelectMinIndexFrom = `SELECT MIN(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1 AND logindex >= $2;`
View Source
const UpdateBackfillIndex = `UPDATE CERTDB_stream SET backfill_logindex = $1 WHERE id = $2 AND backfill_logindex < $1;`

Variables

View Source
var CreateSchema string
View Source
var DbIngestBatchSize = 1000 // number of entries to send to ingest at a time
View Source
var DefaultTransport = &http.Transport{
	TLSHandshakeTimeout:   30 * time.Second,
	ResponseHeaderTimeout: 30 * time.Second,
	MaxIdleConnsPerHost:   2,
	DisableKeepAlives:     false,
	ExpectContinueTimeout: 10 * time.Second,
	ForceAttemptHTTP2:     true,
}
View Source
var ErrHeadLogOpen = errors.New("head log open failed")
View Source
var ErrLogEntriesTooOld = errors.New("log entries are older than max age")
View Source
var ErrLogIdle errLogIdle
View Source
var ErrLogStreamRetryable = errors.New("logstream retryable")
View Source
var ErrSTHDiffTooLow = errors.New("STH diff too low")
View Source
var ErrSunlightClientMissing = errors.New("sunlight client missing")
View Source
var ErrTailLogOpen = errors.New("tail log open failed")
View Source
var FuncIngestBatch string
View Source
var FuncSetSince string
View Source
var FuncSubdomain string
View Source
var FunctionOperatorID string
View Source
var FunctionStreamID string
View Source
var HistoricalBatchSize = 1000 // number of rows to SELECT when getting historical certificates
View Source
var IdleCloseTime = time.Hour * 24 * 7
View Source
var LogBatchSize = int64(1024)
View Source
var MaxErrors = 100

Functions

func GetHTTPCalls added in v0.32.7

func GetHTTPCalls(s string) (n int64)

func GetHTTPCallsMap added in v0.32.7

func GetHTTPCallsMap() (m map[string]int64)

func OperatorDomain

func OperatorDomain(urlString string) string

OperatorDomain returns the TLD+1 given an URL.

func RenderSQL added in v0.26.4

func RenderSQL(query string, args ...any) string

func ScanCertificate added in v0.12.0

func ScanCertificate(row Scanner, cert *PgCertificate) (err error)

func ScanDnsname added in v0.12.0

func ScanDnsname(row Scanner, p *PgDnsname) error

func ScanDnsnamesView added in v0.12.0

func ScanDnsnamesView(row Scanner, dnsname *PgDnsnamesView) (err error)

func ScanIdent added in v0.12.0

func ScanIdent(row Scanner, ident *PgIdent) error

func ScanLogEntry added in v0.12.0

func ScanLogEntry(row Scanner, entry *PgLogEntry) (err error)

Types

type CertStream

type CertStream struct {
	Config                      // copy of config
	C          <-chan *LogEntry // log entry channel
	HeadClient *http.Client     // main HTTP client, uses Config.HeadDialer
	TailClient *http.Client     // may be nil if not backfilling
	LogToggle  atomic.Bool      // if true, log stream activity
	// contains filtered or unexported fields
}

func Start added in v0.12.0

func Start(ctx context.Context, wg *sync.WaitGroup, cfg *Config) (cs *CertStream, err error)

func (*CertStream) CountStreams added in v0.0.3

func (cs *CertStream) CountStreams() (n int)

func (*CertStream) DB added in v0.12.0

func (cs *CertStream) DB() (db *PgDB)

func (*CertStream) GetLogStreamByID added in v0.27.0

func (cs *CertStream) GetLogStreamByID(id int32) (ls *LogStream)

func (*CertStream) LogError added in v0.1.0

func (cs *CertStream) LogError(err error, msg string, args ...any) error

func (*CertStream) LogInfo added in v0.12.0

func (cs *CertStream) LogInfo(msg string, args ...any)

func (*CertStream) Operators added in v0.0.2

func (cs *CertStream) Operators() (operators []*LogOperator)

type Certificate added in v0.10.0

type Certificate struct {
	PreCert   bool
	Seen      time.Time
	Signature []byte
	*x509.Certificate
}

func (*Certificate) GetCommonName added in v0.24.29

func (c *Certificate) GetCommonName() (s string)

type Config added in v0.12.0

type Config struct {
	Logger       Logger              // if not nil Logger to use, no default
	HeadDialer   proxy.ContextDialer // dialer for following the head, defaults to &net.Dialer{}
	HeadLog      string              // log HTTP requests using the head dialer to this file
	TailDialer   proxy.ContextDialer // if not nil, backfill db using this dialer, no default
	PgUser       string              // PostgreSQL user, default "certstream"
	PgPass       string              // PostgreSQL password, default "certstream"
	PgName       string              // PostgreSQL db name, default "certstream"
	PgAddr       string              // PostgreSQL address, no default
	PgPrefix     string              // PostgreSQL naming prefix, default "certdb_"
	PgConns      int                 // max number of database connections, default 100
	PgWorkerBits int                 // number of prefix bits that determine DB workers, default 5 (32 workers)
	PgMaxAge     int                 // maximum age in days to backfill
	PgNoSSL      bool                // if true, do not use SSL
	PgSyncCommit bool                // if true, do not set synchronous_commit=off
	Concurrency  int                 // number of concurrent requests per stream, default is 4
	DataDir      string              // log and cache directory; leave empty to disable
	CacheMaxAge  time.Duration       // remove cached tile data older than this age; zero disables caching of tiles
	TailLog      string              // log HTTP requests using the tail dialer to this file
}

func NewConfig added in v0.12.0

func NewConfig() *Config

NewConfig returns a new default Config

type JsonCertificate added in v0.12.0

type JsonCertificate struct {
	PreCert        bool         `json:",omitempty"`
	Signature      hexEncoded   `json:",omitempty"` // SHA256 signature, searchable on crt.sh
	Issuer         JsonIdentity `json:",omitempty"`
	Subject        JsonIdentity `json:",omitempty"`
	CommonName     string       `json:",omitempty"` // Subject common name
	DNSNames       []string     `json:",omitempty"`
	EmailAddresses []string     `json:",omitempty"`
	IPAddresses    []string     `json:",omitempty"`
	URIs           []string     `json:",omitempty"`
	NotBefore      time.Time    `json:",omitempty"`
	NotAfter       time.Time    `json:",omitempty"`
	Since          time.Time    `json:",omitzero"`
}

func NewJSONCertificate added in v0.12.0

func NewJSONCertificate(cert *Certificate) (jsoncert *JsonCertificate)

func (*JsonCertificate) SetCommonName added in v0.24.29

func (js *JsonCertificate) SetCommonName()

type JsonIdentity added in v0.12.0

type JsonIdentity struct {
	ID           int    `json:",omitempty"`
	Country      string `json:",omitempty"`
	Organization string `json:",omitempty"`
	Province     string `json:",omitempty"`
	CommonName   string `json:",omitempty"`
}

type LogEntry

type LogEntry struct {
	*LogStream
	Err         error // error from RawLogEntryFromLeaf or ToLogEntry, or nil
	LogIndex    int64
	PreCert     bool
	Certificate *x509.Certificate
	Id          int64 // database id, if available
	Historical  bool  // true if the entry is from gap or backfilling
	Signature   []byte
	Seen        time.Time
}

func (*LogEntry) Cert

func (le *LogEntry) Cert() (crt *Certificate)

Cert returns the Certificate given a LogEntry or nil.

func (*LogEntry) Index

func (le *LogEntry) Index() (index int64)

Index returns the log index or -1 if none is available.

func (*LogEntry) String

func (le *LogEntry) String() (s string)

type LogOperator added in v0.0.3

type LogOperator struct {
	*CertStream
	Domain    string       // e.g. "letsencrypt.org" or "googleapis.com"
	Count     atomic.Int64 // atomic; sum of the stream's Count
	Status429 atomic.Int64 // atomic; number of 429 Too Many Requests
	Id        int32        // database ID, if available
	// contains filtered or unexported fields
}

func (*LogOperator) CallCount added in v0.31.6

func (lo *LogOperator) CallCount() (n int64)

func (*LogOperator) Email added in v0.28.0

func (lo *LogOperator) Email() []string

func (*LogOperator) ErrorCount added in v0.22.0

func (lo *LogOperator) ErrorCount() (n int)

func (*LogOperator) Errors added in v0.22.0

func (lo *LogOperator) Errors() (errs []*StreamError)

func (*LogOperator) GetStreamByID added in v0.27.0

func (lo *LogOperator) GetStreamByID(id int32) (ls *LogStream)

func (*LogOperator) Name added in v0.28.0

func (lo *LogOperator) Name() string

func (*LogOperator) StatusCounts added in v0.30.8

func (lo *LogOperator) StatusCounts() (m map[int]int)

func (*LogOperator) StreamCount added in v0.18.0

func (lo *LogOperator) StreamCount() (n int)

func (*LogOperator) Streams added in v0.0.3

func (lo *LogOperator) Streams() (sl []*LogStream)

type LogStream

type LogStream struct {
	*LogOperator
	Count     atomic.Int64 // number of certificates sent to the channel
	MinIndex  atomic.Int64 // atomic: lowest index seen so far, -1 if none seen yet
	MaxIndex  atomic.Int64 // atomic: highest index seen so far, -1 if none seen yet
	LastIndex atomic.Int64 // atomic: highest index that is available from stream source
	Backfill  atomic.Int64 // atomic: number of remaining entries to backfill until we reach head
	Logger    *slog.Logger // toggled by LogToggle
	Id        int32        // database ID, if available
	// contains filtered or unexported fields
}

func (*LogStream) HTTPCalls added in v0.32.7

func (ls *LogStream) HTTPCalls() int64

func (*LogStream) String

func (ls *LogStream) String() string

func (*LogStream) URL added in v0.28.0

func (ls *LogStream) URL() string

type Logger added in v0.1.0

type Logger interface {
	Info(msg string, args ...any)
	Error(msg string, args ...any)
}

type PgCertificate added in v0.12.0

type PgCertificate struct {
	Id         int64
	NotBefore  time.Time
	NotAfter   time.Time
	CommonName string
	SubjectID  int
	IssuerID   int
	Sha256     []byte
	PreCert    bool
	Since      time.Time
}

type PgDB added in v0.12.0

type PgDB struct {
	*CertStream
	*pgxpool.Pool
	Pfx     func(string) string // prefix replacer
	Workers atomic.Int32
	// contains filtered or unexported fields
}

PgDB integrates with sql.DB to manage certificate stream data for a PostgreSQL database

func NewPgDB added in v0.12.0

func NewPgDB(ctx context.Context, cs *CertStream) (cdb *PgDB, err error)

NewPgDB creates a PgDB and creates the needed tables and indices if they don't exist.

func (*PgDB) AverageNewEntryTime added in v0.14.0

func (cdb *PgDB) AverageNewEntryTime() (d time.Duration)

func (*PgDB) Close added in v0.13.0

func (cdb *PgDB) Close()

func (*PgDB) DeleteCertificates added in v0.28.4

func (cdb *PgDB) DeleteCertificates(ctx context.Context, cutoff time.Time, batchSize int) (rowsDeleted int64, err error)

func (*PgDB) DeleteStream added in v0.28.4

func (cdb *PgDB) DeleteStream(ctx context.Context, streamId int32, batchSize int) (rowsDeleted int64, err error)

func (*PgDB) Estimate added in v0.12.0

func (cdb *PgDB) Estimate(table string) (f float64)

func (*PgDB) GetCertificateByHash added in v0.12.0

func (cdb *PgDB) GetCertificateByHash(ctx context.Context, hash []byte) (cert *JsonCertificate, err error)

func (*PgDB) GetCertificateByID added in v0.12.0

func (cdb *PgDB) GetCertificateByID(ctx context.Context, id int64) (cert *JsonCertificate, err error)

func (*PgDB) GetCertificateByLogEntry added in v0.12.0

func (cdb *PgDB) GetCertificateByLogEntry(ctx context.Context, entry *PgLogEntry) (cert *JsonCertificate, err error)

func (*PgDB) GetCertificatesByCommonName added in v0.24.5

func (cdb *PgDB) GetCertificatesByCommonName(ctx context.Context, commonname string) (certs []*JsonCertificate, err error)

func (*PgDB) GetHistoricalCertificates added in v0.28.7

func (cdb *PgDB) GetHistoricalCertificates(ctx context.Context, expiresAfter time.Time, callback func(ctx context.Context, cert *JsonCertificate) (err error)) (err error)

func (*PgDB) QueueUsage added in v0.15.0

func (cdb *PgDB) QueueUsage() (pct int)

type PgDnsname added in v0.12.0

type PgDnsname struct {
	Dnsname string
	CertID  int64
}

type PgDnsnamesView added in v0.12.0

type PgDnsnamesView struct {
	CertID    int64
	FQDN      string
	NotBefore time.Time
	Idna      bool
	Valid     bool
	PreCert   bool
	Issuer    string
	Subject   string
	Crtsh     string
	Domain    string
	Tld       string
}

type PgIdent added in v0.12.0

type PgIdent struct {
	Id           int
	Organization string
	Province     string
	Country      string
}

type PgLogEntry added in v0.12.0

type PgLogEntry struct {
	Seen     time.Time // CT log entry timestamp
	LogIndex int64     // CT log index for the stream
	CertID   int64     // database ID of cert
	StreamID int32     // database ID of stream
}

type Scanner added in v0.12.0

type Scanner interface {
	Scan(dest ...any) error
}

type StreamError added in v0.22.0

type StreamError struct {
	*LogStream
	When time.Time
	Err  error
}

func (StreamError) Error added in v0.22.0

func (ewt StreamError) Error() string

func (StreamError) Unwrap added in v0.22.0

func (ewt StreamError) Unwrap() error

Directories

Path Synopsis
cmd
certstream command

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL