Add AIA certificate prober to boulder-observer (#8624)

Add an AIA certificate prober to Boulder-observer, so we can verify the
served certificates have the right Common Name (preventing mixups),
content type and encoding. We export the certificate notBefore and
notAfter for expiry monitoring purposes.

This PR was largely written by Copilot under Matthew's supervision, and
is modelled after the CRL Prober.

Re-land of #8594, which was accidentally merged to a feature branch (due
to branch stacking).

Fixes #8593

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: Matthew McPherrin <git@mcpherrin.ca>
Co-authored-by: Aaron Gable <aaron@letsencrypt.org>

Author: 4 people

cmd/boulder-observer/README.md

@@ -25,9 +25,12 @@ Prometheus.
       * [CRL](#crl)
         * [Schema](#schema-4)
         * [Example](#example-4)
-      * [TLS](#tls)
+      * [AIA](#aia)
         * [Schema](#schema-5)
         * [Example](#example-5)
+      * [TLS](#tls)
+        * [Schema](#schema-6)
+        * [Example](#example-6)
   * [Metrics](#metrics)
     * [Global Metrics](#global-metrics)
       * [obs_monitors](#obs_monitors)
@@ -36,6 +39,9 @@ Prometheus.
       * [obs_crl_this_update](#obs_crl_this_update)
       * [obs_crl_next_update](#obs_crl_next_update)
       * [obs_crl_revoked_cert_count](#obs_crl_revoked_cert_count)
+    * [AIA Metrics](#aia-metrics)
+      * [obs_aia_not_before](#obs_aia_not_before)
+      * [obs_aia_not_after](#obs_aia_not_after)
     * [TLS Metrics](#tls-metrics)
       * [obs_crl_this_update](#obs_tls_not_after)
       * [obs_crl_next_update](#obs_tls_reason)
@@ -203,6 +209,26 @@ monitors:
       url: http://x1.c.lencr.org/
 ```
 
+#### AIA
+
+##### Schema
+
+`url`: Scheme + Hostname to grab the AIA certificate from (e.g. `http://r3.i.lencr.org/`).
+
+`expectCommonName`: Expected Common Name (CN) of the certificate. The prober verifies the certificate's CN matches this value.
+
+##### Example
+
+```yaml
+monitors:
+  - 
+    period: 1h
+    kind: AIA
+    settings:
+      url: http://r3.i.lencr.org/
+      expectCommonName: "R3"
+```
+
 #### TLS
 
 ##### Schema
@@ -321,6 +347,39 @@ Count of revoked certificates in a CRL.
 
 `url`: Url of the CRL
 
+### AIA Metrics
+
+These metrics will be available whenever a valid AIA prober is configured.
+
+#### obs_aia_not_before
+
+Unix timestamp value (in seconds) of the notBefore field for an AIA certificate.
+
+**Labels:**
+
+`url`: URL of the AIA certificate
+
+#### obs_aia_not_after
+
+Unix timestamp value (in seconds) of the notAfter field for an AIA certificate.
+
+**Labels:**
+
+`url`: URL of the AIA certificate
+
+**Example Usage:**
+
+This is a sample rule that alerts when an AIA certificate has a notAfter timestamp indicating that the certificate will expire within the next 30 days:
+
+```yaml
+- alert: AIACertExpiresSoon
+  expr: obs_aia_not_after{url="http://r3.i.lencr.org/"} <= time() + 2592000
+  labels:
+    severity: warning
+  annotations:
+    description: 'AIA certificate expires within 30 days'
+```
+
 ### TLS Metrics
 
 These metrics will be available whenever a valid TLS prober is configured.

observer/mon_conf.go

@@ -14,7 +14,7 @@ import (
 // MonConf is exported to receive YAML configuration in `ObsConf`.
 type MonConf struct {
 	Period   config.Duration  `yaml:"period"`
-	Kind     string           `yaml:"kind" validate:"required,oneof=DNS HTTP CRL TLS TCP"`
+	Kind     string           `yaml:"kind" validate:"required,oneof=DNS HTTP CRL TLS TCP AIA"`
 	Settings probers.Settings `yaml:"settings" validate:"min=1,dive"`
 }
 

observer/observer.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/letsencrypt/boulder/cmd"
 	blog "github.com/letsencrypt/boulder/log"
+	_ "github.com/letsencrypt/boulder/observer/probers/aia"
 	_ "github.com/letsencrypt/boulder/observer/probers/crl"
 	_ "github.com/letsencrypt/boulder/observer/probers/dns"
 	_ "github.com/letsencrypt/boulder/observer/probers/http"

observer/probers/aia/aia.go

@@ -0,0 +1,77 @@
+package probers
+
+import (
+	"context"
+	"crypto/x509"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// AIAProbe is the exported 'Prober' object for monitors configured to
+// monitor AIA certificate availability & characteristics.
+type AIAProbe struct {
+	url              string
+	expectCommonName string
+	cNotBefore       *prometheus.GaugeVec
+	cNotAfter        *prometheus.GaugeVec
+}
+
+// Name returns a string that uniquely identifies the monitor.
+func (p AIAProbe) Name() string {
+	return p.url
+}
+
+// Kind returns a name that uniquely identifies the `Kind` of `Prober`.
+func (p AIAProbe) Kind() string {
+	return "AIA"
+}
+
+// Probe requests the configured AIA certificate and publishes metrics about it if found.
+func (p AIAProbe) Probe(ctx context.Context) error {
+	req, err := http.NewRequestWithContext(ctx, "GET", p.url, nil)
+	if err != nil {
+		return err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	// Check Content-Type header
+	contentType := resp.Header.Get("Content-Type")
+	if contentType != "application/pkix-cert" {
+		return fmt.Errorf("certificate Content-Type is %q but want application/pkix-cert", contentType)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	// Parse the DER-encoded certificate
+	cert, err := x509.ParseCertificate(body)
+	if err != nil {
+		return err
+	}
+
+	// Check if the certificate is a CA certificate
+	if !cert.IsCA {
+		return fmt.Errorf("certificate is not a CA certificate")
+	}
+
+	// Check if the CommonName matches the expected value
+	if cert.Subject.CommonName != p.expectCommonName {
+		return fmt.Errorf("certificate has CN %q but want %q", cert.Subject.CommonName, p.expectCommonName)
+	}
+
+	// Report metrics for this certificate
+	p.cNotBefore.WithLabelValues(p.url).Set(float64(cert.NotBefore.Unix()))
+	p.cNotAfter.WithLabelValues(p.url).Set(float64(cert.NotAfter.Unix()))
+
+	return nil
+}

observer/probers/aia/aia_conf.go

@@ -0,0 +1,117 @@
+package probers
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/letsencrypt/boulder/observer/probers"
+	"github.com/letsencrypt/boulder/strictyaml"
+)
+
+const (
+	notBeforeName = "obs_aia_not_before"
+	notAfterName  = "obs_aia_not_after"
+)
+
+// AIAConf is exported to receive YAML configuration
+type AIAConf struct {
+	URL              string `yaml:"url"`
+	ExpectCommonName string `yaml:"expectCommonName"`
+}
+
+// Kind returns a name that uniquely identifies the `Kind` of `Configurer`.
+func (c AIAConf) Kind() string {
+	return "AIA"
+}
+
+// UnmarshalSettings constructs a AIAConf object from YAML as bytes.
+func (c AIAConf) UnmarshalSettings(settings []byte) (probers.Configurer, error) {
+	var conf AIAConf
+	err := strictyaml.Unmarshal(settings, &conf)
+
+	if err != nil {
+		return nil, err
+	}
+	return conf, nil
+}
+
+func (c AIAConf) validateURL() error {
+	url, err := url.Parse(c.URL)
+	if err != nil {
+		return fmt.Errorf(
+			"invalid 'url', got: %q, expected a valid url", c.URL)
+	}
+	if url.Scheme == "" {
+		return fmt.Errorf(
+			"invalid 'url', got: %q, missing scheme", c.URL)
+	}
+	return nil
+}
+
+// MakeProber constructs a `AIAProbe` object from the contents of the
+// bound `AIAConf` object. If the `AIAConf` cannot be validated, an
+// error appropriate for end-user consumption is returned instead.
+func (c AIAConf) MakeProber(collectors map[string]prometheus.Collector) (probers.Prober, error) {
+	// validate `url`
+	err := c.validateURL()
+	if err != nil {
+		return nil, err
+	}
+
+	// validate `expectCommonName` is provided
+	if c.ExpectCommonName == "" {
+		return nil, fmt.Errorf("'expectCommonName' is required")
+	}
+
+	// validate the prometheus collectors that were passed in
+	coll, ok := collectors[notBeforeName]
+	if !ok {
+		return nil, fmt.Errorf("aia prober did not receive collector %q", notBeforeName)
+	}
+	notBeforeColl, ok := coll.(*prometheus.GaugeVec)
+	if !ok {
+		return nil, fmt.Errorf("aia prober received collector %q of wrong type, got: %T, expected *prometheus.GaugeVec", notBeforeName, coll)
+	}
+
+	coll, ok = collectors[notAfterName]
+	if !ok {
+		return nil, fmt.Errorf("aia prober did not receive collector %q", notAfterName)
+	}
+	notAfterColl, ok := coll.(*prometheus.GaugeVec)
+	if !ok {
+		return nil, fmt.Errorf("aia prober received collector %q of wrong type, got: %T, expected *prometheus.GaugeVec", notAfterName, coll)
+	}
+
+	return AIAProbe{c.URL, c.ExpectCommonName, notBeforeColl, notAfterColl}, nil
+}
+
+// Instrument constructs any `prometheus.Collector` objects the `AIAProbe` will
+// need to report its own metrics. A map is returned containing the constructed
+// objects, indexed by the name of the prometheus metric. If no objects were
+// constructed, nil is returned.
+func (c AIAConf) Instrument() map[string]prometheus.Collector {
+	notBefore := prometheus.Collector(prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: notBeforeName,
+			Help: "AIA certificate notBefore Unix timestamp in seconds",
+		}, []string{"url"},
+	))
+	notAfter := prometheus.Collector(prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: notAfterName,
+			Help: "AIA certificate notAfter Unix timestamp in seconds",
+		}, []string{"url"},
+	))
+	return map[string]prometheus.Collector{
+		notBeforeName: notBefore,
+		notAfterName:  notAfter,
+	}
+}
+
+// init is called at runtime and registers `AIAConf`, a `Prober`
+// `Configurer` type, as "AIA".
+func init() {
+	probers.Register(AIAConf{})
+}