Add Prometheus Stack

apps/hydra/linkding/kuztomization.yaml

@@ -4,7 +4,7 @@ metadata:
   name: linking
   namespace: linkding
 resources:
-- namespace.yaml
 - srv.yaml
 - pvc.yaml
 - deployment.yaml
+- namespace.yaml

clusters/hydra/infra.yaml

@@ -0,0 +1,19 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: infrastructure
+  namespace: flux-system
+spec:
+  decryption:
+    provider: sops
+    secretRef:
+      name: sops-age
+  interval: 10m0s
+  path: ./infrastructure/hydra
+  prune: true
+  retryInterval: 1m0s
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 5m0s
+status: {}

cmd/apps/main.go

@@ -7,6 +7,7 @@ import (
 	"danicos.dev/daniel/go-kube/pkg/stack"
 	"danicos.dev/daniel/homelab/pkg/flux"
 	"danicos.dev/daniel/homelab/pkg/linkding"
+	"danicos.dev/daniel/homelab/pkg/monitoring"
 	"danicos.dev/daniel/homelab/pkg/root"
 	/*
 		apps "k8s.io/api/apps/v1"
@@ -19,6 +20,15 @@ func main() {
 	err := flux_stack.MarshalYamlFlat(root.FLUX_CLUSTER_HYDRA_PATH)
 	assertNoErr(err)
 
+	hydra_monitoring := map[string]stack.Stack{
+		"controllers": monitoring.Controllers(),
+	}
+	for name, s := range hydra_monitoring {
+		fmt.Printf("STACK: %s\n", name)
+		err = s.MarshalYaml(root.FLUX_INFRA_HYDRA_PATH + "/monitoring")
+		assertNoErr(err)
+	}
+
 	hydra_apps := map[string]stack.Stack{
 		"linkding": linkding.Stack(),
 	}

go.mod

@@ -3,10 +3,11 @@ module danicos.dev/daniel/homelab
 go 1.26.2
 
 require (
-	danicos.dev/daniel/go-kube v1.3.3
+	danicos.dev/daniel/go-kube v1.5.0
 	github.com/fatih/color v1.19.0
 	github.com/fluxcd/kustomize-controller/api v1.8.3
 	github.com/fluxcd/pkg/apis/meta v1.25.1
+	github.com/fluxcd/source-controller/api v1.8.3
 	github.com/magefile/mage v1.17.1
 	k8s.io/api v0.35.2
 	k8s.io/apimachinery v0.35.2
@@ -14,6 +15,8 @@ require (
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fluxcd/helm-controller/api v1.5.4 // indirect
+	github.com/fluxcd/pkg/apis/acl v0.9.0 // indirect
 	github.com/fluxcd/pkg/apis/kustomize v1.15.1 // indirect
 	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
@@ -32,9 +35,9 @@ require (
 	github.com/x448/float16 v0.8.4 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/net v0.49.0 // indirect
+	golang.org/x/net v0.53.0 // indirect
-	golang.org/x/sys v0.42.0 // indirect
+	golang.org/x/sys v0.43.0 // indirect
-	golang.org/x/text v0.33.0 // indirect
+	golang.org/x/text v0.36.0 // indirect
 	google.golang.org/protobuf v1.36.8 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -1,5 +1,5 @@
-danicos.dev/daniel/go-kube v1.3.3 h1:+2WBBZzJQx9CsL5FRFlLlvdQr4LRrL4Uyz8j4c2Ndms=
+danicos.dev/daniel/go-kube v1.5.0 h1:ae8EKxQf7QqrgnOdRG7SnE0OZtfL7QSMHw9um/LMAqM=
-danicos.dev/daniel/go-kube v1.3.3/go.mod h1:GsP3L4S4wBPhxrl/7kI2cr9xcletLLu1V5v5516+C90=
+danicos.dev/daniel/go-kube v1.5.0/go.mod h1:MBGwFBrGyqkEQ55mK0PP2TdKO1oQSih4hLiPjye+8Gg=
 github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
 github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -8,12 +8,18 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w=
 github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE=
+github.com/fluxcd/helm-controller/api v1.5.4 h1:wbAwD+cSGBZEhT3qq1naBKkitdNbqRtWQUFNA3XTXOc=
+github.com/fluxcd/helm-controller/api v1.5.4/go.mod h1:lTgeUmtVYExMKp7mRDncsr4JwHTz3LFtLjRJZeR98lI=
 github.com/fluxcd/kustomize-controller/api v1.8.3 h1:Ux9AAOY0lkP6FgRg5/b/ITvRSy8lz6VBBaZ9bXmTLmI=
 github.com/fluxcd/kustomize-controller/api v1.8.3/go.mod h1:c/mUPIffDDLg1EicXCJtX4N/rc+z5Zh0e/CXjhd7Dyc=
+github.com/fluxcd/pkg/apis/acl v0.9.0 h1:wBpgsKT+jcyZEcM//OmZr9RiF8klL3ebrDp2u2ThsnA=
+github.com/fluxcd/pkg/apis/acl v0.9.0/go.mod h1:TttNS+gocsGLwnvmgVi3/Yscwqrjc17+vhgYfqkfrV4=
 github.com/fluxcd/pkg/apis/kustomize v1.15.1 h1:t9QZh+3ZS8EKmlxrnnbcKZcGTrg8FDvMF1T8BHMCuqI=
 github.com/fluxcd/pkg/apis/kustomize v1.15.1/go.mod h1:IZOy4CCtR/hxMGb7erK1RfbGnczVv4/dRBoVD37AywI=
 github.com/fluxcd/pkg/apis/meta v1.25.1 h1:WG1GIC/SOz0GjxT0uVuO6AMicQ3yFsk6bDozCnq+fto=
 github.com/fluxcd/pkg/apis/meta v1.25.1/go.mod h1:c7o6mJGLCMvNrfdinGZehkrdZuFT9vZdZNrn66DtVD0=
+github.com/fluxcd/source-controller/api v1.8.3 h1:WNEETjmp/YTZx5IMg9ewz2Wn8YzOVETeJJ0LIPivm40=
+github.com/fluxcd/source-controller/api v1.8.3/go.mod h1:sio4t49RDx+S1etHRFAEEw8qfVuw0KKlOg8bRVlEYPM=
 github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
 github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
@@ -87,19 +93,19 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
 go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
-golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
+golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
-golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
+golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
-golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
+golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
-golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
+golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
-golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
-golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
+golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
-golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
+golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
-golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
+golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
-golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=
+golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
-golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
+golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
 google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
 google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

infrastructure/hydra/monitoring/controllers/kube-prometheus-stack.yaml

@@ -0,0 +1,9 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: monitoring
+  namespace: monitoring
+spec:
+  interval: 24h0m0s
+  url: https://prometheus-community.github.io/helm-charts
+status: {}

infrastructure/hydra/monitoring/controllers/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: monitoring
+spec: {}
+status: {}

infrastructure/hydra/monitoring/controllers/release.yaml

@@ -0,0 +1,31 @@
+apiVersion: helm.toolkit.fluxcd.io/v1
+kind: HelmRelease
+metadata:
+  name: monitoring
+  namespace: monitoring
+spec:
+  chart:
+    spec:
+      chart: kube-prometheus-stack
+      interval: 12h0m0s
+      sourceRef:
+        kind: HelmRelease
+        name: monitoring
+        namespace: monitoring
+      version: 66.x
+  driftDetection:
+    ignore:
+    - paths:
+      - /metadata/annotations/prometheus-operator-validated
+      target:
+        kind: PrometheusRule
+    mode: enabled
+  install:
+    crds: Create
+  interval: 30m0s
+  upgrade:
+    crds: CreateReplace
+  values:
+    grafana:
+      adminPassword: grafana-admin
+status: {}

pkg/flux/flux.go

@@ -11,22 +11,20 @@ import (
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-var flux_apps_meta kube.Metadata
 var Flux_namespace = kube.Namespace(root.FLUX_NAMESPACE)
 
-func init() {
-	flux_apps_meta = kube.NewMetadata("apps", Flux_namespace)
-}
-
 func Stack() stack.Stack {
+	flux_apps_meta := kube.NewMetadata("apps", Flux_namespace)
+	flux_infra_meta := kube.NewMetadata("infrastructure", Flux_namespace)
 
 	s := stack.NewStack("flux", map[string]any{
-		"apps": Apps(),
+		"apps":  kuztomization(flux_apps_meta, root.FLUX_APPS_HYDRA_PATH),
+		"infra": kuztomization(flux_infra_meta, root.FLUX_INFRA_HYDRA_PATH),
 	})
 	return s
 }
 
-func Apps() kz.Kustomization {
+func kuztomization(meta kube.Metadata, path string) kz.Kustomization {
 	retryInteval := durMin(1)
 	timeout := durMin(5)
 	spec := kz.KustomizationSpec{
@@ -43,10 +41,10 @@ func Apps() kz.Kustomization {
 			Kind: "GitRepository",
 			Name: Flux_namespace.Name,
 		},
-		Path:  root.FLUX_APPS_HYDRA_PATH,
+		Path:  path,
 		Prune: true,
 	}
-	return kube.NewFluxKustomization(flux_apps_meta, spec)
+	return kube.NewFluxKustomization(meta, spec)
 }
 
 func durMin(d int64) meta.Duration {

pkg/linkding/linkding.go

@@ -47,7 +47,7 @@ func Stack() stack.Stack {
 func deployment() apps.Deployment {
 	storage := kube.NewVolumeFrom(kube.VolumeSourcePVC, "data", pvc.Name)
 	envMapping := map[string]string{
-		"LD_CSRF_TRUSTED_ORIGINS": "https://link.danicos.me",
+		"LD_CSRF_TRUSTED_ORIGINS": root.Linkding.PublicURL,
 	}
 	secretMapping := map[string]string{
 		"LD_SUPERUSER_NAME":     Secret.SuperUserKey,

pkg/monitoring/monitoring.go

@@ -0,0 +1,100 @@
+package monitoring
+
+import (
+	"encoding/json"
+	"time"
+
+	"danicos.dev/daniel/go-kube/pkg/kube"
+	"danicos.dev/daniel/go-kube/pkg/stack"
+	"danicos.dev/daniel/homelab/pkg/root"
+	helm "github.com/fluxcd/helm-controller/api/v2"
+	"github.com/fluxcd/pkg/apis/kustomize"
+	source "github.com/fluxcd/source-controller/api/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var meta kube.Metadata
+var Namespace = kube.Namespace(root.Monitoring)
+
+func init() {
+	meta = kube.NewMetadata(root.Monitoring, Namespace)
+}
+
+func Controllers() stack.Stack {
+	s := stack.NewStack("controllers", map[string]any{
+		"namespace":             Namespace,
+		"kube-prometheus-stack": PrometheusHelmSource(),
+		"release":               PrometheusRelease(),
+	})
+
+	return s
+}
+
+func PrometheusHelmSource() source.HelmRepository {
+	spec := source.HelmRepositorySpec{
+		Interval: durHour(root.FLUX_HELM_MONITORING_INTERVAL),
+		URL:      root.HELM_PROMETHEUS_URL,
+	}
+	return kube.NewFluxHelmRepositorySource(meta, spec)
+}
+
+func PrometheusRelease() helm.HelmRelease {
+	type Grafana struct {
+		AdminPassword string `json:"adminPassword"`
+	}
+	type Values struct {
+		Grafana Grafana `json:"grafana"`
+	}
+	values := Values{
+		Grafana: Grafana{
+			// Note this password is safe because Grafana is only exposed via VPN
+			AdminPassword: "grafana-admin",
+		},
+	}
+	raw, err := json.Marshal(values)
+	if err != nil {
+		panic(err)
+	}
+	interval := durHour(12)
+	spec := helm.HelmReleaseSpec{
+		Interval: durMin(30),
+		Chart: &helm.HelmChartTemplate{
+			Spec: helm.HelmChartTemplateSpec{
+				Chart:    root.HELM_PROMETHEUS_CHART,
+				Version:  root.HELM_PROMETHEUS_CHART_VERSION,
+				Interval: &interval,
+				SourceRef: helm.CrossNamespaceObjectReference{
+					Kind:      kube.FluxHelmReleaseMeta.Kind,
+					Name:      meta.Meta().Name,
+					Namespace: Namespace.Name,
+				},
+			},
+		},
+		Install: &helm.Install{
+			CRDs: helm.Create,
+		},
+		Upgrade: &helm.Upgrade{
+			CRDs: helm.CreateReplace,
+		},
+		DriftDetection: &helm.DriftDetection{
+			Mode: helm.DriftDetectionEnabled,
+			Ignore: []helm.IgnoreRule{{
+				Paths: []string{"/metadata/annotations/prometheus-operator-validated"},
+				Target: &kustomize.Selector{
+					Kind: "PrometheusRule",
+				},
+			}},
+		},
+		Values: &apiextensionsv1.JSON{Raw: raw},
+	}
+	return kube.NewFluxHelmRelease(meta, spec)
+}
+
+func durHour(d int64) metav1.Duration {
+	return metav1.Duration{Duration: (time.Duration(d) * time.Hour)}
+}
+
+func durMin(d int64) metav1.Duration {
+	return metav1.Duration{Duration: (time.Duration(d) * time.Minute)}
+}

pkg/root/root.go

@@ -15,11 +15,19 @@ const (
 )
 
 const (
-	FLUX_NAMESPACE               = "flux-system"
+	FLUX_NAMESPACE                = "flux-system"
-	FLUX_APPS_HYDRA_PATH         = "./apps/" + HYDRA_CLUSTER
+	FLUX_APPS_HYDRA_PATH          = "./apps/" + HYDRA_CLUSTER
-	FLUX_APPS_SECRETS_HYDRA_PATH = "./apps/" + HYDRA_CLUSTER + "/secrets"
+	FLUX_APPS_SECRETS_HYDRA_PATH  = "./apps/" + HYDRA_CLUSTER + "/secrets"
-	FLUX_CLUSTER_HYDRA_PATH      = "./clusters/" + HYDRA_CLUSTER
+	FLUX_CLUSTER_HYDRA_PATH       = "./clusters/" + HYDRA_CLUSTER
-	FLUX_DECRYPTION_PROVIDER     = "sops"
+	FLUX_INFRA_HYDRA_PATH         = "./infrastructure/" + HYDRA_CLUSTER
+	FLUX_DECRYPTION_PROVIDER      = "sops"
+	FLUX_HELM_MONITORING_INTERVAL = 24 // in hours
+)
+
+const (
+	HELM_PROMETHEUS_URL           = "https://prometheus-community.github.io/helm-charts"
+	HELM_PROMETHEUS_CHART         = "kube-prometheus-stack"
+	HELM_PROMETHEUS_CHART_VERSION = "66.x"
 )
 
 var (

pkg/root/services.go

@@ -3,6 +3,7 @@ package root
 type Service struct {
 	Name              string
 	Image             string
+	PublicURL         string
 	Port              int32
 	SecurityContextID int64
 }
@@ -10,6 +11,9 @@ type Service struct {
 var Linkding = Service{
 	Name:              "linkding",
 	Image:             "sissbruecker/linkding:1.45.0",
+	PublicURL:         "https://link.danicos.me",
 	Port:              9090,
 	SecurityContextID: 33, // www-data user, group and FS ID
 }
+
+var Monitoring = "monitoring"