From 3f4ae6525191a1c02b5a33a5f695ca59fb39a403 Mon Sep 17 00:00:00 2001 From: Daniel Cosme Date: Tue, 21 Apr 2026 13:06:42 -0400 Subject: [PATCH] Add Prometheus Stack --- apps/hydra/linkding/kuztomization.yaml | 2 +- clusters/hydra/infra.yaml | 19 ++++ cmd/apps/main.go | 10 ++ go.mod | 11 +- go.sum | 34 +++--- .../controllers/kube-prometheus-stack.yaml | 9 ++ .../monitoring/controllers/namespace.yaml | 6 ++ .../hydra/monitoring/controllers/release.yaml | 31 ++++++ pkg/flux/flux.go | 16 ++- pkg/linkding/linkding.go | 2 +- pkg/monitoring/monitoring.go | 100 ++++++++++++++++++ pkg/root/root.go | 18 +++- pkg/root/services.go | 4 + 13 files changed, 228 insertions(+), 34 deletions(-) create mode 100644 clusters/hydra/infra.yaml create mode 100644 infrastructure/hydra/monitoring/controllers/kube-prometheus-stack.yaml create mode 100644 infrastructure/hydra/monitoring/controllers/namespace.yaml create mode 100644 infrastructure/hydra/monitoring/controllers/release.yaml create mode 100644 pkg/monitoring/monitoring.go diff --git a/apps/hydra/linkding/kuztomization.yaml b/apps/hydra/linkding/kuztomization.yaml index c922b44..96e2f9e 100644 --- a/apps/hydra/linkding/kuztomization.yaml +++ b/apps/hydra/linkding/kuztomization.yaml @@ -4,7 +4,7 @@ metadata: name: linking namespace: linkding resources: -- namespace.yaml - srv.yaml - pvc.yaml - deployment.yaml +- namespace.yaml diff --git a/clusters/hydra/infra.yaml b/clusters/hydra/infra.yaml new file mode 100644 index 0000000..51b4aa0 --- /dev/null +++ b/clusters/hydra/infra.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: infrastructure + namespace: flux-system +spec: + decryption: + provider: sops + secretRef: + name: sops-age + interval: 10m0s + path: ./infrastructure/hydra + prune: true + retryInterval: 1m0s + sourceRef: + kind: GitRepository + name: flux-system + timeout: 5m0s +status: {} diff --git a/cmd/apps/main.go b/cmd/apps/main.go index 26454e8..0146785 100644 --- a/cmd/apps/main.go +++ b/cmd/apps/main.go @@ -7,6 +7,7 @@ import ( "danicos.dev/daniel/go-kube/pkg/stack" "danicos.dev/daniel/homelab/pkg/flux" "danicos.dev/daniel/homelab/pkg/linkding" + "danicos.dev/daniel/homelab/pkg/monitoring" "danicos.dev/daniel/homelab/pkg/root" /* apps "k8s.io/api/apps/v1" @@ -19,6 +20,15 @@ func main() { err := flux_stack.MarshalYamlFlat(root.FLUX_CLUSTER_HYDRA_PATH) assertNoErr(err) + hydra_monitoring := map[string]stack.Stack{ + "controllers": monitoring.Controllers(), + } + for name, s := range hydra_monitoring { + fmt.Printf("STACK: %s\n", name) + err = s.MarshalYaml(root.FLUX_INFRA_HYDRA_PATH + "/monitoring") + assertNoErr(err) + } + hydra_apps := map[string]stack.Stack{ "linkding": linkding.Stack(), } diff --git a/go.mod b/go.mod index d84d127..eb6d736 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,11 @@ module danicos.dev/daniel/homelab go 1.26.2 require ( - danicos.dev/daniel/go-kube v1.3.3 + danicos.dev/daniel/go-kube v1.5.0 github.com/fatih/color v1.19.0 github.com/fluxcd/kustomize-controller/api v1.8.3 github.com/fluxcd/pkg/apis/meta v1.25.1 + github.com/fluxcd/source-controller/api v1.8.3 github.com/magefile/mage v1.17.1 k8s.io/api v0.35.2 k8s.io/apimachinery v0.35.2 @@ -14,6 +15,8 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fluxcd/helm-controller/api v1.5.4 // indirect + github.com/fluxcd/pkg/apis/acl v0.9.0 // indirect github.com/fluxcd/pkg/apis/kustomize v1.15.1 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-errors/errors v1.4.2 // indirect @@ -32,9 +35,9 @@ require ( github.com/x448/float16 v0.8.4 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/net v0.49.0 // indirect - golang.org/x/sys v0.42.0 // indirect - golang.org/x/text v0.33.0 // indirect + golang.org/x/net v0.53.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/text v0.36.0 // indirect google.golang.org/protobuf v1.36.8 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 94807fe..12a5519 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -danicos.dev/daniel/go-kube v1.3.3 h1:+2WBBZzJQx9CsL5FRFlLlvdQr4LRrL4Uyz8j4c2Ndms= -danicos.dev/daniel/go-kube v1.3.3/go.mod h1:GsP3L4S4wBPhxrl/7kI2cr9xcletLLu1V5v5516+C90= +danicos.dev/daniel/go-kube v1.5.0 h1:ae8EKxQf7QqrgnOdRG7SnE0OZtfL7QSMHw9um/LMAqM= +danicos.dev/daniel/go-kube v1.5.0/go.mod h1:MBGwFBrGyqkEQ55mK0PP2TdKO1oQSih4hLiPjye+8Gg= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -8,12 +8,18 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= +github.com/fluxcd/helm-controller/api v1.5.4 h1:wbAwD+cSGBZEhT3qq1naBKkitdNbqRtWQUFNA3XTXOc= +github.com/fluxcd/helm-controller/api v1.5.4/go.mod h1:lTgeUmtVYExMKp7mRDncsr4JwHTz3LFtLjRJZeR98lI= github.com/fluxcd/kustomize-controller/api v1.8.3 h1:Ux9AAOY0lkP6FgRg5/b/ITvRSy8lz6VBBaZ9bXmTLmI= github.com/fluxcd/kustomize-controller/api v1.8.3/go.mod h1:c/mUPIffDDLg1EicXCJtX4N/rc+z5Zh0e/CXjhd7Dyc= +github.com/fluxcd/pkg/apis/acl v0.9.0 h1:wBpgsKT+jcyZEcM//OmZr9RiF8klL3ebrDp2u2ThsnA= +github.com/fluxcd/pkg/apis/acl v0.9.0/go.mod h1:TttNS+gocsGLwnvmgVi3/Yscwqrjc17+vhgYfqkfrV4= github.com/fluxcd/pkg/apis/kustomize v1.15.1 h1:t9QZh+3ZS8EKmlxrnnbcKZcGTrg8FDvMF1T8BHMCuqI= github.com/fluxcd/pkg/apis/kustomize v1.15.1/go.mod h1:IZOy4CCtR/hxMGb7erK1RfbGnczVv4/dRBoVD37AywI= github.com/fluxcd/pkg/apis/meta v1.25.1 h1:WG1GIC/SOz0GjxT0uVuO6AMicQ3yFsk6bDozCnq+fto= github.com/fluxcd/pkg/apis/meta v1.25.1/go.mod h1:c7o6mJGLCMvNrfdinGZehkrdZuFT9vZdZNrn66DtVD0= +github.com/fluxcd/source-controller/api v1.8.3 h1:WNEETjmp/YTZx5IMg9ewz2Wn8YzOVETeJJ0LIPivm40= +github.com/fluxcd/source-controller/api v1.8.3/go.mod h1:sio4t49RDx+S1etHRFAEEw8qfVuw0KKlOg8bRVlEYPM= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= @@ -87,19 +93,19 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= -golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= -golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= -golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= -golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= -golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= -golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= +golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/infrastructure/hydra/monitoring/controllers/kube-prometheus-stack.yaml b/infrastructure/hydra/monitoring/controllers/kube-prometheus-stack.yaml new file mode 100644 index 0000000..0c9a571 --- /dev/null +++ b/infrastructure/hydra/monitoring/controllers/kube-prometheus-stack.yaml @@ -0,0 +1,9 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: monitoring + namespace: monitoring +spec: + interval: 24h0m0s + url: https://prometheus-community.github.io/helm-charts +status: {} diff --git a/infrastructure/hydra/monitoring/controllers/namespace.yaml b/infrastructure/hydra/monitoring/controllers/namespace.yaml new file mode 100644 index 0000000..f639240 --- /dev/null +++ b/infrastructure/hydra/monitoring/controllers/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring +spec: {} +status: {} diff --git a/infrastructure/hydra/monitoring/controllers/release.yaml b/infrastructure/hydra/monitoring/controllers/release.yaml new file mode 100644 index 0000000..45dce6c --- /dev/null +++ b/infrastructure/hydra/monitoring/controllers/release.yaml @@ -0,0 +1,31 @@ +apiVersion: helm.toolkit.fluxcd.io/v1 +kind: HelmRelease +metadata: + name: monitoring + namespace: monitoring +spec: + chart: + spec: + chart: kube-prometheus-stack + interval: 12h0m0s + sourceRef: + kind: HelmRelease + name: monitoring + namespace: monitoring + version: 66.x + driftDetection: + ignore: + - paths: + - /metadata/annotations/prometheus-operator-validated + target: + kind: PrometheusRule + mode: enabled + install: + crds: Create + interval: 30m0s + upgrade: + crds: CreateReplace + values: + grafana: + adminPassword: grafana-admin +status: {} diff --git a/pkg/flux/flux.go b/pkg/flux/flux.go index 46307a4..635d6b0 100644 --- a/pkg/flux/flux.go +++ b/pkg/flux/flux.go @@ -11,22 +11,20 @@ import ( meta "k8s.io/apimachinery/pkg/apis/meta/v1" ) -var flux_apps_meta kube.Metadata var Flux_namespace = kube.Namespace(root.FLUX_NAMESPACE) -func init() { - flux_apps_meta = kube.NewMetadata("apps", Flux_namespace) -} - func Stack() stack.Stack { + flux_apps_meta := kube.NewMetadata("apps", Flux_namespace) + flux_infra_meta := kube.NewMetadata("infrastructure", Flux_namespace) s := stack.NewStack("flux", map[string]any{ - "apps": Apps(), + "apps": kuztomization(flux_apps_meta, root.FLUX_APPS_HYDRA_PATH), + "infra": kuztomization(flux_infra_meta, root.FLUX_INFRA_HYDRA_PATH), }) return s } -func Apps() kz.Kustomization { +func kuztomization(meta kube.Metadata, path string) kz.Kustomization { retryInteval := durMin(1) timeout := durMin(5) spec := kz.KustomizationSpec{ @@ -43,10 +41,10 @@ func Apps() kz.Kustomization { Kind: "GitRepository", Name: Flux_namespace.Name, }, - Path: root.FLUX_APPS_HYDRA_PATH, + Path: path, Prune: true, } - return kube.NewFluxKustomization(flux_apps_meta, spec) + return kube.NewFluxKustomization(meta, spec) } func durMin(d int64) meta.Duration { diff --git a/pkg/linkding/linkding.go b/pkg/linkding/linkding.go index 9610280..b0977eb 100644 --- a/pkg/linkding/linkding.go +++ b/pkg/linkding/linkding.go @@ -47,7 +47,7 @@ func Stack() stack.Stack { func deployment() apps.Deployment { storage := kube.NewVolumeFrom(kube.VolumeSourcePVC, "data", pvc.Name) envMapping := map[string]string{ - "LD_CSRF_TRUSTED_ORIGINS": "https://link.danicos.me", + "LD_CSRF_TRUSTED_ORIGINS": root.Linkding.PublicURL, } secretMapping := map[string]string{ "LD_SUPERUSER_NAME": Secret.SuperUserKey, diff --git a/pkg/monitoring/monitoring.go b/pkg/monitoring/monitoring.go new file mode 100644 index 0000000..a10a277 --- /dev/null +++ b/pkg/monitoring/monitoring.go @@ -0,0 +1,100 @@ +package monitoring + +import ( + "encoding/json" + "time" + + "danicos.dev/daniel/go-kube/pkg/kube" + "danicos.dev/daniel/go-kube/pkg/stack" + "danicos.dev/daniel/homelab/pkg/root" + helm "github.com/fluxcd/helm-controller/api/v2" + "github.com/fluxcd/pkg/apis/kustomize" + source "github.com/fluxcd/source-controller/api/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var meta kube.Metadata +var Namespace = kube.Namespace(root.Monitoring) + +func init() { + meta = kube.NewMetadata(root.Monitoring, Namespace) +} + +func Controllers() stack.Stack { + s := stack.NewStack("controllers", map[string]any{ + "namespace": Namespace, + "kube-prometheus-stack": PrometheusHelmSource(), + "release": PrometheusRelease(), + }) + + return s +} + +func PrometheusHelmSource() source.HelmRepository { + spec := source.HelmRepositorySpec{ + Interval: durHour(root.FLUX_HELM_MONITORING_INTERVAL), + URL: root.HELM_PROMETHEUS_URL, + } + return kube.NewFluxHelmRepositorySource(meta, spec) +} + +func PrometheusRelease() helm.HelmRelease { + type Grafana struct { + AdminPassword string `json:"adminPassword"` + } + type Values struct { + Grafana Grafana `json:"grafana"` + } + values := Values{ + Grafana: Grafana{ + // Note this password is safe because Grafana is only exposed via VPN + AdminPassword: "grafana-admin", + }, + } + raw, err := json.Marshal(values) + if err != nil { + panic(err) + } + interval := durHour(12) + spec := helm.HelmReleaseSpec{ + Interval: durMin(30), + Chart: &helm.HelmChartTemplate{ + Spec: helm.HelmChartTemplateSpec{ + Chart: root.HELM_PROMETHEUS_CHART, + Version: root.HELM_PROMETHEUS_CHART_VERSION, + Interval: &interval, + SourceRef: helm.CrossNamespaceObjectReference{ + Kind: kube.FluxHelmReleaseMeta.Kind, + Name: meta.Meta().Name, + Namespace: Namespace.Name, + }, + }, + }, + Install: &helm.Install{ + CRDs: helm.Create, + }, + Upgrade: &helm.Upgrade{ + CRDs: helm.CreateReplace, + }, + DriftDetection: &helm.DriftDetection{ + Mode: helm.DriftDetectionEnabled, + Ignore: []helm.IgnoreRule{{ + Paths: []string{"/metadata/annotations/prometheus-operator-validated"}, + Target: &kustomize.Selector{ + Kind: "PrometheusRule", + }, + }}, + }, + Values: &apiextensionsv1.JSON{Raw: raw}, + } + return kube.NewFluxHelmRelease(meta, spec) +} + +func durHour(d int64) metav1.Duration { + return metav1.Duration{Duration: (time.Duration(d) * time.Hour)} +} + +func durMin(d int64) metav1.Duration { + return metav1.Duration{Duration: (time.Duration(d) * time.Minute)} +} diff --git a/pkg/root/root.go b/pkg/root/root.go index a970023..60f7807 100644 --- a/pkg/root/root.go +++ b/pkg/root/root.go @@ -15,11 +15,19 @@ const ( ) const ( - FLUX_NAMESPACE = "flux-system" - FLUX_APPS_HYDRA_PATH = "./apps/" + HYDRA_CLUSTER - FLUX_APPS_SECRETS_HYDRA_PATH = "./apps/" + HYDRA_CLUSTER + "/secrets" - FLUX_CLUSTER_HYDRA_PATH = "./clusters/" + HYDRA_CLUSTER - FLUX_DECRYPTION_PROVIDER = "sops" + FLUX_NAMESPACE = "flux-system" + FLUX_APPS_HYDRA_PATH = "./apps/" + HYDRA_CLUSTER + FLUX_APPS_SECRETS_HYDRA_PATH = "./apps/" + HYDRA_CLUSTER + "/secrets" + FLUX_CLUSTER_HYDRA_PATH = "./clusters/" + HYDRA_CLUSTER + FLUX_INFRA_HYDRA_PATH = "./infrastructure/" + HYDRA_CLUSTER + FLUX_DECRYPTION_PROVIDER = "sops" + FLUX_HELM_MONITORING_INTERVAL = 24 // in hours +) + +const ( + HELM_PROMETHEUS_URL = "https://prometheus-community.github.io/helm-charts" + HELM_PROMETHEUS_CHART = "kube-prometheus-stack" + HELM_PROMETHEUS_CHART_VERSION = "66.x" ) var ( diff --git a/pkg/root/services.go b/pkg/root/services.go index 0687343..a704316 100644 --- a/pkg/root/services.go +++ b/pkg/root/services.go @@ -3,6 +3,7 @@ package root type Service struct { Name string Image string + PublicURL string Port int32 SecurityContextID int64 } @@ -10,6 +11,9 @@ type Service struct { var Linkding = Service{ Name: "linkding", Image: "sissbruecker/linkding:1.45.0", + PublicURL: "https://link.danicos.me", Port: 9090, SecurityContextID: 33, // www-data user, group and FS ID } + +var Monitoring = "monitoring"