daniel/homelab
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finish TrueNAS CSI Driver

Browse Source
This commit is contained in:
Daniel Cosme
2026-04-29 19:15:55 -04:00
parent bc2747bef8
commit a388ac53e2
22 changed files with 1064 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
apps/hydra/secrets/kuztomization.yaml
View File

@@ -4,3 +4,4 @@ metadata:
name: secrets
resources:
- linkding.yaml
- truenas-csi.yaml

18
apps/hydra/secrets/linkding.yaml
View File

@@ -4,20 +4,20 @@ metadata:
name: linkding
namespace: linkding
stringData:
supe_user_name: ENC[AES256_GCM,data:vBUmSZzQnDMY9GfGbzanZXE=,iv:VD99G6KvmWNmQ/ciVCrnw/pinE/83/l3gC2fLCi+vSE=,tag:XyqtSQT+uhVCf6SsH15Pgg==,type:str]
supe_user_password: ENC[AES256_GCM,data:4Ktxmh8fogYKoxSfWpEUFAhpiFTvfRq4yJ/nekQJl9cBbbVy4UqTZqlVF3A=,iv:wjkkoih66NxSVAinDmv8enoyqMnhn/8+c3KwUoe0yMg=,tag:R6vk7hMsZwdE7V8VePVM0w==,type:str]
supe_user_name: ENC[AES256_GCM,data:XvTjgXWqxeY7kTdEu4ez3/w=,iv:7v9BWmQpqnNYYdWPyD07xIcHoJAwkrGq11d2wP49j14=,tag:GyZtZme1DheHjNFuBp7nbA==,type:str]
supe_user_password: ENC[AES256_GCM,data:ATUaLra8h2OFUP8DkRG5kvPqR+OZKzbGZRQ60ECrCTkh+//M81o0GBrX0Nc=,iv:UzKVJRYWjKhEs50GNkijG0XiPAkiGKXWtqHZSEFYEpY=,tag:ROali/QL3ihSyWgSXh091w==,type:str]
sops:
age:
- recipient: age1lelpkv7u2xh5wezuwp09fmf9gsa8gp4rzy92jz0t203au82a7u5sutsjwa
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFUGpwMm9WcHBnRU1aeUFY
aktaV3BYdUZ1RjkrMGVwYnlFbU1IUmR6SEVZCllVbGk4MHdXWElNT2pVRytLcFQx
ck1BN2srd0V6NGNXdEh5eUowSjY2em8KLS0tICtBZzBxU0ZjSXFUWUVmQ0NyWDZH
dHhjZEo2VDhQMVY5WVpzQ0sxRHNkcXcK/hogutoNEBHZeHzc07uj8W7PKeX09KTS
FrEPVxt3w2tbZ8LmY19vNv/EMFd8l9wttIzVDqmqmzTZASwJUVQN9w==
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBocEdFTVhWT0dZUGZlTHhK
MjcxbFpFd3lydnJPMTV5T2pqblRVdnBZZ1FRCnFlV09oaFptY2JvTGVmZ3poQ2Nz
cm1IME13djMwbHJraVhPOEpBN1FqOEkKLS0tIEV5SkN1OXkxZDkrNFRhSEhoRDZC
RzlpNytqZGJOYW1BU0hOdFEyV3RjeEkKIWRRXhJTevlTCnlhoV3xoP6Kwtqt+aaE
wZECZ5N9Gk8JehsLkv5ShYxqcuenC8Rg/0Lc9Pmp6xhgJgWwJJzl+w==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2026-04-21T14:19:39Z"
mac: ENC[AES256_GCM,data:kYq1vulOhGvFDJVE0sclrJpkTHTUgiE9xD7Onr5fya51nJtm+KL38zl9FxcTlbi5H5wdyCoRgNq0vBycIenaCBm7pg19gDORlgu8WvOUNV9/CGZrahPnYSty72PNlMFvZ3+SIF3glZdOhKT2bTrVXbxLbccFPR+8MJT7ETaXeTo=,iv:mKF3NZBw3nqLILexxDWypFCVF8NHn8I/RzYI9rX2Pic=,tag:SJwnv6MjLvthQHqjZLUCvw==,type:str]
lastmodified: "2026-04-27T20:50:36Z"
mac: ENC[AES256_GCM,data:lh8FgtmZI59b/lHNAW6ScWG4yE/63hBkAbwhoaPwQNRSOAgTGG0xy147zqO7R/dryQmgjNBiZU8tD9KOmqoKRYvi10BxHbnT83gR3IpKSx2dTZldw2Odp1y7MJxsiG646N/CqsEKP4+K7oP4GZT/ERrq03dDDhN3ZFdsxg4Xuu0=,iv:TIswHRnyihQrrBPozXUiZv8XjXiZGqptlf7ckxLWTJo=,tag:x6z5SE94x9Ewej3XjHcUyA==,type:str]
encrypted_regex: ^(data|stringData)$
version: 3.12.2

22
apps/hydra/secrets/truenas-csi.yaml Normal file
View File

@@ -0,0 +1,22 @@
apiVersion: v1
kind: Secret
metadata:
name: truenas-csi-api-credentials
namespace: truenas-csi
stringData:
api-key: ENC[AES256_GCM,data:rLckxqJRQRrRf5t9r/9tkGau0Jmq0GWvIS6CuIb8DSa0p3PnmWZ8XxptPf0zYylcwVmcHTypU/rQXL1cVjovj61U,iv:nD50QitcpDVJ7Xrduqg4N75qa8m6Kei7LtDc5ZO0+fI=,tag:RFKG8rZ0HLQ+skJIzAV5NA==,type:str]
sops:
age:
- recipient: age1lelpkv7u2xh5wezuwp09fmf9gsa8gp4rzy92jz0t203au82a7u5sutsjwa
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBVWldlbGNyK2lHUTFQUGI2
aWxVZERyYXRDYVEwVTRyVkorSG1sMkxnWkRZCk1NM0hPNEc1YjY2Y2lFL3lMUkFk
RmZYamhJSGFUc2hXQm9IMnJFdUZoRGMKLS0tIFZWM1FTSkZnU0NEd1YycnpnYVFQ
M2NsdTdjMlZvSUluU1d5TG1CMXdpcnMKQWmdbo9Clk7SGmD6AwXfcZnbbXKrMgti
q2Cn+ZRDvZEYwQtMp/ob8iwbrl9KrUURNq/1GkmCjy73fy+MzTCnCQ==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2026-04-27T20:50:36Z"
mac: ENC[AES256_GCM,data:Rc3fcgj1BZR+jK4pHmukqPfdsZuxv/31RFLQJ8oV8XvU3eN1eedaS9DPUPss3VyLSnc0hjwlcCkd/QwNxeAUg3rHgWt5tc5m2nxIcjuHyuTMpoXvQ3xzOzTfC/DsewpKHuGR6lfF74x4SFZrwoocZztMh6i930lzfBk4FV4q0/Q=,iv:gA+XJvUYYDpPmNRmoeJvcu/J0rFvWGU+umUnem5tcfI=,tag:92ArjqfdFjV3qtJn2bK+Jw==,type:str]
encrypted_regex: ^(data|stringData)$
version: 3.12.2

2
go.mod
View File

@@ -3,7 +3,7 @@ module danicos.dev/daniel/homelab
go 1.26.2
require (
danicos.dev/daniel/go-kube v1.5.1
danicos.dev/daniel/go-kube v1.9.0
github.com/fatih/color v1.19.0
github.com/fluxcd/helm-controller/api v1.5.4
github.com/fluxcd/kustomize-controller/api v1.8.3

4
go.sum
View File

@@ -1,5 +1,5 @@
danicos.dev/daniel/go-kube v1.5.1 h1:EtKHQGu0I82Sl8Ud2Tj5qXELq9zDlkOWj5oSMfX96+8=
danicos.dev/daniel/go-kube v1.5.1/go.mod h1:MBGwFBrGyqkEQ55mK0PP2TdKO1oQSih4hLiPjye+8Gg=
danicos.dev/daniel/go-kube v1.9.0 h1:agofABwT1oa/gaxV4Q/KvUvxz1iVr8aiAB+X41WSv8s=
danicos.dev/daniel/go-kube v1.9.0/go.mod h1:MBGwFBrGyqkEQ55mK0PP2TdKO1oQSih4hLiPjye+8Gg=
github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=

11
infrastructure/hydra/truenas-csi/CSIDriver.yaml Normal file
View File

@@ -0,0 +1,11 @@
apiVersion: storage.k8s.io/v1
kind: CSIDriver
metadata:
name: csi.truenas.io
spec:
attachRequired: true
fsGroupPolicy: File
podInfoOnMount: true
volumeLifecycleModes:
- Ephemeral
- Persistent

12
infrastructure/hydra/truenas-csi/config.yaml Normal file
View File

@@ -0,0 +1,12 @@
apiVersion: v1
data:
defaultPool: datapool
iscsiIQNBase: iqn.net.ts.orca-uaru.apex-truenas
iscsiPortal: apex-truenas.orca-uaru.ts.net:3260
nfsServer: apex-truenas.orca-uaru.ts.net
truenasInsecure: "true"
truenasURL: wss://apex-truenas.orca-uaru.ts.net/api/current
kind: ConfigMap
metadata:
name: truenas-csi
namespace: truenas-csi

12
infrastructure/hydra/truenas-csi/controller-binding.yaml Normal file
View File

@@ -0,0 +1,12 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: truenas-csi-controller-role-binding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: truenas-csi-controller-role
subjects:
- kind: ServiceAccount
name: truenas-csi-controller-sa
namespace: truenas-csi

138
infrastructure/hydra/truenas-csi/controller-cluster-role.yaml Normal file
View File

@@ -0,0 +1,138 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: truenas-csi-controller-role
rules:
- apiGroups:
- ""
resources:
- persistentvolumes
verbs:
- get
- create
- update
- patch
- delete
- list
- watch
- apiGroups:
- ""
resources:
- persistentvolumeclaims
verbs:
- get
- list
- watch
- update
- patch
- apiGroups:
- ""
resources:
- persistentvolumeclaims/status
verbs:
- update
- patch
- apiGroups:
- ""
resources:
- events
verbs:
- get
- list
- watch
- update
- patch
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- list
- watch
- apiGroups:
- storage.k8s.io
resources:
- storageclasses
verbs:
- get
- list
- watch
- apiGroups:
- storage.k8s.io
resources:
- csinodes
verbs:
- get
- list
- watch
- apiGroups:
- storage.k8s.io
resources:
- volumeattachments
verbs:
- get
- create
- update
- patch
- delete
- list
- watch
- apiGroups:
- storage.k8s.io
resources:
- volumeattachments/status
verbs:
- patch
- apiGroups:
- snapshot.storage.k8s.io
resources:
- volumesnapshots
verbs:
- get
- list
- watch
- update
- patch
- apiGroups:
- snapshot.storage.k8s.io
resources:
- volumesnapshots/status
verbs:
- update
- patch
- apiGroups:
- snapshot.storage.k8s.io
resources:
- volumesnapshotcontents
verbs:
- get
- create
- update
- patch
- delete
- list
- watch
- apiGroups:
- snapshot.storage.k8s.io
resources:
- volumesnapshotcontents/status
verbs:
- update
- patch
- apiGroups:
- snapshot.storage.k8s.io
resources:
- volumesnapshotclasses
verbs:
- get
- list
- watch

153
infrastructure/hydra/truenas-csi/controller-deployment.yaml Normal file
View File

@@ -0,0 +1,153 @@
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: truenas-csi-controller
name: truenas-csi-controller
namespace: truenas-csi
spec:
selector:
matchLabels:
app: truenas-csi-controller
strategy: {}
template:
metadata:
labels:
app: truenas-csi-controller
spec:
containers:
- args:
- --endpoint=$(CSI_ENDPOINT)
- --node-id=$(NODE_ID)
- --mode=controller
- --v=4
env:
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
- name: TRUENAS_API_KEY
valueFrom:
secretKeyRef:
key: api-key
name: truenas-csi-api-credentials
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: TRUENAS_DEFAULT_POOL
valueFrom:
configMapKeyRef:
key: defaultPool
name: truenas-csi
- name: TRUENAS_INSECURE_SKIP_VERIFY
valueFrom:
configMapKeyRef:
key: truenasInsecure
name: truenas-csi
- name: TRUENAS_ISCSI_IQN_BASE
valueFrom:
configMapKeyRef:
key: iscsiIQNBase
name: truenas-csi
- name: TRUENAS_ISCSI_PORTAL
valueFrom:
configMapKeyRef:
key: iscsiPortal
name: truenas-csi
- name: TRUENAS_NFS_SERVER
valueFrom:
configMapKeyRef:
key: nfsServer
name: truenas-csi
- name: TRUENAS_URL
valueFrom:
configMapKeyRef:
key: truenasURL
name: truenas-csi
image: ghcr.io/truenas/truenas-csi:latest
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 5
httpGet:
path: /healthz
port: 9808
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 3
name: csi-controller
resources: {}
volumeMounts:
- mountPath: /csi
name: socket-dir
- args:
- --csi-address=$(ADDRESS)
- --v=5
- --feature-gates=Topology=true
- --extra-create-metadata
- --leader-election=true
- --default-fstype=ext4
- --timeout=60s
env:
- name: ADDRESS
value: /csi/csi.sock
image: registry.k8s.io/sig-storage/csi-provisioner:v6.1.1
name: csi-provisioner
resources: {}
volumeMounts:
- mountPath: /csi
name: socket-dir
- args:
- --csi-address=$(ADDRESS)
- --v=5
- --leader-election=true
- --timeout=60s
env:
- name: ADDRESS
value: /csi/csi.sock
image: registry.k8s.io/sig-storage/csi-attacher:v4.11.0
name: csi-attacher
resources: {}
volumeMounts:
- mountPath: /csi
name: socket-dir
- args:
- --csi-address=$(ADDRESS)
- --v=5
- --leader-election=true
- --timeout=60s
env:
- name: ADDRESS
value: /csi/csi.sock
image: registry.k8s.io/sig-storage/csi-snapshotter:v8.5.0
name: csi-snapshotter
resources: {}
volumeMounts:
- mountPath: /csi
name: socket-dir
- args:
- --csi-address=$(ADDRESS)
- --v=5
- --leader-election=true
- --timeout=60s
env:
- name: ADDRESS
value: /csi/csi.sock
image: registry.k8s.io/sig-storage/csi-resizer:v2.1.0
name: csi-resizer
resources: {}
volumeMounts:
- mountPath: /csi
name: socket-dir
- args:
- --csi-address=/csi/csi.sock
- --health-port=9808
image: registry.k8s.io/sig-storage/livenessprobe:v2.18.0
name: liveness-probe
resources: {}
volumeMounts:
- mountPath: /csi
name: socket-dir
serviceAccountName: truenas-csi-controller-sa
volumes:
- emptyDir: {}
name: socket-dir
status: {}

5
infrastructure/hydra/truenas-csi/controller-service-account.yaml Normal file
View File

@@ -0,0 +1,5 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: truenas-csi-controller-sa
namespace: truenas-csi

10
infrastructure/hydra/truenas-csi/kuztomization.yaml
View File

@@ -4,4 +4,14 @@ metadata:
name: truenas-csi
namespace: truenas-csi
resources:
- node-service-account.yaml
- node-binding.yaml
- node-deamonset.yaml
- config.yaml
- namespace.yaml
- controller-deployment.yaml
- controller-cluster-role.yaml
- controller-binding.yaml
- node-cluster-role.yaml
- CSIDriver.yaml
- controller-service-account.yaml

12
infrastructure/hydra/truenas-csi/node-binding.yaml Normal file
View File

@@ -0,0 +1,12 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: truenas-csi-node-role-binding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: truenas-csi-node-role
subjects:
- kind: ServiceAccount
name: truenas-csi-node-sa
namespace: truenas-csi

27
infrastructure/hydra/truenas-csi/node-cluster-role.yaml Normal file
View File

@@ -0,0 +1,27 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: truenas-csi-node-role
rules:
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- list
- watch
- apiGroups:
- storage.k8s.io
resources:
- volumeattachments
verbs:
- get
- list
- watch

170
infrastructure/hydra/truenas-csi/node-deamonset.yaml Normal file
View File

@@ -0,0 +1,170 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
labels:
app: truenas-csi-node
name: truenas-csi-node
namespace: truenas-csi
spec:
selector:
matchLabels:
app: truenas-csi-node
template:
metadata:
labels:
app: truenas-csi-node
spec:
containers:
- args:
- --endpoint=$(CSI_ENDPOINT)
- --node-id=$(NODE_ID)
- --mode=node
- --v=4
env:
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
- name: TRUENAS_API_KEY
valueFrom:
secretKeyRef:
key: api-key
name: truenas-csi-api-credentials
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: TRUENAS_DEFAULT_POOL
valueFrom:
configMapKeyRef:
key: defaultPool
name: truenas-csi
- name: TRUENAS_INSECURE_SKIP_VERIFY
valueFrom:
configMapKeyRef:
key: truenasInsecure
name: truenas-csi
- name: TRUENAS_ISCSI_IQN_BASE
valueFrom:
configMapKeyRef:
key: iscsiIQNBase
name: truenas-csi
- name: TRUENAS_ISCSI_PORTAL
valueFrom:
configMapKeyRef:
key: iscsiPortal
name: truenas-csi
- name: TRUENAS_NFS_SERVER
valueFrom:
configMapKeyRef:
key: nfsServer
name: truenas-csi
- name: TRUENAS_URL
valueFrom:
configMapKeyRef:
key: truenasURL
name: truenas-csi
image: ghcr.io/truenas/truenas-csi:latest
imagePullPolicy: IfNotPresent
lifecycle:
postStart:
exec:
command:
- /bin/sh
- -c
- mkdir -p /run/lock/iscsi && mv /usr/sbin/iscsiadm /usr/sbin/iscsiadm.orig
2>/dev/null; printf '#!/bin/sh\nnsenter --mount=/host/proc/1/ns/mnt
-- /usr/sbin/iscsiadm "$@"\n' > /usr/sbin/iscsiadm && chmod +x /usr/sbin/iscsiadm
livenessProbe:
failureThreshold: 5
httpGet:
path: /healthz
port: 9808
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 3
name: csi-node
resources: {}
securityContext:
privileged: true
volumeMounts:
- mountPath: /csi
name: plugin-dir
- mountPath: /var/lib/kubelet
name: kubelet-dir
- mountPath: /dev
name: device-dir
- mountPath: /etc/iscsi
mountPropagation: Bidirectional
name: iscsi-dir
- mountPath: /var/lib/iscsi
mountPropagation: Bidirectional
name: iscsi-lib
- mountPath: /
mountPropagation: Bidirectional
name: host-root
- args:
- --csi-address=$(ADDRESS)
- --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
- --v=5
env:
- name: ADDRESS
value: /csi/csi.sock
- name: DRIVER_REG_SOCK_PATH
value: /var/lib/kubelet/plugins/csi.truenas.io/csi.sock
image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.16.0
name: csi-node-driver-registrar
resources: {}
volumeMounts:
- mountPath: /csi
name: plugin-dir
- mountPath: /registration
name: registration-dir
- args:
- --csi-address=/csi/csi.sock
- --health-port=9808
image: registry.k8s.io/sig-storage/livenessprobe:v2.18.0
name: liveness-probe
resources: {}
volumeMounts:
- mountPath: /csi
name: plugin-dir
hostIPC: true
hostNetwork: true
hostPID: true
priorityClassName: system-node-critical
serviceAccountName: truenas-csi-node-sa
tolerations:
- operator: Exists
volumes:
- hostPath:
path: /var/lib/kubelet/plugins_registry/
type: DirectoryOrCreate
name: registration-dir
- hostPath:
path: /var/lib/kubelet/plugins/csi.truenas.io/
type: DirectoryOrCreate
name: plugin-dir
- hostPath:
path: /var/lib/kubelet
type: Directory
name: kubelet-dir
- hostPath:
path: /dev
name: device-dir
- hostPath:
path: /etc/iscsi
type: Directory
name: iscsi-dir
- hostPath:
path: /var/lib/iscsi
type: DirectoryOrCreate
name: iscsi-lib
- hostPath:
path: /
type: Directory
name: host-root
updateStrategy: {}
status:
currentNumberScheduled: 0
desiredNumberScheduled: 0
numberMisscheduled: 0
numberReady: 0

5
infrastructure/hydra/truenas-csi/node-service-account.yaml Normal file
View File

@@ -0,0 +1,5 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: truenas-csi-node-sa
namespace: truenas-csi

3
pkg/flux/flux.go
View File

@@ -3,6 +3,7 @@ package flux
import (
"time"
"danicos.dev/daniel/go-kube/pkg/flux"
"danicos.dev/daniel/go-kube/pkg/kube"
"danicos.dev/daniel/go-kube/pkg/stack"
"danicos.dev/daniel/homelab/pkg/root"
@@ -44,7 +45,7 @@ func kuztomization(meta kube.Metadata, path string) kz.Kustomization {
Path: path,
Prune: true,
}
return kube.NewFluxKustomization(meta, spec)
return flux.NewFluxKustomization(meta, spec)
}
func durMin(d int64) meta.Duration {

7
pkg/longhorn/longhorn.go
View File

@@ -3,6 +3,7 @@ package longhorn
import (
"time"
"danicos.dev/daniel/go-kube/pkg/flux"
"danicos.dev/daniel/go-kube/pkg/kube"
"danicos.dev/daniel/go-kube/pkg/stack"
"danicos.dev/daniel/homelab/pkg/root"
@@ -32,7 +33,7 @@ func LonghornHelmSource() source.HelmRepository {
Interval: durHour(root.FLUX_HELM_MONITORING_INTERVAL),
URL: root.HELM_LONGHORN_URL,
}
return kube.NewFluxHelmRepositorySource(meta, spec)
return flux.NewFluxHelmRepositorySource(meta, spec)
}
func LonghornHelmRelease() helm.HelmRelease {
@@ -45,7 +46,7 @@ func LonghornHelmRelease() helm.HelmRelease {
Version: root.HELM_LONGHORN_CHART_VERSION,
Interval: &interval,
SourceRef: helm.CrossNamespaceObjectReference{
Kind: kube.FluxHelmRepositoryMeta.Kind,
Kind: flux.MetaHelmRepository.Kind,
Name: meta.Meta().Name,
Namespace: Namespace.Name,
},
@@ -58,7 +59,7 @@ func LonghornHelmRelease() helm.HelmRelease {
CRDs: helm.CreateReplace,
},
}
return kube.NewFluxHelmRelease(meta, spec)
return flux.NewFluxHelmRelease(meta, spec)
}
func durHour(d int64) metav1.Duration {

7
pkg/monitoring/monitoring.go
View File

@@ -4,6 +4,7 @@ import (
"encoding/json"
"time"
"danicos.dev/daniel/go-kube/pkg/flux"
"danicos.dev/daniel/go-kube/pkg/kube"
"danicos.dev/daniel/go-kube/pkg/stack"
"danicos.dev/daniel/homelab/pkg/root"
@@ -36,7 +37,7 @@ func PrometheusHelmSource() source.HelmRepository {
Interval: durHour(root.FLUX_HELM_MONITORING_INTERVAL),
URL: root.HELM_PROMETHEUS_URL,
}
return kube.NewFluxHelmRepositorySource(meta, spec)
return flux.NewFluxHelmRepositorySource(meta, spec)
}
func PrometheusRelease() helm.HelmRelease {
@@ -65,7 +66,7 @@ func PrometheusRelease() helm.HelmRelease {
Version: root.HELM_PROMETHEUS_CHART_VERSION,
Interval: &interval,
SourceRef: helm.CrossNamespaceObjectReference{
Kind: kube.FluxHelmRepositoryMeta.Kind,
Kind: flux.MetaHelmRepository.Kind,
Name: meta.Meta().Name,
Namespace: Namespace.Name,
},
@@ -88,7 +89,7 @@ func PrometheusRelease() helm.HelmRelease {
},
Values: &apiextensionsv1.JSON{Raw: raw},
}
return kube.NewFluxHelmRelease(meta, spec)
return flux.NewFluxHelmRelease(meta, spec)
}
func durHour(d int64) metav1.Duration {

9
pkg/root/services.go
View File

@@ -16,6 +16,9 @@ var Linkding = Service{
SecurityContextID: 33, // www-data user, group and FS ID
}
var Longhorn = "longhorn"
var Monitoring = "monitoring"
var TrueNAS_CSI = "truenas-csi"
var (
Longhorn = "longhorn"
Monitoring = "monitoring"
TrueNAS_CSI = "truenas-csi"
TrueNASURL = "apex-truenas.orca-uaru.ts.net"
)

39
pkg/truenas/rbac.go Normal file
View File

@@ -0,0 +1,39 @@
package truenas
import (
"danicos.dev/daniel/go-kube/pkg/kube"
"danicos.dev/daniel/homelab/pkg/root"
rbac "k8s.io/api/rbac/v1"
)
func controllerClusterRole() rbac.ClusterRole {
verbsReadUpdate := append(kube.VerbsRead(), kube.VerbsMutate()...)
rules := []rbac.PolicyRule{
kube.PolicyRule(kube.APIGroupCore, kube.ResourcePVs, kube.VerbsAll()),
kube.PolicyRule(kube.APIGroupCore, kube.ResourcePVCs, verbsReadUpdate),
kube.PolicyRule(kube.APIGroupCore, kube.ResourcePVCsStatus, kube.VerbsMutate()),
kube.PolicyRule(kube.APIGroupCore, kube.ResourceEvents, verbsReadUpdate),
kube.PolicyRule(kube.APIGroupCore, kube.ResourceNodes, kube.VerbsRead()),
kube.PolicyRule(kube.APIGroupCore, kube.ResourcePods, kube.VerbsRead()),
kube.PolicyRule(kube.APIGroupStorage, kube.ResourceStorageClasses, kube.VerbsRead()),
kube.PolicyRule(kube.APIGroupStorage, kube.ResourceCSINodes, kube.VerbsRead()),
kube.PolicyRule(kube.APIGroupStorage, kube.ResourceVolumeAttachments, kube.VerbsAll()),
kube.PolicyRule(kube.APIGroupStorage, kube.ResourceVolumeAttachmentsStatus, []string{kube.VerbPatch}),
kube.PolicyRule(kube.APIGroupSnapshot, kube.ResourceVolumeSnapshots, verbsReadUpdate),
kube.PolicyRule(kube.APIGroupSnapshot, kube.ResourceVolumeSnapshotsStatus, kube.VerbsMutate()),
kube.PolicyRule(kube.APIGroupSnapshot, kube.ResourceVolumeSnapshotContents, kube.VerbsAll()),
kube.PolicyRule(kube.APIGroupSnapshot, kube.ResourceVolumeSnapshotContentsStatus, kube.VerbsMutate()),
kube.PolicyRule(kube.APIGroupSnapshot, kube.ResourceVolumeSnapshotClases, kube.VerbsRead()),
}
return kube.ClusterRole(root.TrueNAS_CSI+"-controller-role", rules)
}
func nodeClusterRole() rbac.ClusterRole {
rules := []rbac.PolicyRule{
kube.PolicyRule(kube.APIGroupCore, kube.ResourceNodes, []string{kube.VerbGet}),
kube.PolicyRule(kube.APIGroupCore, kube.ResourcePods, kube.VerbsRead()),
kube.PolicyRule(kube.APIGroupStorage, kube.ResourceVolumeAttachments, kube.VerbsRead()),
}
return kube.ClusterRole(root.TrueNAS_CSI+"-node-role", rules)
}

420
pkg/truenas/truenas.go
View File

@@ -1,24 +1,440 @@
package truenas
import (
"fmt"
"slices"
"strings"
"danicos.dev/daniel/go-kube/pkg/kube"
"danicos.dev/daniel/go-kube/pkg/stack"
"danicos.dev/daniel/homelab/pkg/root"
apps "k8s.io/api/apps/v1"
core "k8s.io/api/core/v1"
storage "k8s.io/api/storage/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
var meta kube.Metadata
var Namespace = kube.Namespace(root.TrueNAS_CSI)
var Secret = struct {
Name string
APIKey string
}{
Name: root.TrueNAS_CSI + "-api-credentials",
APIKey: "api-key",
}
var Config = struct {
TruenasURL string
TrueNASInsecure string
DefaultPool string
NFSServer string
ISCSIPortal string
ISCSIIQNBase string
}{
TruenasURL: "truenasURL",
TrueNASInsecure: "truenasInsecure",
DefaultPool: "defaultPool",
NFSServer: "nfsServer",
ISCSIPortal: "iscsiPortal",
ISCSIIQNBase: "iscsiIQNBase",
}
var (
// Origin: https://github.com/truenas/truenas-csi
Namespace core.Namespace
meta kube.Metadata
controllerSA core.ServiceAccount
nodeSA core.ServiceAccount
config core.ConfigMap
)
func init() {
Namespace = kube.Namespace(root.TrueNAS_CSI)
meta = kube.NewMetadata(root.TrueNAS_CSI, Namespace)
controllerSA = kube.ServiceAccount(root.TrueNAS_CSI+"-controller", Namespace)
nodeSA = kube.ServiceAccount(root.TrueNAS_CSI+"-node", Namespace)
split := strings.Split(root.TrueNASURL, ".")
slices.Reverse(split)
config = core.ConfigMap{
TypeMeta: kube.ConfigMapMeta,
ObjectMeta: meta.Meta(),
Data: map[string]string{
Config.TruenasURL: fmt.Sprintf("wss://%s/api/current", root.TrueNASURL),
Config.TrueNASInsecure: "true",
Config.DefaultPool: "datapool",
Config.NFSServer: root.TrueNASURL,
Config.ISCSIPortal: fmt.Sprintf("%s:3260", root.TrueNASURL),
Config.ISCSIIQNBase: fmt.Sprintf("iqn.%s", strings.Join(split, ".")),
},
}
}
func Stack() stack.Stack {
controllerRole := controllerClusterRole()
nodeRole := nodeClusterRole()
kz := kube.NewKuztomizedStack(
meta,
map[string]any{
"namespace": Namespace,
"controller-deployment": controllerDeployment(),
"controller-service-account": controllerSA,
"controller-cluster-role": controllerRole,
"controller-binding": kube.ClusterRoleBinding(controllerRole.Name+"-binding", controllerSA, controllerRole),
"node-service-account": nodeSA,
"node-cluster-role": nodeRole,
"node-binding": kube.ClusterRoleBinding(nodeRole.Name+"-binding", nodeSA, nodeRole),
"node-deamonset": nodeCSI(),
"CSIDriver": CSIDriver("csi.truenas.io"),
"config": config,
},
)
return kz.Stack(root.TrueNAS_CSI)
}
func controllerDeployment() apps.Deployment {
meta := kube.NewMetadata(root.TrueNAS_CSI+"-controller", Namespace)
vol := core.Volume{
Name: "socket-dir",
VolumeSource: core.VolumeSource{
EmptyDir: &core.EmptyDirVolumeSource{},
},
}
spec := core.PodSpec{
ServiceAccountName: controllerSA.Name,
Containers: []core.Container{
{
Name: "csi-controller",
Image: "ghcr.io/truenas/truenas-csi:latest",
ImagePullPolicy: core.PullIfNotPresent,
Args: []string{
"--endpoint=$(CSI_ENDPOINT)",
"--node-id=$(NODE_ID)",
"--mode=controller",
"--v=4",
},
Env: controllerEnv(),
VolumeMounts: []core.VolumeMount{{Name: vol.Name, MountPath: "/csi"}},
LivenessProbe: &core.Probe{
ProbeHandler: core.ProbeHandler{
HTTPGet: &core.HTTPGetAction{
Path: "/healthz",
Port: intstr.FromInt(9808),
},
},
InitialDelaySeconds: 10,
TimeoutSeconds: 3,
PeriodSeconds: 10,
FailureThreshold: 5,
},
},
{
Name: "csi-provisioner",
Image: "registry.k8s.io/sig-storage/csi-provisioner:v6.1.1",
Args: []string{
"--csi-address=$(ADDRESS)",
"--v=5",
"--feature-gates=Topology=true",
"--extra-create-metadata",
"--leader-election=true",
"--default-fstype=ext4",
"--timeout=60s",
},
Env: []core.EnvVar{{
Name: "ADDRESS",
Value: "/csi/csi.sock",
}},
VolumeMounts: []core.VolumeMount{{Name: vol.Name, MountPath: "/csi"}},
},
{
Name: "csi-attacher",
Image: "registry.k8s.io/sig-storage/csi-attacher:v4.11.0",
Args: []string{
"--csi-address=$(ADDRESS)",
"--v=5",
"--leader-election=true",
"--timeout=60s",
},
Env: []core.EnvVar{{
Name: "ADDRESS",
Value: "/csi/csi.sock",
}},
VolumeMounts: []core.VolumeMount{{Name: vol.Name, MountPath: "/csi"}},
},
{
Name: "csi-snapshotter",
Image: "registry.k8s.io/sig-storage/csi-snapshotter:v8.5.0",
Args: []string{
"--csi-address=$(ADDRESS)",
"--v=5",
"--leader-election=true",
"--timeout=60s",
},
Env: []core.EnvVar{{
Name: "ADDRESS",
Value: "/csi/csi.sock",
}},
VolumeMounts: []core.VolumeMount{{Name: vol.Name, MountPath: "/csi"}},
},
{
Name: "csi-resizer",
Image: "registry.k8s.io/sig-storage/csi-resizer:v2.1.0",
Args: []string{
"--csi-address=$(ADDRESS)",
"--v=5",
"--leader-election=true",
"--timeout=60s",
},
Env: []core.EnvVar{{
Name: "ADDRESS",
Value: "/csi/csi.sock",
}},
VolumeMounts: []core.VolumeMount{{Name: vol.Name, MountPath: "/csi"}},
},
{
Name: "liveness-probe",
Image: "registry.k8s.io/sig-storage/livenessprobe:v2.18.0",
Args: []string{
"--csi-address=/csi/csi.sock",
"--health-port=9808",
},
VolumeMounts: []core.VolumeMount{{Name: vol.Name, MountPath: "/csi"}},
},
},
Volumes: []core.Volume{vol},
}
return kube.NewDeployment(meta, spec)
}
func controllerEnv() []core.EnvVar {
envMapping := map[string]string{
"CSI_ENDPOINT": "unix:///csi/csi.sock",
}
secretMapping := map[string]string{
"TRUENAS_API_KEY": Secret.APIKey,
}
env1 := kube.NewEnvVarWithSecret(envMapping, secretMapping, Secret.Name)
envMapping = map[string]string{
"TRUENAS_URL": Config.TruenasURL,
"TRUENAS_DEFAULT_POOL": Config.DefaultPool,
"TRUENAS_NFS_SERVER": Config.NFSServer,
"TRUENAS_ISCSI_PORTAL": Config.ISCSIPortal,
"TRUENAS_ISCSI_IQN_BASE": Config.ISCSIIQNBase,
"TRUENAS_INSECURE_SKIP_VERIFY": Config.TrueNASInsecure,
}
env2 := kube.NewEnvVarWithConfig(envMapping, config)
nodeEnv := core.EnvVar{
Name: "NODE_ID", // value
ValueFrom: &core.EnvVarSource{
FieldRef: &core.ObjectFieldSelector{
FieldPath: "spec.nodeName",
},
},
}
env3 := append(env1, nodeEnv)
return append(env3, env2...)
}
func CSIDriver(name string) storage.CSIDriver {
fsGroupPolicy := storage.FileFSGroupPolicy
spec := storage.CSIDriverSpec{
AttachRequired: new(true),
PodInfoOnMount: new(true),
VolumeLifecycleModes: []storage.VolumeLifecycleMode{
storage.VolumeLifecycleEphemeral,
storage.VolumeLifecyclePersistent,
},
FSGroupPolicy: &fsGroupPolicy,
}
return kube.CSIDriver(name, spec)
}
func nodeCSI() apps.DaemonSet {
registrationDir := core.Volume{
Name: "registration-dir",
VolumeSource: core.VolumeSource{
HostPath: &core.HostPathVolumeSource{
Path: "/var/lib/kubelet/plugins_registry/",
Type: new(core.HostPathDirectoryOrCreate),
},
},
}
pluginDir := core.Volume{
Name: "plugin-dir",
VolumeSource: core.VolumeSource{
HostPath: &core.HostPathVolumeSource{
Path: "/var/lib/kubelet/plugins/csi.truenas.io/",
Type: new(core.HostPathDirectoryOrCreate),
},
},
}
kubeletDir := core.Volume{
Name: "kubelet-dir",
VolumeSource: core.VolumeSource{
HostPath: &core.HostPathVolumeSource{
Path: "/var/lib/kubelet",
Type: new(core.HostPathDirectory),
},
},
}
deviceDir := core.Volume{
Name: "device-dir",
VolumeSource: core.VolumeSource{
HostPath: &core.HostPathVolumeSource{
Path: "/dev",
},
},
}
iscsiDir := core.Volume{
Name: "iscsi-dir",
VolumeSource: core.VolumeSource{
HostPath: &core.HostPathVolumeSource{
Path: "/etc/iscsi",
Type: new(core.HostPathDirectory),
},
},
}
iscsiLib := core.Volume{
Name: "iscsi-lib",
VolumeSource: core.VolumeSource{
HostPath: &core.HostPathVolumeSource{
Path: "/var/lib/iscsi",
Type: new(core.HostPathDirectoryOrCreate),
},
},
}
hostRoot := core.Volume{
Name: "host-root",
VolumeSource: core.VolumeSource{
HostPath: &core.HostPathVolumeSource{
Path: "/",
Type: new(core.HostPathDirectory),
},
},
}
meta := kube.NewMetadata(root.TrueNAS_CSI+"-node", Namespace)
podSpec := core.PodSpec{
ServiceAccountName: nodeSA.Name,
HostNetwork: true,
HostPID: true,
HostIPC: true,
PriorityClassName: "system-node-critical",
Tolerations: []core.Toleration{{
Operator: core.TolerationOpExists,
}},
Containers: []core.Container{
{
Name: "csi-node",
Image: "ghcr.io/truenas/truenas-csi:latest",
ImagePullPolicy: core.PullIfNotPresent,
Lifecycle: &core.Lifecycle{
PostStart: &core.LifecycleHandler{
Exec: &core.ExecAction{
Command: []string{
"/bin/sh",
"-c",
"mkdir -p /run/lock/iscsi && mv /usr/sbin/iscsiadm /usr/sbin/iscsiadm.orig 2>/dev/null; printf '#!/bin/sh\\nnsenter --mount=/host/proc/1/ns/mnt -- /usr/sbin/iscsiadm \"$@\"\\n' > /usr/sbin/iscsiadm && chmod +x /usr/sbin/iscsiadm",
},
},
},
},
SecurityContext: &core.SecurityContext{Privileged: new(true)},
Args: []string{
"--endpoint=$(CSI_ENDPOINT)",
"--node-id=$(NODE_ID)",
"--mode=node",
"--v=4",
},
Env: controllerEnv(),
VolumeMounts: []core.VolumeMount{
{
Name: pluginDir.Name,
MountPath: "/csi",
},
{
Name: kubeletDir.Name,
MountPath: "/var/lib/kubelet",
},
{
Name: deviceDir.Name,
MountPath: "/dev",
},
{
Name: iscsiDir.Name,
MountPath: "/etc/iscsi",
MountPropagation: new(core.MountPropagationBidirectional),
},
{
Name: iscsiLib.Name,
MountPath: "/var/lib/iscsi",
MountPropagation: new(core.MountPropagationBidirectional),
},
{
Name: hostRoot.Name,
MountPath: "/",
MountPropagation: new(core.MountPropagationBidirectional),
},
},
LivenessProbe: &core.Probe{
ProbeHandler: core.ProbeHandler{
HTTPGet: &core.HTTPGetAction{
Path: "/healthz",
Port: intstr.FromInt(9808),
},
},
InitialDelaySeconds: 10,
TimeoutSeconds: 3,
PeriodSeconds: 10,
FailureThreshold: 5,
},
},
{
Name: "csi-node-driver-registrar",
Image: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.16.0",
Args: []string{
"--csi-address=$(ADDRESS)",
"--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)",
"--v=5",
},
Env: []core.EnvVar{
{
Name: "ADDRESS",
Value: "/csi/csi.sock",
},
{
Name: "DRIVER_REG_SOCK_PATH",
Value: "/var/lib/kubelet/plugins/csi.truenas.io/csi.sock",
},
},
VolumeMounts: []core.VolumeMount{
{
Name: pluginDir.Name,
MountPath: "/csi",
},
{
Name: registrationDir.Name,
MountPath: "/registration",
},
},
},
{
Name: "liveness-probe",
Image: "registry.k8s.io/sig-storage/livenessprobe:v2.18.0",
Args: []string{
"--csi-address=/csi/csi.sock",
"--health-port=9808",
},
VolumeMounts: []core.VolumeMount{{Name: pluginDir.Name, MountPath: "/csi"}},
},
},
Volumes: []core.Volume{
registrationDir,
pluginDir,
kubeletDir,
deviceDir,
iscsiDir,
iscsiLib,
hostRoot,
},
}
return kube.NewDeamonSet(meta, podSpec)
}