daniel/homelab
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finish TrueNAS CSI Driver

Browse Source
This commit is contained in:
Daniel Cosme
2026-04-29 19:15:55 -04:00
parent bc2747bef8
commit a388ac53e2
22 changed files with 1064 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
pkg/flux/flux.go
View File

@@ -3,6 +3,7 @@ package flux
import (
"time"
"danicos.dev/daniel/go-kube/pkg/flux"
"danicos.dev/daniel/go-kube/pkg/kube"
"danicos.dev/daniel/go-kube/pkg/stack"
"danicos.dev/daniel/homelab/pkg/root"
@@ -44,7 +45,7 @@ func kuztomization(meta kube.Metadata, path string) kz.Kustomization {
Path: path,
Prune: true,
}
return kube.NewFluxKustomization(meta, spec)
return flux.NewFluxKustomization(meta, spec)
}
func durMin(d int64) meta.Duration {

7
pkg/longhorn/longhorn.go
View File

@@ -3,6 +3,7 @@ package longhorn
import (
"time"
"danicos.dev/daniel/go-kube/pkg/flux"
"danicos.dev/daniel/go-kube/pkg/kube"
"danicos.dev/daniel/go-kube/pkg/stack"
"danicos.dev/daniel/homelab/pkg/root"
@@ -32,7 +33,7 @@ func LonghornHelmSource() source.HelmRepository {
Interval: durHour(root.FLUX_HELM_MONITORING_INTERVAL),
URL: root.HELM_LONGHORN_URL,
}
return kube.NewFluxHelmRepositorySource(meta, spec)
return flux.NewFluxHelmRepositorySource(meta, spec)
}
func LonghornHelmRelease() helm.HelmRelease {
@@ -45,7 +46,7 @@ func LonghornHelmRelease() helm.HelmRelease {
Version: root.HELM_LONGHORN_CHART_VERSION,
Interval: &interval,
SourceRef: helm.CrossNamespaceObjectReference{
Kind: kube.FluxHelmRepositoryMeta.Kind,
Kind: flux.MetaHelmRepository.Kind,
Name: meta.Meta().Name,
Namespace: Namespace.Name,
},
@@ -58,7 +59,7 @@ func LonghornHelmRelease() helm.HelmRelease {
CRDs: helm.CreateReplace,
},
}
return kube.NewFluxHelmRelease(meta, spec)
return flux.NewFluxHelmRelease(meta, spec)
}
func durHour(d int64) metav1.Duration {

7
pkg/monitoring/monitoring.go
View File

@@ -4,6 +4,7 @@ import (
"encoding/json"
"time"
"danicos.dev/daniel/go-kube/pkg/flux"
"danicos.dev/daniel/go-kube/pkg/kube"
"danicos.dev/daniel/go-kube/pkg/stack"
"danicos.dev/daniel/homelab/pkg/root"
@@ -36,7 +37,7 @@ func PrometheusHelmSource() source.HelmRepository {
Interval: durHour(root.FLUX_HELM_MONITORING_INTERVAL),
URL: root.HELM_PROMETHEUS_URL,
}
return kube.NewFluxHelmRepositorySource(meta, spec)
return flux.NewFluxHelmRepositorySource(meta, spec)
}
func PrometheusRelease() helm.HelmRelease {
@@ -65,7 +66,7 @@ func PrometheusRelease() helm.HelmRelease {
Version: root.HELM_PROMETHEUS_CHART_VERSION,
Interval: &interval,
SourceRef: helm.CrossNamespaceObjectReference{
Kind: kube.FluxHelmRepositoryMeta.Kind,
Kind: flux.MetaHelmRepository.Kind,
Name: meta.Meta().Name,
Namespace: Namespace.Name,
},
@@ -88,7 +89,7 @@ func PrometheusRelease() helm.HelmRelease {
},
Values: &apiextensionsv1.JSON{Raw: raw},
}
return kube.NewFluxHelmRelease(meta, spec)
return flux.NewFluxHelmRelease(meta, spec)
}
func durHour(d int64) metav1.Duration {

9
pkg/root/services.go
View File

@@ -16,6 +16,9 @@ var Linkding = Service{
SecurityContextID: 33, // www-data user, group and FS ID
}
var Longhorn = "longhorn"
var Monitoring = "monitoring"
var TrueNAS_CSI = "truenas-csi"
var (
Longhorn = "longhorn"
Monitoring = "monitoring"
TrueNAS_CSI = "truenas-csi"
TrueNASURL = "apex-truenas.orca-uaru.ts.net"
)

39
pkg/truenas/rbac.go Normal file
View File

@@ -0,0 +1,39 @@
package truenas
import (
"danicos.dev/daniel/go-kube/pkg/kube"
"danicos.dev/daniel/homelab/pkg/root"
rbac "k8s.io/api/rbac/v1"
)
func controllerClusterRole() rbac.ClusterRole {
verbsReadUpdate := append(kube.VerbsRead(), kube.VerbsMutate()...)
rules := []rbac.PolicyRule{
kube.PolicyRule(kube.APIGroupCore, kube.ResourcePVs, kube.VerbsAll()),
kube.PolicyRule(kube.APIGroupCore, kube.ResourcePVCs, verbsReadUpdate),
kube.PolicyRule(kube.APIGroupCore, kube.ResourcePVCsStatus, kube.VerbsMutate()),
kube.PolicyRule(kube.APIGroupCore, kube.ResourceEvents, verbsReadUpdate),
kube.PolicyRule(kube.APIGroupCore, kube.ResourceNodes, kube.VerbsRead()),
kube.PolicyRule(kube.APIGroupCore, kube.ResourcePods, kube.VerbsRead()),
kube.PolicyRule(kube.APIGroupStorage, kube.ResourceStorageClasses, kube.VerbsRead()),
kube.PolicyRule(kube.APIGroupStorage, kube.ResourceCSINodes, kube.VerbsRead()),
kube.PolicyRule(kube.APIGroupStorage, kube.ResourceVolumeAttachments, kube.VerbsAll()),
kube.PolicyRule(kube.APIGroupStorage, kube.ResourceVolumeAttachmentsStatus, []string{kube.VerbPatch}),
kube.PolicyRule(kube.APIGroupSnapshot, kube.ResourceVolumeSnapshots, verbsReadUpdate),
kube.PolicyRule(kube.APIGroupSnapshot, kube.ResourceVolumeSnapshotsStatus, kube.VerbsMutate()),
kube.PolicyRule(kube.APIGroupSnapshot, kube.ResourceVolumeSnapshotContents, kube.VerbsAll()),
kube.PolicyRule(kube.APIGroupSnapshot, kube.ResourceVolumeSnapshotContentsStatus, kube.VerbsMutate()),
kube.PolicyRule(kube.APIGroupSnapshot, kube.ResourceVolumeSnapshotClases, kube.VerbsRead()),
}
return kube.ClusterRole(root.TrueNAS_CSI+"-controller-role", rules)
}
func nodeClusterRole() rbac.ClusterRole {
rules := []rbac.PolicyRule{
kube.PolicyRule(kube.APIGroupCore, kube.ResourceNodes, []string{kube.VerbGet}),
kube.PolicyRule(kube.APIGroupCore, kube.ResourcePods, kube.VerbsRead()),
kube.PolicyRule(kube.APIGroupStorage, kube.ResourceVolumeAttachments, kube.VerbsRead()),
}
return kube.ClusterRole(root.TrueNAS_CSI+"-node-role", rules)
}

422
pkg/truenas/truenas.go
View File

@@ -1,24 +1,440 @@
package truenas
import (
"fmt"
"slices"
"strings"
"danicos.dev/daniel/go-kube/pkg/kube"
"danicos.dev/daniel/go-kube/pkg/stack"
"danicos.dev/daniel/homelab/pkg/root"
apps "k8s.io/api/apps/v1"
core "k8s.io/api/core/v1"
storage "k8s.io/api/storage/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
var meta kube.Metadata
var Namespace = kube.Namespace(root.TrueNAS_CSI)
var Secret = struct {
Name string
APIKey string
}{
Name: root.TrueNAS_CSI + "-api-credentials",
APIKey: "api-key",
}
var Config = struct {
TruenasURL string
TrueNASInsecure string
DefaultPool string
NFSServer string
ISCSIPortal string
ISCSIIQNBase string
}{
TruenasURL: "truenasURL",
TrueNASInsecure: "truenasInsecure",
DefaultPool: "defaultPool",
NFSServer: "nfsServer",
ISCSIPortal: "iscsiPortal",
ISCSIIQNBase: "iscsiIQNBase",
}
var (
// Origin: https://github.com/truenas/truenas-csi
Namespace core.Namespace
meta kube.Metadata
controllerSA core.ServiceAccount
nodeSA core.ServiceAccount
config core.ConfigMap
)
func init() {
Namespace = kube.Namespace(root.TrueNAS_CSI)
meta = kube.NewMetadata(root.TrueNAS_CSI, Namespace)
controllerSA = kube.ServiceAccount(root.TrueNAS_CSI+"-controller", Namespace)
nodeSA = kube.ServiceAccount(root.TrueNAS_CSI+"-node", Namespace)
split := strings.Split(root.TrueNASURL, ".")
slices.Reverse(split)
config = core.ConfigMap{
TypeMeta: kube.ConfigMapMeta,
ObjectMeta: meta.Meta(),
Data: map[string]string{
Config.TruenasURL: fmt.Sprintf("wss://%s/api/current", root.TrueNASURL),
Config.TrueNASInsecure: "true",
Config.DefaultPool: "datapool",
Config.NFSServer: root.TrueNASURL,
Config.ISCSIPortal: fmt.Sprintf("%s:3260", root.TrueNASURL),
Config.ISCSIIQNBase: fmt.Sprintf("iqn.%s", strings.Join(split, ".")),
},
}
}
func Stack() stack.Stack {
controllerRole := controllerClusterRole()
nodeRole := nodeClusterRole()
kz := kube.NewKuztomizedStack(
meta,
map[string]any{
"namespace": Namespace,
"namespace": Namespace,
"controller-deployment": controllerDeployment(),
"controller-service-account": controllerSA,
"controller-cluster-role": controllerRole,
"controller-binding": kube.ClusterRoleBinding(controllerRole.Name+"-binding", controllerSA, controllerRole),
"node-service-account": nodeSA,
"node-cluster-role": nodeRole,
"node-binding": kube.ClusterRoleBinding(nodeRole.Name+"-binding", nodeSA, nodeRole),
"node-deamonset": nodeCSI(),
"CSIDriver": CSIDriver("csi.truenas.io"),
"config": config,
},
)
return kz.Stack(root.TrueNAS_CSI)
}
func controllerDeployment() apps.Deployment {
meta := kube.NewMetadata(root.TrueNAS_CSI+"-controller", Namespace)
vol := core.Volume{
Name: "socket-dir",
VolumeSource: core.VolumeSource{
EmptyDir: &core.EmptyDirVolumeSource{},
},
}
spec := core.PodSpec{
ServiceAccountName: controllerSA.Name,
Containers: []core.Container{
{
Name: "csi-controller",
Image: "ghcr.io/truenas/truenas-csi:latest",
ImagePullPolicy: core.PullIfNotPresent,
Args: []string{
"--endpoint=$(CSI_ENDPOINT)",
"--node-id=$(NODE_ID)",
"--mode=controller",
"--v=4",
},
Env: controllerEnv(),
VolumeMounts: []core.VolumeMount{{Name: vol.Name, MountPath: "/csi"}},
LivenessProbe: &core.Probe{
ProbeHandler: core.ProbeHandler{
HTTPGet: &core.HTTPGetAction{
Path: "/healthz",
Port: intstr.FromInt(9808),
},
},
InitialDelaySeconds: 10,
TimeoutSeconds: 3,
PeriodSeconds: 10,
FailureThreshold: 5,
},
},
{
Name: "csi-provisioner",
Image: "registry.k8s.io/sig-storage/csi-provisioner:v6.1.1",
Args: []string{
"--csi-address=$(ADDRESS)",
"--v=5",
"--feature-gates=Topology=true",
"--extra-create-metadata",
"--leader-election=true",
"--default-fstype=ext4",
"--timeout=60s",
},
Env: []core.EnvVar{{
Name: "ADDRESS",
Value: "/csi/csi.sock",
}},
VolumeMounts: []core.VolumeMount{{Name: vol.Name, MountPath: "/csi"}},
},
{
Name: "csi-attacher",
Image: "registry.k8s.io/sig-storage/csi-attacher:v4.11.0",
Args: []string{
"--csi-address=$(ADDRESS)",
"--v=5",
"--leader-election=true",
"--timeout=60s",
},
Env: []core.EnvVar{{
Name: "ADDRESS",
Value: "/csi/csi.sock",
}},
VolumeMounts: []core.VolumeMount{{Name: vol.Name, MountPath: "/csi"}},
},
{
Name: "csi-snapshotter",
Image: "registry.k8s.io/sig-storage/csi-snapshotter:v8.5.0",
Args: []string{
"--csi-address=$(ADDRESS)",
"--v=5",
"--leader-election=true",
"--timeout=60s",
},
Env: []core.EnvVar{{
Name: "ADDRESS",
Value: "/csi/csi.sock",
}},
VolumeMounts: []core.VolumeMount{{Name: vol.Name, MountPath: "/csi"}},
},
{
Name: "csi-resizer",
Image: "registry.k8s.io/sig-storage/csi-resizer:v2.1.0",
Args: []string{
"--csi-address=$(ADDRESS)",
"--v=5",
"--leader-election=true",
"--timeout=60s",
},
Env: []core.EnvVar{{
Name: "ADDRESS",
Value: "/csi/csi.sock",
}},
VolumeMounts: []core.VolumeMount{{Name: vol.Name, MountPath: "/csi"}},
},
{
Name: "liveness-probe",
Image: "registry.k8s.io/sig-storage/livenessprobe:v2.18.0",
Args: []string{
"--csi-address=/csi/csi.sock",
"--health-port=9808",
},
VolumeMounts: []core.VolumeMount{{Name: vol.Name, MountPath: "/csi"}},
},
},
Volumes: []core.Volume{vol},
}
return kube.NewDeployment(meta, spec)
}
func controllerEnv() []core.EnvVar {
envMapping := map[string]string{
"CSI_ENDPOINT": "unix:///csi/csi.sock",
}
secretMapping := map[string]string{
"TRUENAS_API_KEY": Secret.APIKey,
}
env1 := kube.NewEnvVarWithSecret(envMapping, secretMapping, Secret.Name)
envMapping = map[string]string{
"TRUENAS_URL": Config.TruenasURL,
"TRUENAS_DEFAULT_POOL": Config.DefaultPool,
"TRUENAS_NFS_SERVER": Config.NFSServer,
"TRUENAS_ISCSI_PORTAL": Config.ISCSIPortal,
"TRUENAS_ISCSI_IQN_BASE": Config.ISCSIIQNBase,
"TRUENAS_INSECURE_SKIP_VERIFY": Config.TrueNASInsecure,
}
env2 := kube.NewEnvVarWithConfig(envMapping, config)
nodeEnv := core.EnvVar{
Name: "NODE_ID", // value
ValueFrom: &core.EnvVarSource{
FieldRef: &core.ObjectFieldSelector{
FieldPath: "spec.nodeName",
},
},
}
env3 := append(env1, nodeEnv)
return append(env3, env2...)
}
func CSIDriver(name string) storage.CSIDriver {
fsGroupPolicy := storage.FileFSGroupPolicy
spec := storage.CSIDriverSpec{
AttachRequired: new(true),
PodInfoOnMount: new(true),
VolumeLifecycleModes: []storage.VolumeLifecycleMode{
storage.VolumeLifecycleEphemeral,
storage.VolumeLifecyclePersistent,
},
FSGroupPolicy: &fsGroupPolicy,
}
return kube.CSIDriver(name, spec)
}
func nodeCSI() apps.DaemonSet {
registrationDir := core.Volume{
Name: "registration-dir",
VolumeSource: core.VolumeSource{
HostPath: &core.HostPathVolumeSource{
Path: "/var/lib/kubelet/plugins_registry/",
Type: new(core.HostPathDirectoryOrCreate),
},
},
}
pluginDir := core.Volume{
Name: "plugin-dir",
VolumeSource: core.VolumeSource{
HostPath: &core.HostPathVolumeSource{
Path: "/var/lib/kubelet/plugins/csi.truenas.io/",
Type: new(core.HostPathDirectoryOrCreate),
},
},
}
kubeletDir := core.Volume{
Name: "kubelet-dir",
VolumeSource: core.VolumeSource{
HostPath: &core.HostPathVolumeSource{
Path: "/var/lib/kubelet",
Type: new(core.HostPathDirectory),
},
},
}
deviceDir := core.Volume{
Name: "device-dir",
VolumeSource: core.VolumeSource{
HostPath: &core.HostPathVolumeSource{
Path: "/dev",
},
},
}
iscsiDir := core.Volume{
Name: "iscsi-dir",
VolumeSource: core.VolumeSource{
HostPath: &core.HostPathVolumeSource{
Path: "/etc/iscsi",
Type: new(core.HostPathDirectory),
},
},
}
iscsiLib := core.Volume{
Name: "iscsi-lib",
VolumeSource: core.VolumeSource{
HostPath: &core.HostPathVolumeSource{
Path: "/var/lib/iscsi",
Type: new(core.HostPathDirectoryOrCreate),
},
},
}
hostRoot := core.Volume{
Name: "host-root",
VolumeSource: core.VolumeSource{
HostPath: &core.HostPathVolumeSource{
Path: "/",
Type: new(core.HostPathDirectory),
},
},
}
meta := kube.NewMetadata(root.TrueNAS_CSI+"-node", Namespace)
podSpec := core.PodSpec{
ServiceAccountName: nodeSA.Name,
HostNetwork: true,
HostPID: true,
HostIPC: true,
PriorityClassName: "system-node-critical",
Tolerations: []core.Toleration{{
Operator: core.TolerationOpExists,
}},
Containers: []core.Container{
{
Name: "csi-node",
Image: "ghcr.io/truenas/truenas-csi:latest",
ImagePullPolicy: core.PullIfNotPresent,
Lifecycle: &core.Lifecycle{
PostStart: &core.LifecycleHandler{
Exec: &core.ExecAction{
Command: []string{
"/bin/sh",
"-c",
"mkdir -p /run/lock/iscsi && mv /usr/sbin/iscsiadm /usr/sbin/iscsiadm.orig 2>/dev/null; printf '#!/bin/sh\\nnsenter --mount=/host/proc/1/ns/mnt -- /usr/sbin/iscsiadm \"$@\"\\n' > /usr/sbin/iscsiadm && chmod +x /usr/sbin/iscsiadm",
},
},
},
},
SecurityContext: &core.SecurityContext{Privileged: new(true)},
Args: []string{
"--endpoint=$(CSI_ENDPOINT)",
"--node-id=$(NODE_ID)",
"--mode=node",
"--v=4",
},
Env: controllerEnv(),
VolumeMounts: []core.VolumeMount{
{
Name: pluginDir.Name,
MountPath: "/csi",
},
{
Name: kubeletDir.Name,
MountPath: "/var/lib/kubelet",
},
{
Name: deviceDir.Name,
MountPath: "/dev",
},
{
Name: iscsiDir.Name,
MountPath: "/etc/iscsi",
MountPropagation: new(core.MountPropagationBidirectional),
},
{
Name: iscsiLib.Name,
MountPath: "/var/lib/iscsi",
MountPropagation: new(core.MountPropagationBidirectional),
},
{
Name: hostRoot.Name,
MountPath: "/",
MountPropagation: new(core.MountPropagationBidirectional),
},
},
LivenessProbe: &core.Probe{
ProbeHandler: core.ProbeHandler{
HTTPGet: &core.HTTPGetAction{
Path: "/healthz",
Port: intstr.FromInt(9808),
},
},
InitialDelaySeconds: 10,
TimeoutSeconds: 3,
PeriodSeconds: 10,
FailureThreshold: 5,
},
},
{
Name: "csi-node-driver-registrar",
Image: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.16.0",
Args: []string{
"--csi-address=$(ADDRESS)",
"--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)",
"--v=5",
},
Env: []core.EnvVar{
{
Name: "ADDRESS",
Value: "/csi/csi.sock",
},
{
Name: "DRIVER_REG_SOCK_PATH",
Value: "/var/lib/kubelet/plugins/csi.truenas.io/csi.sock",
},
},
VolumeMounts: []core.VolumeMount{
{
Name: pluginDir.Name,
MountPath: "/csi",
},
{
Name: registrationDir.Name,
MountPath: "/registration",
},
},
},
{
Name: "liveness-probe",
Image: "registry.k8s.io/sig-storage/livenessprobe:v2.18.0",
Args: []string{
"--csi-address=/csi/csi.sock",
"--health-port=9808",
},
VolumeMounts: []core.VolumeMount{{Name: pluginDir.Name, MountPath: "/csi"}},
},
},
Volumes: []core.Volume{
registrationDir,
pluginDir,
kubeletDir,
deviceDir,
iscsiDir,
iscsiLib,
hostRoot,
},
}
return kube.NewDeamonSet(meta, podSpec)
}