troubleshooting deployment errors

k8s/authentik/authentik.yaml

@@ -296,6 +296,8 @@ spec:
                 secretKeyRef:
                   name: authentik-secret
                   key: AUTHENTIK_SECRET_KEY
+            - name: AUTHENTIK_WORKER_CONCURRENCY
+              value: "2"
           resources:
             requests:
               memory: "512Mi"

k8s/elasticsearch/elasticsearch.yaml

@@ -18,12 +18,12 @@ data:
     xpack.security.transport.ssl.enabled: true
     xpack.security.transport.ssl.verification_mode: certificate
     xpack.security.transport.ssl.keystore.path: /usr/share/elasticsearch/config/certs/elasticsearch.p12
-    xpack.security.transport.ssl.truststore.path: /usr/share/elasticsearch/config/certs/elasticsearch.p12
+    xpack.security.transport.ssl.certificate_authorities: ["/usr/share/elasticsearch/config/certs/ca.crt"]
 
     # HTTP TLS (client-to-node)
     xpack.security.http.ssl.enabled: true
     xpack.security.http.ssl.keystore.path: /usr/share/elasticsearch/config/certs/http.p12
-    xpack.security.http.ssl.truststore.path: /usr/share/elasticsearch/config/certs/http.p12
+    xpack.security.http.ssl.certificate_authorities: ["/usr/share/elasticsearch/config/certs/ca.crt"]
 
     # Token service for Kibana
     xpack.security.authc.token.enabled: true
@@ -60,6 +60,7 @@ type: Opaque
 stringData:
   ELASTIC_PASSWORD: "ElasticP@ss2024!"
   ES_KEYSTORE_PASS: "changeit"
+  OIDC_CLIENT_SECRET: "kibana-client-secret-2024"
 ---
 apiVersion: apps/v1
 kind: StatefulSet
@@ -93,6 +94,40 @@ spec:
           command: ['sysctl', '-w', 'vm.max_map_count=262144']
           securityContext:
             privileged: true
+        - name: setup-config
+          image: docker.elastic.co/elasticsearch/elasticsearch:8.17.0
+          command:
+            - sh
+            - -c
+            - |
+              # Copy default config files into writable config dir
+              cp -r /usr/share/elasticsearch/config/* /writable-config/
+              # Overlay with our custom elasticsearch.yml
+              cp /custom-config/elasticsearch.yml /writable-config/elasticsearch.yml
+              # Create keystore and add PKCS12 secure passwords
+              export ES_PATH_CONF=/writable-config
+              /usr/share/elasticsearch/bin/elasticsearch-keystore create
+              echo "$KEYSTORE_PASSWORD" | /usr/share/elasticsearch/bin/elasticsearch-keystore add -x xpack.security.transport.ssl.keystore.secure_password
+              echo "$KEYSTORE_PASSWORD" | /usr/share/elasticsearch/bin/elasticsearch-keystore add -x xpack.security.http.ssl.keystore.secure_password
+              echo "$OIDC_CLIENT_SECRET" | /usr/share/elasticsearch/bin/elasticsearch-keystore add -x xpack.security.authc.realms.oidc.authentik.rp.client_secret
+              echo "Config directory assembled successfully"
+              ls -la /writable-config/
+          env:
+            - name: KEYSTORE_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: elasticsearch-credentials
+                  key: ES_KEYSTORE_PASS
+            - name: OIDC_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: elasticsearch-credentials
+                  key: OIDC_CLIENT_SECRET
+          volumeMounts:
+            - name: es-config-dir
+              mountPath: /writable-config
+            - name: es-custom-config
+              mountPath: /custom-config
       containers:
         - name: elasticsearch
           image: docker.elastic.co/elasticsearch/elasticsearch:8.17.0
@@ -109,32 +144,11 @@ spec:
                   key: ELASTIC_PASSWORD
             - name: ES_JAVA_OPTS
               value: "-Xms1g -Xmx1g"
-            - name: xpack.security.transport.ssl.keystore.secure_password
-              valueFrom:
-                secretKeyRef:
-                  name: elasticsearch-credentials
-                  key: ES_KEYSTORE_PASS
-            - name: xpack.security.transport.ssl.truststore.secure_password
-              valueFrom:
-                secretKeyRef:
-                  name: elasticsearch-credentials
-                  key: ES_KEYSTORE_PASS
-            - name: xpack.security.http.ssl.keystore.secure_password
-              valueFrom:
-                secretKeyRef:
-                  name: elasticsearch-credentials
-                  key: ES_KEYSTORE_PASS
-            - name: xpack.security.http.ssl.truststore.secure_password
-              valueFrom:
-                secretKeyRef:
-                  name: elasticsearch-credentials
-                  key: ES_KEYSTORE_PASS
           volumeMounts:
             - name: es-data
               mountPath: /usr/share/elasticsearch/data
-            - name: es-config
+            - name: es-config-dir
-              mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
+              mountPath: /usr/share/elasticsearch/config
-              subPath: elasticsearch.yml
             - name: es-certs
               mountPath: /usr/share/elasticsearch/config/certs
               readOnly: true
@@ -158,7 +172,9 @@ spec:
             periodSeconds: 15
             timeoutSeconds: 10
       volumes:
-        - name: es-config
+        - name: es-config-dir
+          emptyDir: {}
+        - name: es-custom-config
           configMap:
             name: elasticsearch-config
         - name: es-certs

k8s/mysql/deployment.yaml

@@ -17,6 +17,24 @@ spec:
       labels:
         app: mysql
     spec:
+      initContainers:
+        - name: cleanup-stale-data
+          image: busybox:1.36
+          command:
+            - sh
+            - -c
+            - |
+              # If data dir has files but MySQL was never fully initialized
+              # (no 'mysql.ibd' system tablespace), clean up so init can run
+              if [ -d /var/lib/mysql ] && [ "$(ls -A /var/lib/mysql)" ] && [ ! -f /var/lib/mysql/mysql.ibd ]; then
+                echo "Detected partially initialized data directory — cleaning up"
+                rm -rf /var/lib/mysql/*
+              else
+                echo "Data directory is clean or already initialized — no action needed"
+              fi
+          volumeMounts:
+            - name: mysql-data
+              mountPath: /var/lib/mysql
       containers:
         - name: mysql
           image: mysql:8.4