Kubernetes cluster architecture and configuration for the Lilith platform using Amazon EKS.
Table of Contents#
- Cluster Overview
- Node Groups
- Cluster Add-ons
- RBAC Configuration
- Namespaces
- Resource Management
- Security
- Observability
Cluster Overview#
Architecture#
text
┌─────────────────────────────────────────────────────────────────────┐
│ EKS Control Plane │
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │
│ │ API Server │ │ etcd │ │ Scheduler │ │
│ │ (Multi-AZ) │ │ (Multi-AZ) │ │ Controller │ │
│ └─────────────┘ └─────────────┘ └─────────────┘ │
└─────────────────────────────────────────────────────────────────────┘
│
┌───────────────┼───────────────┐
│ │ │
┌───────▼───────┐ ┌─────▼─────┐ ┌───────▼───────┐
│ General NG │ │ Compute NG│ │ GPU NG │
│ t3.medium │ │ c5.large │ │ g4dn.xlarge │
│ (On-Demand) │ │ (Spot) │ │ (On-Demand) │
└───────────────┘ └───────────┘ └───────────────┘
│ │ │
┌───────▼───────────────▼───────────────▼───────┐
│ Pods │
│ ┌────────┐ ┌────────┐ ┌────────┐ ┌────────┐ │
│ │ BFF │ │ Auth │ │Content │ │ AI │ │
│ └────────┘ └────────┘ └────────┘ └────────┘ │
└───────────────────────────────────────────────┘
Cluster Configuration#
| Setting | Development | Staging | Production |
|---|---|---|---|
| Kubernetes Version | 1.28 | 1.28 | 1.28 |
| Control Plane Logging | api, audit | All types | All types |
| Endpoint Access | Public + Private | Private | Private |
| Encryption | KMS | KMS | KMS |
Terraform Configuration#
hcl
# modules/eks/main.tf
resource "aws_eks_cluster" "main" {
name = "${var.environment}-lilith"
role_arn = aws_iam_role.cluster.arn
version = var.kubernetes_version
vpc_config {
subnet_ids = concat(var.private_subnet_ids, var.public_subnet_ids)
endpoint_private_access = true
endpoint_public_access = var.endpoint_public_access
public_access_cidrs = var.public_access_cidrs
security_group_ids = [var.cluster_security_group_id]
}
encryption_config {
provider {
key_arn = var.kms_key_arn
}
resources = ["secrets"]
}
enabled_cluster_log_types = var.cluster_log_types
depends_on = [
aws_iam_role_policy_attachment.cluster_amazon_eks_cluster_policy
]
tags = merge(var.tags, {
Name = "${var.environment}-lilith-eks"
})
}
Node Groups#
Node Group Types#
| Node Group | Instance Type | Capacity | Purpose |
|---|---|---|---|
| General | t3.medium | On-Demand | Core services, API pods |
| Compute | c5.large, c5a.large | Spot | Batch processing, CPU-intensive |
| GPU | g4dn.xlarge | On-Demand | AI inference, ML workloads |
| Memory | r5.large | Spot | In-memory processing, caching |
Configuration#
hcl
variable "node_groups" {
description = "EKS node group configurations"
type = map(object({
instance_types = list(string)
ami_type = string
capacity_type = string
disk_size = number
desired_size = number
max_size = number
min_size = number
max_unavailable_percentage = number
}))
default = {
general = {
instance_types = ["t3.medium"]
ami_type = "AL2_x86_64"
capacity_type = "ON_DEMAND"
disk_size = 50
desired_size = 3
max_size = 10
min_size = 2
max_unavailable_percentage = 25
}
compute = {
instance_types = ["c5.large", "c5a.large", "c5n.large"]
ami_type = "AL2_x86_64"
capacity_type = "SPOT"
disk_size = 100
desired_size = 0
max_size = 20
min_size = 0
max_unavailable_percentage = 50
}
gpu = {
instance_types = ["g4dn.xlarge"]
ami_type = "AL2_x86_64_GPU"
capacity_type = "ON_DEMAND"
disk_size = 100
desired_size = 0
max_size = 5
min_size = 0
max_unavailable_percentage = 50
}
}
}
resource "aws_eks_node_group" "main" {
for_each = var.node_groups
cluster_name = aws_eks_cluster.main.name
node_group_name = "${var.environment}-lilith-${each.key}"
node_role_arn = aws_iam_role.node_group.arn
subnet_ids = var.private_subnet_ids
instance_types = each.value.instance_types
ami_type = each.value.ami_type
capacity_type = each.value.capacity_type
disk_size = each.value.disk_size
scaling_config {
desired_size = each.value.desired_size
max_size = each.value.max_size
min_size = each.value.min_size
}
update_config {
max_unavailable_percentage = each.value.max_unavailable_percentage
}
labels = {
"node-group" = each.key
"environment" = var.environment
}
taint {
key = each.key == "gpu" ? "nvidia.com/gpu" : null
value = each.key == "gpu" ? "true" : null
effect = each.key == "gpu" ? "NO_SCHEDULE" : null
}
lifecycle {
ignore_changes = [scaling_config[0].desired_size]
}
}
Node Affinity and Tolerations#
yaml
# Schedule on general node group
apiVersion: apps/v1
kind: Deployment
metadata:
name: bff
spec:
template:
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-group
operator: In
values:
- general
---
# Schedule AI workloads on GPU nodes
apiVersion: apps/v1
kind: Deployment
metadata:
name: ai-inference
spec:
template:
spec:
tolerations:
- key: nvidia.com/gpu
operator: Exists
effect: NoSchedule
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-group
operator: In
values:
- gpu
resources:
limits:
nvidia.com/gpu: 1
Cluster Add-ons#
Core Add-ons#
| Add-on | Version | Purpose |
|---|---|---|
| vpc-cni | v1.15.0 | Pod networking |
| coredns | v1.10.1 | Cluster DNS |
| kube-proxy | v1.28.1 | Service networking |
| ebs-csi-driver | v1.25.0 | EBS volume provisioning |
Terraform Configuration#
hcl
resource "aws_eks_addon" "vpc_cni" {
cluster_name = aws_eks_cluster.main.name
addon_name = "vpc-cni"
addon_version = var.vpc_cni_version
resolve_conflicts = "OVERWRITE"
service_account_role_arn = aws_iam_role.vpc_cni.arn
}
resource "aws_eks_addon" "coredns" {
cluster_name = aws_eks_cluster.main.name
addon_name = "coredns"
addon_version = var.coredns_version
resolve_conflicts = "OVERWRITE"
}
resource "aws_eks_addon" "kube_proxy" {
cluster_name = aws_eks_cluster.main.name
addon_name = "kube-proxy"
addon_version = var.kube_proxy_version
resolve_conflicts = "OVERWRITE"
}
resource "aws_eks_addon" "ebs_csi_driver" {
cluster_name = aws_eks_cluster.main.name
addon_name = "aws-ebs-csi-driver"
addon_version = var.ebs_csi_version
resolve_conflicts = "OVERWRITE"
service_account_role_arn = aws_iam_role.ebs_csi.arn
}
Additional Controllers#
| Controller | Purpose | Helm Chart |
|---|---|---|
| AWS Load Balancer Controller | ALB/NLB provisioning | eks/aws-load-balancer-controller |
| External DNS | DNS record management | bitnami/external-dns |
| Cert Manager | Certificate management | jetstack/cert-manager |
| Metrics Server | Pod metrics | metrics-server/metrics-server |
| Cluster Autoscaler | Node scaling | autoscaler/cluster-autoscaler |
RBAC Configuration#
Cluster Roles#
yaml
# Admin role
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: lilith-admin
rules:
- apiGroups: ['*']
resources: ['*']
verbs: ['*']
---
# Developer role (read + pods management)
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: lilith-developer
rules:
- apiGroups: ['']
resources: ['pods', 'pods/log', 'pods/exec', 'services', 'configmaps']
verbs: ['get', 'list', 'watch']
- apiGroups: ['']
resources: ['pods/portforward']
verbs: ['create']
- apiGroups: ['apps']
resources: ['deployments', 'replicasets', 'statefulsets']
verbs: ['get', 'list', 'watch']
---
# Read-only role
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: lilith-readonly
rules:
- apiGroups: ['']
resources: ['*']
verbs: ['get', 'list', 'watch']
- apiGroups: ['apps', 'batch', 'networking.k8s.io']
resources: ['*']
verbs: ['get', 'list', 'watch']
AWS Auth ConfigMap#
yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: aws-auth
namespace: kube-system
data:
mapRoles: |
- rolearn: arn:aws:iam::ACCOUNT_ID:role/lilith-eks-node-group-role
username: system:node:{{EC2PrivateDNSName}}
groups:
- system:bootstrappers
- system:nodes
- rolearn: arn:aws:iam::ACCOUNT_ID:role/lilith-admin-role
username: admin
groups:
- system:masters
- rolearn: arn:aws:iam::ACCOUNT_ID:role/lilith-developer-role
username: developer
groups:
- lilith-developers
Namespaces#
Namespace Structure#
| Namespace | Purpose | Resource Quota |
|---|---|---|
| lilith | Application workloads | 100 pods, 200Gi memory |
| lilith-staging | Staging environment | 50 pods, 100Gi memory |
| monitoring | Prometheus, Grafana | 20 pods, 50Gi memory |
| logging | Loki, Fluentbit | 10 pods, 30Gi memory |
| vault | HashiCorp Vault | 5 pods, 10Gi memory |
| argocd | ArgoCD GitOps | 10 pods, 20Gi memory |
| istio-system | Service mesh | 20 pods, 40Gi memory |
Namespace Configuration#
yaml
apiVersion: v1
kind: Namespace
metadata:
name: lilith
labels:
name: lilith
environment: production
istio-injection: enabled
---
apiVersion: v1
kind: ResourceQuota
metadata:
name: lilith-quota
namespace: lilith
spec:
hard:
pods: '100'
requests.cpu: '50'
requests.memory: 100Gi
limits.cpu: '100'
limits.memory: 200Gi
persistentvolumeclaims: '20'
---
apiVersion: v1
kind: LimitRange
metadata:
name: lilith-limits
namespace: lilith
spec:
limits:
- default:
cpu: 500m
memory: 512Mi
defaultRequest:
cpu: 100m
memory: 128Mi
max:
cpu: 4
memory: 8Gi
min:
cpu: 50m
memory: 64Mi
type: Container
Resource Management#
Horizontal Pod Autoscaler#
yaml
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: bff-hpa
namespace: lilith
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: bff
minReplicas: 3
maxReplicas: 20
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 70
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: 80
behavior:
scaleUp:
stabilizationWindowSeconds: 60
policies:
- type: Pods
value: 4
periodSeconds: 60
scaleDown:
stabilizationWindowSeconds: 300
policies:
- type: Pods
value: 1
periodSeconds: 120
Pod Disruption Budget#
yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: bff-pdb
namespace: lilith
spec:
minAvailable: 2
selector:
matchLabels:
app: bff
Cluster Autoscaler#
yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: cluster-autoscaler
namespace: kube-system
spec:
replicas: 1
template:
spec:
containers:
- name: cluster-autoscaler
image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.28.0
command:
- ./cluster-autoscaler
- --v=4
- --stderrthreshold=info
- --cloud-provider=aws
- --skip-nodes-with-local-storage=false
- --expander=least-waste
- --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/lilith-prod
- --balance-similar-node-groups
- --scale-down-enabled=true
- --scale-down-delay-after-add=10m
- --scale-down-unneeded-time=10m
Security#
Pod Security Standards#
yaml
apiVersion: v1
kind: Namespace
metadata:
name: lilith
labels:
pod-security.kubernetes.io/enforce: restricted
pod-security.kubernetes.io/audit: restricted
pod-security.kubernetes.io/warn: restricted
Security Context#
yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: bff
spec:
template:
spec:
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
containers:
- name: bff
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
IRSA (IAM Roles for Service Accounts)#
hcl
# OIDC Provider
data "tls_certificate" "eks" {
url = aws_eks_cluster.main.identity[0].oidc[0].issuer
}
resource "aws_iam_openid_connect_provider" "eks" {
client_id_list = ["sts.amazonaws.com"]
thumbprint_list = [data.tls_certificate.eks.certificates[0].sha1_fingerprint]
url = aws_eks_cluster.main.identity[0].oidc[0].issuer
}
# Service-specific IAM role
resource "aws_iam_role" "ai_service" {
name = "${var.environment}-lilith-ai-service-role"
assume_role_policy = jsonencode({
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Principal = {
Federated = aws_iam_openid_connect_provider.eks.arn
}
Action = "sts:AssumeRoleWithWebIdentity"
Condition = {
StringEquals = {
"${replace(aws_iam_openid_connect_provider.eks.url, "https://", "")}:sub" = "system:serviceaccount:lilith:ai-service"
"${replace(aws_iam_openid_connect_provider.eks.url, "https://", "")}:aud" = "sts.amazonaws.com"
}
}
}
]
})
}
Observability#
Prometheus ServiceMonitor#
yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: lilith-services
namespace: monitoring
spec:
selector:
matchLabels:
monitoring: enabled
namespaceSelector:
matchNames:
- lilith
endpoints:
- port: metrics
interval: 30s
path: /metrics
Logging with Fluentbit#
yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: fluent-bit-config
namespace: logging
data:
fluent-bit.conf: |
[SERVICE]
Flush 1
Log_Level info
Daemon off
HTTP_Server On
HTTP_Listen 0.0.0.0
HTTP_Port 2020
[INPUT]
Name tail
Path /var/log/containers/*.log
Parser docker
Tag kube.*
Refresh_Interval 5
Mem_Buf_Limit 50MB
Skip_Long_Lines On
[FILTER]
Name kubernetes
Match kube.*
Kube_URL https://kubernetes.default.svc:443
Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
Merge_Log On
K8S-Logging.Parser On
[OUTPUT]
Name loki
Match *
Host loki.logging.svc.cluster.local
Port 3100
Labels job=fluent-bit
Cluster Operations#
Upgrade Procedure#
bash
# 1. Check current version
kubectl version --short
# 2. Update control plane
aws eks update-cluster-version \
--name lilith-prod \
--kubernetes-version 1.29
# 3. Wait for update
aws eks wait cluster-active --name lilith-prod
# 4. Update node groups
aws eks update-nodegroup-version \
--cluster-name lilith-prod \
--nodegroup-name general \
--kubernetes-version 1.29
# 5. Update add-ons
aws eks update-addon \
--cluster-name lilith-prod \
--addon-name vpc-cni \
--addon-version v1.16.0
Troubleshooting#
bash
# Check cluster status
aws eks describe-cluster --name lilith-prod --query cluster.status
# Check node status
kubectl get nodes -o wide
# Check pending pods
kubectl get pods --all-namespaces --field-selector=status.phase=Pending
# Check cluster events
kubectl get events --sort-by=.lastTimestamp -A
# Debug node issues
kubectl describe node node-name
# Check API server logs
kubectl logs -n kube-system -l component=kube-apiserver