Common Operations

Frequently performed operational tasks and procedures

Daily Operations

Health Checks

Cluster Status

# Check K3s cluster health
kubectl get nodes
kubectl get pods --all-namespaces | grep -v Running

# Check Proxmox cluster
ssh root@192.168.1.3 pvecm status

# Check storage
kubectl get pv
kubectl get pvc --all-namespaces

Service Availability

# Check ingress endpoints
kubectl get ingress --all-namespaces

# Test service connectivity
curl -I https://grafana.home.dodwell.us
curl -I https://argocd.home.dodwell.us

Resource Management

Check Resource Usage

# Node resources
kubectl top nodes

# Pod resources by namespace
kubectl top pods -n gitea
kubectl top pods -n home-assistant

# Proxmox VM resources
ssh root@192.168.1.3 qm list

Scale Applications

# Scale deployment
kubectl scale deployment gitea -n gitea --replicas=2

# Check HPA status
kubectl get hpa --all-namespaces

Deployment Operations

Deploy New Application

  1. Create namespace:
kubectl create namespace myapp
  1. Create application manifest:
# k3s/apps/myapp/base/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

namespace: myapp

resources:
  - namespace.yaml
  - deployment.yaml
  - service.yaml
  - ingress.yaml
  1. Add to ArgoCD:
# k3s/envs/local/myapp.yaml
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: myapp
  namespace: argocd
spec:
  project: default
  source:
    repoURL: https://gitea.home.dodwell.us/exodus/homelab.git
    targetRevision: main
    path: k3s/apps/myapp/overlays/local
  destination:
    server: https://kubernetes.default.svc
    namespace: myapp
  syncPolicy:
    automated:
      prune: true
      selfHeal: true

Update Application

Via GitOps

# Edit manifest in git
vim k3s/apps/myapp/base/deployment.yaml

# Commit and push
git add .
git commit -m "Update myapp to version 2.0"
git push

# ArgoCD will automatically sync

Manual Update

# Edit directly (not recommended)
kubectl edit deployment myapp -n myapp

# Or apply changes
kubectl apply -f updated-manifest.yaml

Maintenance Tasks

Certificate Renewal

Certificates are auto-renewed by cert-manager, but to check:

# List certificates
kubectl get certificates --all-namespaces

# Check expiry
kubectl describe certificate wiki-tls -n static-web

# Force renewal if needed
kubectl delete certificate wiki-tls -n static-web

Storage Maintenance

Clean Up Old Images

# On each Proxmox node
ssh root@192.168.1.3 docker system prune -a

# In Harbor
# Use Harbor UI to run garbage collection

Expand PVC

# Edit PVC size
kubectl edit pvc gitea-data -n gitea

# Check expansion progress
kubectl describe pvc gitea-data -n gitea

Node Maintenance

Drain K3s Node

# Cordon node (prevent new pods)
kubectl cordon k3s-worker-01

# Drain node (move pods)
kubectl drain k3s-worker-01 --ignore-daemonsets --delete-emptydir-data

# Perform maintenance
ssh ubuntu@192.168.1.212

# Uncordon node
kubectl uncordon k3s-worker-01

Proxmox Updates

# Check for updates
ssh root@192.168.1.3 apt update
ssh root@192.168.1.3 apt list --upgradable

# Apply updates (schedule maintenance window)
ssh root@192.168.1.3 apt upgrade

# Reboot if kernel updated
ssh root@192.168.1.3 reboot

Troubleshooting

Pod Issues

Pod Won’t Start

# Describe pod
kubectl describe pod <pod-name> -n <namespace>

# Check logs
kubectl logs <pod-name> -n <namespace>
kubectl logs <pod-name> -n <namespace> --previous

# Check events
kubectl get events -n <namespace> --sort-by='.lastTimestamp'

CrashLoopBackOff

# Get detailed info
kubectl get pod <pod-name> -n <namespace> -o yaml

# Check resource limits
kubectl describe node <node-name>

# Force restart
kubectl delete pod <pod-name> -n <namespace>

Storage Issues

PVC Won’t Bind

# Check PV status
kubectl get pv

# Check storage class
kubectl get storageclass

# Check iSCSI connection
ssh ubuntu@192.168.1.210 sudo iscsiadm -m session

Disk Full

# Find large files
kubectl exec -it <pod-name> -n <namespace> -- du -sh /*

# Clean up logs
kubectl exec -it <pod-name> -n <namespace> -- find /var/log -type f -delete

Network Issues

Service Not Reachable

# Check service endpoints
kubectl get endpoints <service-name> -n <namespace>

# Test from inside cluster
kubectl run test-curl --image=curlimages/curl -it --rm -- sh
curl http://service-name.namespace.svc.cluster.local

# Check ingress
kubectl describe ingress <ingress-name> -n <namespace>

DNS Resolution

# Test DNS
kubectl run test-dns --image=busybox -it --rm -- nslookup gitea.gitea.svc.cluster.local

# Check CoreDNS
kubectl logs -n kube-system -l k8s-app=kube-dns

Backup Procedures

Manual Backup

etcd Backup

# On master node
ssh ubuntu@192.168.1.210
sudo k3s etcd-snapshot save

# List snapshots
sudo k3s etcd-snapshot list

Application Data

# Backup GitOps repo
git clone https://gitea.home.dodwell.us/exodus/homelab.git homelab-backup

# Backup PVCs
kubectl get pvc --all-namespaces -o yaml > pvc-backup.yaml

Restore Procedures

Restore from etcd Snapshot

# Stop K3s
sudo systemctl stop k3s

# Restore snapshot
sudo k3s etcd-snapshot restore <snapshot-name>

# Start K3s
sudo systemctl start k3s

Restore Application

# Restore from GitOps
kubectl apply -f k3s/envs/local/

# Restore PVC data from backup
# Mount backup volume and copy data