# Example with Dex as OIDC provider
apiVersion: v1
kind: ConfigMap
metadata:
  name: kube-apiserver
  namespace: kube-system
data:
  kube-apiserver.yaml: |
    apiVersion: v1
    kind: Pod
    metadata:
      name: kube-apiserver
      labels:
        component: kube-apiserver
        tier: control-plane
    spec:
      containers:
      - name: kube-apiserver
        image: k8s.gcr.io/kube-apiserver:v1.23.0
        command:
        - kube-apiserver
        - --oidc-issuer-url=https://dex.example.com        # OIDC provider URL
        - --oidc-client-id=kubernetes                      # Client ID for the OIDC provider
        - --oidc-username-claim=email                      # Which claim to use as the username
        - --oidc-groups-claim=groups                       # Which claim to use for user groups
        - --oidc-ca-file=/etc/kubernetes/ssl/dex-ca.crt    # CA cert for the OIDC provider
        - --oidc-required-claim=hd=example.com             # Required claims for authentication
        volumeMounts:
        - mountPath: /etc/kubernetes/ssl
          name: ssl-certs
      volumes:
      - name: ssl-certs
        hostPath:
          path: /etc/kubernetes/ssl

# ClusterRole defining permissions
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: app-viewer
rules:
- apiGroups: [""]
  resources: ["pods", "services", "configmaps"]
  verbs: ["get", "list", "watch"]
- apiGroups: ["apps"]
  resources: ["deployments", "statefulsets"]
  verbs: ["get", "list", "watch"]
---
# ClusterRoleBinding for a group from OIDC provider
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: app-viewers
subjects:
- kind: Group
  name: app-team@example.com  # This matches the group from OIDC provider
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: ClusterRole
  name: app-viewer
  apiGroup: rbac.authorization.k8s.io

# AWS IAM Authenticator ConfigMap
apiVersion: v1
kind: ConfigMap
metadata:
  name: aws-auth
  namespace: kube-system
data:
  mapRoles: |
    - rolearn: arn:aws:iam::123456789012:role/KubernetesAdmin
      username: kubernetes-admin
      groups:
        - system:masters
    - rolearn: arn:aws:iam::123456789012:role/DeveloperRole
      username: developer-{{ SessionName }}
      groups:
        - developers
  mapUsers: |
    - userarn: arn:aws:iam::123456789012:user/alice
      username: alice
      groups:
        - system:masters
    - userarn: arn:aws:iam::123456789012:user/bob
      username: bob
      groups:
        - developers
        - app-team

# Istio multi-cluster setup

## Backup and Disaster Recovery

::alert{type="warning"}
Multi-cluster backup strategies:
1. **Use backup tools like Velero across all clusters**
   - Deploy Velero to each cluster
   - Standardize backup configurations
   - Centralize backup management
   - Use consistent storage providers
   - Implement proper RBAC for backup operations

2. **Implement cross-cluster backup storage**
   - Use cloud object storage (S3, GCS, Azure Blob)
   - Ensure storage is accessible from all clusters
   - Consider compliance requirements for data location
   - Set up appropriate IAM permissions
   - Implement encryption for backups at rest

3. **Create consistent backup schedules and retention policies**
   - Define appropriate backup frequencies
   - Establish retention periods for different data types
   - Balance frequency with storage costs
   - Schedule backups to minimize impact
   - Implement tiered retention (daily, weekly, monthly)

4. **Test backup restoration regularly**
   - Schedule regular restoration drills
   - Validate data integrity post-restore
   - Test cross-cluster recovery scenarios
   - Measure recovery time objectives (RTOs)
   - Automate restoration testing where possible

5. **Document recovery procedures for each cluster**
   - Create detailed runbooks for different failure scenarios
   - Define roles and responsibilities during recovery
   - Include all dependencies and prerequisites
   - Document communication procedures
   - Keep documentation updated with environment changes

6. **Implement geo-redundant backup storage**
   - Replicate backups across multiple regions
   - Use cross-region replication features
   - Ensure backup metadata is also replicated
   - Test recovery from secondary regions
   - Consider data sovereignty requirements
::

```yaml
# Example Velero backup configuration
apiVersion: velero.io/v1
kind: Backup
metadata:
  name: daily-backup
  namespace: velero
  labels:
    backup-type: scheduled
    environment: production
  annotations:
    description: "Daily backup of all namespaces except system"
    retention: "30 days"
spec:
  # Include all namespaces except those excluded below
  includedNamespaces:
  - '*'
  # Exclude system namespaces to reduce backup size
  excludedNamespaces:
  - kube-system
  - istio-system
  - monitoring
  # Exclude transient resources that don't need backup
  excludedResources:
  - events
  - nodes
  - events.events.k8s.io
  - metrics.k8s.io
  # Include all other resources
  includedResources:
  - '*'
  # Hooks to run before/after backup
  hooks:
    resources:
    # Database consistency hook
    - name: backup-hook
      includedNamespaces:
      - '*'
      labelSelector:
        matchLabels:
          app: database
      # Pre-backup actions
      pre:
      - exec:
          container: database
          command:
          - /bin/sh
          - -c
          - pg_dump -U postgres > /backup/db.sql
          onError: Fail  # Fail backup if hook fails
          timeout: 300s  # 5-minute timeout
      # Post-backup actions
      post:
      - exec:
          container: database
          command:
          - /bin/sh
          - -c
          - rm -f /backup/db.sql
          onError: Continue  # Continue even if cleanup fails
  # Include volume snapshots
  snapshotVolumes: true
  # Default snapshot provider
  defaultVolumesToFsBackup: false
  # TTL (time to live) - 30 days
  ttl: 720h0m0s
  # Storage location for this backup
  storageLocation: aws-s3
  # Volume snapshot location
  volumeSnapshotLocations:
  - aws-us-east-1
  # Ordered resources to ensure proper backup order
  orderedResources:
    persistentvolumes: pvc
    persistentvolumeclaims: pod

# Velero Schedule for Coordinated Backups Across Clusters
apiVersion: velero.io/v1
kind: Schedule
metadata:
  name: multi-cluster-coordinated-backup
  namespace: velero
spec:
  schedule: "0 1 * * *"  # Daily at 1 AM
  template:
    includedNamespaces:
    - '*'
    excludedNamespaces:
    - kube-system
    - velero
    includedResources:
    - '*'
    excludedResources:
    - events
    labelSelector:
      matchExpressions:
      - key: backup-eligible
        operator: In
        values:
        - "true"
    snapshotVolumes: true
    ttl: 720h0m0s
    storageLocation: shared-storage

# Thanos Sidecar configuration for Prometheus
apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
  name: prometheus
  namespace: monitoring
spec:
  replicas: 2
  version: v2.32.1
  thanos:
    baseImage: quay.io/thanos/thanos
    version: v0.24.0
    objectStorageConfig:
      name: thanos-objectstorage
      key: thanos.yaml
  storage:
    volumeClaimTemplate:
      spec:
        storageClassName: fast
        resources:
          requests:
            storage: 100Gi
  serviceMonitorSelector:
    matchLabels:
      team: frontend

---
# Thanos Querier deployment for global view
apiVersion: apps/v1
kind: Deployment
metadata:
  name: thanos-query
  namespace: monitoring
spec:
  replicas: 2
  selector:
    matchLabels:
      app: thanos-query
  template:
    metadata:
      labels:
        app: thanos-query
    spec:
      containers:
      - name: thanos-query
        image: quay.io/thanos/thanos:v0.24.0
        args:
        - query
        - --log.level=info
        - --query.replica-label=prometheus_replica
        - --store=dnssrv+_grpc._tcp.thanos-store-gateway.monitoring.svc.cluster.local
        - --store=dnssrv+_grpc._tcp.thanos-sidecar.monitoring.svc.cluster.local
        - --store=dnssrv+_grpc._tcp.thanos-receive.monitoring.svc.cluster.local
        - --query.auto-downsampling
        ports:
        - name: http
          containerPort: 10902
        - name: grpc
          containerPort: 10901
        resources:
          limits:
            cpu: "1"
            memory: 1Gi
          requests:
            cpu: 500m
            memory: 500Mi

---
# OpenTelemetry Collector for distributed tracing
apiVersion: opentelemetry.io/v1alpha1
kind: OpenTelemetryCollector
metadata:
  name: cluster-collector
  namespace: monitoring
spec:
  mode: deployment
  config: |
    receivers:
      otlp:
        protocols:
          grpc:
            endpoint: 0.0.0.0:4317
          http:
            endpoint: 0.0.0.0:4318
      jaeger:
        protocols:
          thrift_http:
            endpoint: 0.0.0.0:14268
      zipkin:
        endpoint: 0.0.0.0:9411

    processors:
      batch:
        timeout: 10s
      memory_limiter:
        check_interval: 5s
        limit_mib: 1800
      k8s_attributes:
        auth_type: serviceAccount
        passthrough: false
        filter:
          node_from_env_var: KUBE_NODE_NAME
        extract:
          metadata:
            - k8s.pod.name
            - k8s.pod.uid
            - k8s.deployment.name
            - k8s.namespace.name
            - k8s.node.name
            - k8s.container.name
      resource:
        attributes:
        - key: cluster.name
          value: "cluster-1"
          action: upsert

    exporters:
      jaeger:
        endpoint: jaeger-collector.tracing:14250
        tls:
          insecure: true
      otlp:
        endpoint: tempo.tracing:4317
        tls:
          insecure: true

    service:
      pipelines:
        traces:
          receivers: [otlp, jaeger, zipkin]
          processors: [memory_limiter, k8s_attributes, resource, batch]
          exporters: [otlp, jaeger]

# Example Kubecost configuration
apiVersion: apps/v1
kind: Deployment
metadata:
  name: cost-analyzer
  namespace: kubecost
spec:
  selector:
    matchLabels:
      app: cost-analyzer
  template:
    metadata:
      labels:
        app: cost-analyzer
    spec:
      containers:
      - name: cost-analyzer
        image: kubecost/cost-analyzer:latest
        env:
        # Prometheus connection for metrics
        - name: PROMETHEUS_SERVER_ENDPOINT
          value: http://prometheus-server.monitoring.svc.cluster.local:9090
        # Cloud provider credentials for accurate billing data
        - name: CLOUD_PROVIDER_API_KEY
          valueFrom:
            secretKeyRef:
              name: cloud-provider-secret
              key: api-key
        # Multi-cluster configuration
        - name: KUBECOST_CLUSTER_ID
          value: cluster-east-prod
        # ETL settings for data processing
        - name: KUBECOST_ETL_ENABLED
          value: "true"
        # Retention settings for cost data
        - name: RETENTION_RESOLUTION_HOURS
          value: "3h"
        - name: RETENTION_RESOLUTION_DAYS
          value: "15d"
        - name: RETENTION_RESOLUTION_WEEKS
          value: "8w"
        - name: RETENTION_RESOLUTION_MONTHS
          value: "13M"
        # Enable federated metrics
        - name: FEDERATED_STORE_ENABLED
          value: "true"
        ports:
        - containerPort: 9003
          name: http
        resources:
          requests:
            cpu: 100m
            memory: 128Mi
          limits:
            cpu: 1
            memory: 1Gi
        volumeMounts:
        - name: persistent-configs
          mountPath: /var/configs
      volumes:
      - name: persistent-configs
        persistentVolumeClaim:
          claimName: kubecost-configs
---
# Kubecost Multi-Cluster ConfigMap
apiVersion: v1
kind: ConfigMap
metadata:
  name: kubecost-clusters-config
  namespace: kubecost
data:
  clusters.yaml: |
    clusters:
      - name: "cluster-east-prod"
        address: "http://kubecost-cost-analyzer.kubecost.svc:9090"
        local: true
        monthlySpendThreshold: 1000.0
      - name: "cluster-west-prod"
        address: "http://kubecost-aggregator.kubecost.svc:9090/federate"
        bearerToken: "${WEST_BEARER_TOKEN}"
        monthlySpendThreshold: 2000.0
      - name: "cluster-dev"
        address: "http://kubecost-aggregator.kubecost.svc:9090/federate"
        bearerToken: "${DEV_BEARER_TOKEN}"
        monthlySpendThreshold: 500.0

# ArgoCD ApplicationSet for multi-cluster deployment
apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: guestbook
  namespace: argocd
spec:
  generators:
  # Generate applications for multiple clusters
  - clusters:
      selector:
        matchLabels:
          environment: production
  # Template for the generated applications
  template:
    metadata:
      name: '{{name}}-guestbook'
      labels:
        environment: '{{metadata.labels.environment}}'
    spec:
      project: default
      source:
        repoURL: https://github.com/argoproj/argocd-example-apps.git
        targetRevision: HEAD
        path: guestbook
        # Environment-specific configuration
        helm:
          valueFiles:
          - values-{{metadata.labels.environment}}.yaml
          # Cluster-specific values
          values: |
            replicaCount: "{{metadata.annotations.preferred_replica_count}}"
            ingress:
              host: guestbook-{{name}}.example.com
            metrics:
              enabled: true
      # Destination cluster from the generator
      destination:
        server: '{{server}}'
        namespace: guestbook
      # Sync policy with automatic syncing
      syncPolicy:
        automated:
          prune: true
          selfHeal: true
        syncOptions:
        - CreateNamespace=true
        # Sync wave for ordering
        retry:
          limit: 5
          backoff:
            duration: 5s
            factor: 2
            maxDuration: 3m

# Jenkins Pipeline for Multi-Cluster Deployment
pipeline {
    agent any
    
    environment {
        APP_NAME = 'my-application'
        REGISTRY = 'registry.example.com'
        VERSION = "${env.BUILD_NUMBER}"
        CLUSTERS = 'cluster1,cluster2,cluster3'
    }
    
    stages {
        stage('Build & Test') {
            steps {
                sh 'mvn clean package'
                sh 'docker build -t ${REGISTRY}/${APP_NAME}:${VERSION} .'
            }
        }
        
        stage('Push Image') {
            steps {
                sh 'docker push ${REGISTRY}/${APP_NAME}:${VERSION}'
            }
        }
        
        stage('Deploy to Dev Cluster') {
            steps {
                sh 'kubectl config use-context cluster-dev'
                sh 'helm upgrade --install ${APP_NAME} ./charts/${APP_NAME} --set image.tag=${VERSION} --namespace=${APP_NAME}'
                sh 'kubectl rollout status deployment/${APP_NAME} -n ${APP_NAME} --timeout=300s'
            }
        }
        
        stage('Run Integration Tests') {
            steps {
                sh 'mvn verify -Pintegration-tests -Dtest.host=cluster-dev-ingress.example.com'
            }
        }
        
        stage('Promote to Staging') {
            when {
                branch 'main'
            }
            steps {
                sh 'kubectl config use-context cluster-staging'
                sh 'helm upgrade --install ${APP_NAME} ./charts/${APP_NAME} --set image.tag=${VERSION} --namespace=${APP_NAME}'
                sh 'kubectl rollout status deployment/${APP_NAME} -n ${APP_NAME} --timeout=300s'
            }
        }
        
        stage('Canary Deploy to Production') {
            when {
                branch 'main'
            }
            steps {
                script {
                    // Deploy canary to first production cluster
                    sh 'kubectl config use-context cluster-prod-east'
                    sh 'helm upgrade --install ${APP_NAME}-canary ./charts/${APP_NAME} --set image.tag=${VERSION} --set replicas=1 --namespace=${APP_NAME}'
                    
                    // Monitor canary metrics
                    sh 'sleep 300' // Replace with actual metric evaluation
                    
                    // If successful, continue with full deployment
                    def clusters = env.CLUSTERS.split(',')
                    for (cluster in clusters) {
                        sh "kubectl config use-context ${cluster}"
                        sh "helm upgrade --install ${APP_NAME} ./charts/${APP_NAME} --set image.tag=${VERSION} --namespace=${APP_NAME}"
                        sh "kubectl rollout status deployment/${APP_NAME} -n ${APP_NAME} --timeout=300s"
                    }
                }
            }
        }
    }
    
    post {
        failure {
            // Rollback on failure
            sh 'kubectl config use-context cluster-prod-east'
            sh 'helm rollback ${APP_NAME} 0 --namespace=${APP_NAME}'
        }
    }
}