🌐 EGS Installer

# egs-installer-config.yaml
global_kubeconfig: "path/to/kubeconfig"    # Required - Path to your kubeconfig file
global_kubecontext: "your-context"         # Required - Context to use
use_global_context: true

enable_install_controller: true
enable_install_ui: true
enable_install_worker: true
enable_install_additional_apps: true       # Enables prerequisites (PostgreSQL, Prometheus, GPU Operator)
enable_custom_apps: false                  # Set true for NVIDIA drivers on GKE
run_commands: false                        # Set true for MIG configuration
add_node_label: true                       # Auto-label gateway nodes

kubeslice_controller_egs:
  skip_installation: false
  use_global_kubeconfig: true
  specific_use_local_charts: true
  kubeconfig: ""                               # Empty = uses global_kubeconfig
  kubecontext: ""                              # Empty = uses global_kubecontext
  namespace: "kubeslice-controller"
  release: "egs-controller"
  chart: "kubeslice-controller-egs"
  inline_values:
    global:
      imageRegistry: harbor.saas1.smart-scaler.io/avesha/aveshasystems
      kubeTally:
        enabled: true
        # PostgreSQL Configuration - Update if using your own PostgreSQL
        postgresSecretName: kubetally-db-credentials
        existingSecret: false                  # Set true if using pre-created secret
        postgresAddr: "kt-postgresql.kt-postgresql.svc.cluster.local"
        postgresPort: 5432
        postgresUser: "postgres"
        postgresPassword: "postgres"
        postgresDB: "postgres"
        postgresSslmode: disable
        prometheusUrl: http://prometheus-kube-prometheus-prometheus.egs-monitoring.svc.cluster.local:9090
    kubeslice:
      controller:
        endpoint: ""                           # Auto-fetched if empty
        replication:
          minio:
            install: "true"
            storage: 1Gi
            username: minioadmin
            password: minioadmin
            service:
              type: "LoadBalancer"
    serviceMonitor:
      enabled: true
      namespace: egs-monitoring
  helm_flags: "--wait --timeout 5m --debug"
  verify_install: false
  verify_install_timeout: 30

kubeslice_ui_egs:
  skip_installation: false
  use_global_kubeconfig: true
  kubeconfig: ""                               # Empty = uses global_kubeconfig
  kubecontext: ""                              # Empty = uses global_kubecontext
  namespace: "kubeslice-controller"
  release: "egs-ui"
  chart: "kubeslice-ui-egs"
  specific_use_local_charts: true
  inline_values:
    global:
      imageRegistry: harbor.saas1.smart-scaler.io/avesha/aveshasystems
    kubeslice:
      prometheus:
        url: http://prometheus-kube-prometheus-prometheus.egs-monitoring.svc.cluster.local:9090
      uiproxy:
        service:
          type: LoadBalancer                   # Use NodePort if LoadBalancer unavailable
      apigw:
        env:
          - name: DCGM_METRIC_JOB_VALUE
            value: nvidia-dcgm-exporter        # Must match Prometheus scrape job name
      egsCoreApis:
        enabled: true
  helm_flags: "--wait --timeout 5m --debug"
  verify_install: false

kubeslice_worker_egs:
  - name: "worker-1"                           # Must match cluster_registration
    use_global_kubeconfig: true
    kubeconfig: ""                             # Empty = uses global_kubeconfig
    kubecontext: ""                            # Empty = uses global_kubecontext
    skip_installation: false
    specific_use_local_charts: true
    namespace: "kubeslice-system"
    release: "egs-worker"
    chart: "kubeslice-worker-egs"
    inline_values:
      global:
        imageRegistry: harbor.saas1.smart-scaler.io/avesha/aveshasystems
      kubesliceNetworking:
        enabled: false                         # Set true if using KubeSlice networking
      operator:
        env:
          - name: DCGM_EXPORTER_JOB_NAME
            value: gpu-metrics                 # Must match Prometheus scrape job name
      egs:
        prometheusEndpoint: "http://prometheus-kube-prometheus-prometheus.egs-monitoring.svc.cluster.local:9090"
        grafanaDashboardBaseUrl: "http://<grafana-lb>/d/Oxed_c6Wz"
      egsGpuAgent:
        env:
          - name: REMOTE_HE_INFO
            value: "nvidia-dcgm.egs-gpu-operator.svc.cluster.local:5555"
          - name: HEALTH_CHECK_INTERVAL
            value: "15m"
      monitoring:
        podMonitor:
          enabled: true
          namespace: egs-monitoring
    helm_flags: "--wait --timeout 5m --debug"
    verify_install: true
    verify_install_timeout: 60

cluster_registration:
  - cluster_name: "worker-1"                   # Must match worker name
    project_name: "avesha"
    telemetry:
      enabled: true
      endpoint: "http://prometheus-kube-prometheus-prometheus.egs-monitoring.svc.cluster.local:9090"
      telemetryProvider: "prometheus"
    geoLocation:
      cloudProvider: ""                        # e.g., "GCP", "AWS", "azure"
      cloudRegion: ""                          # e.g., "us-central1", "us-east-1"

additional_apps:
  # GPU Operator - Required for GPU workloads
  - name: "gpu-operator"
    skip_installation: false
    use_global_kubeconfig: true
    kubeconfig: ""
    kubecontext: ""
    namespace: "egs-gpu-operator"
    release: "gpu-operator"
    chart: "gpu-operator"
    repo_url: "https://helm.ngc.nvidia.com/nvidia"
    version: "v25.3.4"
    specific_use_local_charts: true
    inline_values:
      hostPaths:
        driverInstallDir: "/home/kubernetes/bin/nvidia"
      toolkit:
        installDir: "/home/kubernetes/bin/nvidia"
      cdi:
        enabled: true
        default: true
      driver:
        enabled: false                         # Set true if nodes need NVIDIA drivers
    helm_flags: "--debug"
    verify_install: false
    verify_install_timeout: 600

  # Prometheus - Required for monitoring
  - name: "prometheus"
    skip_installation: false
    use_global_kubeconfig: true
    kubeconfig: ""
    kubecontext: ""
    namespace: "egs-monitoring"
    release: "prometheus"
    chart: "kube-prometheus-stack"
    repo_url: "https://prometheus-community.github.io/helm-charts"
    version: "v45.0.0"
    specific_use_local_charts: true
    inline_values:
      prometheus:
        service:
          type: ClusterIP
        prometheusSpec:
          additionalScrapeConfigs:
            - job_name: gpu-metrics
              scrape_interval: 1s
              metrics_path: /metrics
              kubernetes_sd_configs:
                - role: endpoints
                  namespaces:
                    names:
                      - egs-gpu-operator
      grafana:
        enabled: true
    helm_flags: "--debug"
    verify_install: false

  # PostgreSQL - Required for KubeTally
  - name: "postgresql"
    skip_installation: false
    use_global_kubeconfig: true
    kubeconfig: ""
    kubecontext: ""
    namespace: "kt-postgresql"
    release: "kt-postgresql"
    chart: "postgresql"
    repo_url: "oci://registry-1.docker.io/bitnamicharts/postgresql"
    version: "16.7.27"
    specific_use_local_charts: true
    inline_values:
      auth:
        postgresPassword: "postgres"
        username: "postgres"
        password: "postgres"
        database: "postgres"
      primary:
        persistence:
          enabled: true
          size: 10Gi
    helm_flags: "--debug"
    verify_install: false

projects:
  - name: "avesha"
    username: "admin"

# Manifests for GPU quota and NVIDIA driver
manifests:
  - appname: "gpu-operator-quota"
    use_global_kubeconfig: true
    namespace: "egs-gpu-operator"
    skip_installation: false
    inline_yaml: |
      apiVersion: v1
      kind: ResourceQuota
      metadata:
        name: gpu-operator-quota
        namespace: egs-gpu-operator
      spec:
        hard:
          pods: "100"
        scopeSelector:
          matchExpressions:
            - operator: In
              scopeName: PriorityClass
              values:
                - system-node-critical
                - system-cluster-critical

  - appname: "nvidia-driver-installer"
    use_global_kubeconfig: true
    namespace: "kube-system"
    skip_installation: false
    manifest: "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/master/nvidia-driver-installer/cos/daemonset-preloaded.yaml"

# Commands for NVIDIA MIG configuration (when run_commands: true)
commands:
  - use_global_kubeconfig: true
    skip_installation: false
    namespace: kube-system
    command_stream: |
      kubectl create namespace egs-gpu-operator --dry-run=client -o yaml | kubectl apply -f - || true
      kubectl get nodes -o json | jq -r '.items[] | select(.status.capacity["nvidia.com/gpu"] != null) | .metadata.name' | xargs -I {} kubectl label nodes {} gke-no-default-nvidia-gpu-device-plugin=true cloud.google.com/gke-accelerator=true --overwrite || true

# Global monitoring endpoint settings
global_auto_fetch_endpoint: true               # Auto-fetch Prometheus/Grafana endpoints
global_grafana_namespace: egs-monitoring
global_grafana_service_type: ClusterIP         # Use LoadBalancer for multi-cluster
global_grafana_service_name: prometheus-grafana
global_prometheus_namespace: egs-monitoring
global_prometheus_service_name: prometheus-kube-prometheus-prometheus
global_prometheus_service_type: ClusterIP      # Use LoadBalancer for multi-cluster

kubeslice_worker_egs:
  - name: "worker-1"
    use_global_kubeconfig: false               # Do NOT use global kubeconfig
    kubeconfig: "worker-1-kubeconfig.yaml"     # Path to worker-1 specific kubeconfig
    kubecontext: "worker-1-context"            # Context name in the kubeconfig file
    skip_installation: false
    specific_use_local_charts: true
    namespace: "kubeslice-system"
    release: "egs-worker"
    chart: "kubeslice-worker-egs"
    inline_values:
      egs:
        prometheusEndpoint: "http://<worker-1-prometheus-lb>:9090"  # External endpoint
        grafanaDashboardBaseUrl: "http://<worker-1-grafana-lb>/d/Oxed_c6Wz"
    # ... other values

  - name: "worker-2"
    use_global_kubeconfig: false               # Do NOT use global kubeconfig
    kubeconfig: "worker-2-kubeconfig.yaml"     # Path to worker-2 specific kubeconfig
    kubecontext: "worker-2-context"            # Context name in the kubeconfig file
    skip_installation: false
    specific_use_local_charts: true
    namespace: "kubeslice-system"
    release: "egs-worker-2"                    # Unique release name
    chart: "kubeslice-worker-egs"
    inline_values:
      egs:
        prometheusEndpoint: "http://<worker-2-prometheus-lb>:9090"  # External endpoint
        grafanaDashboardBaseUrl: "http://<worker-2-grafana-lb>/d/Oxed_c6Wz"
    # ... other values

cluster_registration:
  - cluster_name: "worker-1"
    project_name: "avesha"
    telemetry:
      enabled: true
      endpoint: "http://<worker-1-prometheus-lb>:9090"  # Must be accessible from controller
      telemetryProvider: "prometheus"
    geoLocation:
      cloudProvider: "GCP"
      cloudRegion: "us-west1"
      
  - cluster_name: "worker-2"
    project_name: "avesha"
    telemetry:
      enabled: true
      endpoint: "http://<worker-2-prometheus-lb>:9090"  # Must be accessible from controller
      telemetryProvider: "prometheus"
    geoLocation:
      cloudProvider: "AWS"
      cloudRegion: "us-east-1"