[Feature] Alternative Upgrade Order (#1731)

Author: ajanikow

CHANGELOG.md

@@ -35,6 +35,7 @@
 - (Improvement) Improve Metrics Handling
 - (Feature) (Scheduler) Create Integration Profile
 - (Feature) (Scheduler) Additional types
+- (Feature) Alternative Upgrade Order Feature
 
 ## [1.2.42](https://github.com/arangodb/kube-arangodb/tree/1.2.42) (2024-07-23)
 - (Maintenance) Go 1.22.4 & Kubernetes 1.29.6 libraries

pkg/deployment/features/upgrade.go

@@ -23,6 +23,7 @@ package features
 func init() {
 	registerFeature(upgradeVersionCheck)
 	registerFeature(upgradeVersionCheckV2)
+	registerFeature(upgradeAlternativeOrder)
 }
 
 var upgradeVersionCheck Feature = &feature{
@@ -39,10 +40,20 @@ var upgradeVersionCheckV2 Feature = &feature{
 	enabledByDefault:   false,
 }
 
+var upgradeAlternativeOrder Feature = &feature{
+	name:               "upgrade-alternative-order",
+	description:        "Changes order of the upgrade process - Coordinators are upgraded before DBServers",
+	enterpriseRequired: false,
+	enabledByDefault:   false,
+	hidden:             true,
+}
+
 func UpgradeVersionCheck() Feature {
 	return upgradeVersionCheck
 }
 
 func UpgradeVersionCheckV2() Feature {
 	return upgradeVersionCheckV2
 }
+
+func UpgradeAlternativeOrder() Feature { return upgradeAlternativeOrder }

pkg/deployment/reconcile/plan_builder_rotate_upgrade.go

@@ -51,6 +51,17 @@ var (
 		api.ServerGroupSyncWorkers,
 		api.ServerGroupGateways,
 	}
+
+	// alternativeUpgradeOrder contains execution order which enforce upgrade of Coordinators before DBServers
+	alternativeUpgradeOrder = []api.ServerGroup{
+		api.ServerGroupAgents,
+		api.ServerGroupSingle,
+		api.ServerGroupCoordinators,
+		api.ServerGroupDBServers,
+		api.ServerGroupSyncMasters,
+		api.ServerGroupSyncWorkers,
+		api.ServerGroupGateways,
+	}
 )
 
 // upgradeDecision is the result of an upgrade check.

pkg/deployment/reconcile/plan_builder_rotate_upgrade_decision.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2024 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@ package reconcile
 
 import (
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/features"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/rotation"
 	"github.com/arangodb/kube-arangodb/pkg/util"
 )
@@ -62,7 +63,10 @@ func (r *Reconciler) createRotateOrUpgradeDecision(spec api.DeploymentSpec, stat
 	d := updateUpgradeDecisionMap{}
 
 	// Init phase
-	for _, m := range status.Members.AsList() {
+
+	upgradeOrder := util.BoolSwitch(features.UpgradeAlternativeOrder().Enabled(), alternativeUpgradeOrder, api.AllServerGroups)
+
+	for _, m := range status.Members.AsListInGroups(upgradeOrder...) {
 		d[m.Member.ID] = r.createRotateOrUpgradeDecisionMember(spec, status, context, m)
 	}
 

pkg/deployment/reconcile/plan_builder_rotate_upgrade_test.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2024 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -31,6 +31,24 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/util"
 )
 
+func Test_EnsureGroupsContainsAll(t *testing.T) {
+	ensure := func(t *testing.T, groups ...api.ServerGroup) {
+		require.Equal(t, groups, util.UniqueList(groups))
+
+		for _, expected := range api.AllServerGroups {
+			t.Run(expected.AsRole(), func(t *testing.T) {
+				require.Contains(t, groups, expected)
+			})
+		}
+	}
+	t.Run("rotationByAnnotationOrder", func(t *testing.T) {
+		ensure(t, rotationByAnnotationOrder...)
+	})
+	t.Run("alternativeUpgradeOrder", func(t *testing.T) {
+		ensure(t, alternativeUpgradeOrder...)
+	})
+}
+
 func Test_RotateUpgrade_Condition(t *testing.T) {
 	type testCase struct {
 		status api.MemberStatus

pkg/util/list.go

@@ -112,6 +112,28 @@ func FormatList[A, B any](in []A, format func(A) B) []B {
 	return r
 }
 
+func ContainsList[A comparable](in []A, item A) bool {
+	for _, el := range in {
+		if el == item {
+			return true
+		}
+	}
+
+	return false
+}
+
+func UniqueList[A comparable](in []A) []A {
+	var r = make([]A, 0, len(in))
+
+	for _, el := range in {
+		if !ContainsList(r, el) {
+			r = append(r, el)
+		}
+	}
+
+	return r
+}
+
 func FormatListErr[A, B any](in []A, format func(A) (B, error)) ([]B, error) {
 	var r = make([]B, len(in))