commit 9eba55e7ee8492c74cba8e5104fc4a092196970d6e12615382a6a23c55a271d9
| author | 斟酌 鵬兄 <tgckpg@gmail.com> |
|---|
| date | 2026-04-21T21:01:48Z |
|---|
| subject | Drop admission logic. Use a plain controller instead |
|---|
commit 9eba55e7ee8492c74cba8e5104fc4a092196970d6e12615382a6a23c55a271d9
Author: 斟酌 鵬兄 <tgckpg@gmail.com>
Date: 2026-04-21T21:01:48Z
Drop admission logic. Use a plain controller instead
---
clitools/devtools/gen-admission-payload.sh | 52 ----
clitools/devtools/run-controller.sh | 2 +-
clitools/pkg/apis/monok8s/v1alpha1/osupgrade.go | 30 +-
.../apis/monok8s/v1alpha1/zz_generated.deepcopy.go | 16 -
clitools/pkg/cmd/controller/controller.go | 115 +++++--
clitools/pkg/controller/admission/server.go | 157 ----------
clitools/pkg/controller/osupgrade/watch.go | 346 +++++++++++++++++++++
clitools/pkg/controller/server.go | 92 ++++++
devtools/test-upgrade.sh | 1 -
9 files changed, 535 insertions(+), 276 deletions(-)
diff --git a/clitools/devtools/gen-admission-payload.sh b/clitools/devtools/gen-admission-payload.sh
deleted file mode 100755
index 7b0232f..0000000
--- a/clitools/devtools/gen-admission-payload.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/usr/bin/env bash
-
-# Example: pipe this into curl
-# ../../devtools/test-upgrade.sh \
-# | this_script
-# | curl -k -H 'Content-Type: application/json' --data-binary @- https://127.0.0.1:8443/admission
-
-set -euo pipefail
-
-uid=$(uuidgen 2>/dev/null)
-
-yq -o=json '.' | jq --arg uid "$uid" '
-{
- apiVersion: "admission.k8s.io/v1",
- kind: "AdmissionReview",
- request: {
- uid: $uid,
- kind: {
- group: "monok8s.io",
- version: "v1alpha1",
- kind: "OSUpgrade"
- },
- resource: {
- group: "monok8s.io",
- version: "v1alpha1",
- resource: "osupgrades"
- },
- requestKind: {
- group: "monok8s.io",
- version: "v1alpha1",
- kind: "OSUpgrade"
- },
- requestResource: {
- group: "monok8s.io",
- version: "v1alpha1",
- resource: "osupgrades"
- },
- name: .metadata.name,
- namespace: (.metadata.namespace // ""),
- operation: "CREATE",
- userInfo: {
- username: "debug-user"
- },
- object: .,
- oldObject: null,
- dryRun: false,
- options: {
- apiVersion: "meta.k8s.io/v1",
- kind: "CreateOptions"
- }
- }
-}'
diff --git a/clitools/devtools/run-controller.sh b/clitools/devtools/run-controller.sh
index 395b2f1..8e70f15 100755
--- a/clitools/devtools/run-controller.sh
+++ b/clitools/devtools/run-controller.sh
@@ -14,4 +14,4 @@ else
-addext "subjectAltName=IP:127.0.0.1,DNS:localhost"
fi
-go run "$PROJ_ROOT"/cmd/ctl controller --tls-cert-file "$OUT_DIR"/tls.crt --tls-private-key-file "$OUT_DIR"/tls.key
+go run "$PROJ_ROOT"/cmd/ctl controller --tls-cert-file "$OUT_DIR"/tls.crt --tls-private-key-file "$OUT_DIR"/tls.key --namespace default
diff --git a/clitools/pkg/apis/monok8s/v1alpha1/osupgrade.go b/clitools/pkg/apis/monok8s/v1alpha1/osupgrade.go
index a0f8881..0f58277 100644
--- a/clitools/pkg/apis/monok8s/v1alpha1/osupgrade.go
+++ b/clitools/pkg/apis/monok8s/v1alpha1/osupgrade.go
@@ -7,11 +7,9 @@ import (
type OSUpgradePhase string
const (
- OSUpgradePhasePending OSUpgradePhase = "Pending"
- OSUpgradePhaseAccepted OSUpgradePhase = "Accepted"
- OSUpgradePhaseRollingOut OSUpgradePhase = "RollingOut"
- OSUpgradePhaseCompleted OSUpgradePhase = "Completed"
- OSUpgradePhaseRejected OSUpgradePhase = "Rejected"
+ OSUpgradePhasePending OSUpgradePhase = "Pending"
+ OSUpgradePhaseAccepted OSUpgradePhase = "Accepted"
+ OSUpgradePhaseRejected OSUpgradePhase = "Rejected"
)
type OSUpgradeProgressPhase string
@@ -34,9 +32,6 @@ const (
// +kubebuilder:printcolumn:name="Desired",type=string,JSONPath=`.spec.desiredVersion`
// +kubebuilder:printcolumn:name="Resolved",type=string,JSONPath=`.status.resolvedVersion`
// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase`
-// +kubebuilder:printcolumn:name="Targets",type=integer,JSONPath=`.status.summary.targetedNodes`
-// +kubebuilder:printcolumn:name="OK",type=integer,JSONPath=`.status.summary.succeededNodes`
-// +kubebuilder:printcolumn:name="Fail",type=integer,JSONPath=`.status.summary.failedNodes`
type OSUpgrade struct {
metav1.TypeMeta `json:",inline" yaml:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"`
@@ -74,20 +69,11 @@ type OSUpgradeStatus struct {
Phase OSUpgradePhase `json:"phase,omitempty" yaml:"phase,omitempty"`
ResolvedVersion string `json:"resolvedVersion,omitempty" yaml:"resolvedVersion,omitempty"`
ObservedGeneration int64 `json:"observedGeneration,omitempty" yaml:"observedGeneration,omitempty"`
- Summary OSUpgradeSummary `json:"summary,omitempty" yaml:"summary,omitempty"`
Conditions []metav1.Condition `json:"conditions,omitempty" yaml:"conditions,omitempty"`
Reason string `json:"reason,omitempty" yaml:"reason,omitempty"`
Message string `json:"message,omitempty" yaml:"message,omitempty"`
}
-type OSUpgradeSummary struct {
- TargetedNodes int32 `json:"targetedNodes,omitempty" yaml:"targetedNodes,omitempty"`
- PendingNodes int32 `json:"pendingNodes,omitempty" yaml:"pendingNodes,omitempty"`
- RunningNodes int32 `json:"runningNodes,omitempty" yaml:"runningNodes,omitempty"`
- SucceededNodes int32 `json:"succeededNodes,omitempty" yaml:"succeededNodes,omitempty"`
- FailedNodes int32 `json:"failedNodes,omitempty" yaml:"failedNodes,omitempty"`
-}
-
// +genclient
// +kubebuilder:object:root=true
// +kubebuilder:subresource:status
@@ -96,7 +82,7 @@ type OSUpgradeSummary struct {
// +kubebuilder:printcolumn:name="Source",type=string,JSONPath=`.spec.sourceRef.name`
// +kubebuilder:printcolumn:name="Current",type=string,JSONPath=`.status.currentVersion`
// +kubebuilder:printcolumn:name="Target",type=string,JSONPath=`.status.targetVersion`
-// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.phase`
+// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase`
type OSUpgradeProgress struct {
metav1.TypeMeta `json:",inline" yaml:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"`
@@ -138,3 +124,11 @@ type OSUpgradeProgressStatus struct {
CurrentFrom string `json:"currentFrom,omitempty" yaml:"currentFrom,omitempty"`
CurrentTo string `json:"currentTo,omitempty" yaml:"currentTo,omitempty"`
}
+
+func (osu OSUpgrade) StatusPhase() string {
+ phase := ""
+ if osu.Status != nil {
+ phase = string(osu.Status.Phase)
+ }
+ return phase
+}
diff --git a/clitools/pkg/apis/monok8s/v1alpha1/zz_generated.deepcopy.go b/clitools/pkg/apis/monok8s/v1alpha1/zz_generated.deepcopy.go
index df833d7..94a75a6 100644
--- a/clitools/pkg/apis/monok8s/v1alpha1/zz_generated.deepcopy.go
+++ b/clitools/pkg/apis/monok8s/v1alpha1/zz_generated.deepcopy.go
@@ -360,7 +360,6 @@ func (in *OSUpgradeSpec) DeepCopy() *OSUpgradeSpec {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *OSUpgradeStatus) DeepCopyInto(out *OSUpgradeStatus) {
*out = *in
- out.Summary = in.Summary
if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions
*out = make([]v1.Condition, len(*in))
@@ -380,21 +379,6 @@ func (in *OSUpgradeStatus) DeepCopy() *OSUpgradeStatus {
return out
}
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *OSUpgradeSummary) DeepCopyInto(out *OSUpgradeSummary) {
- *out = *in
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OSUpgradeSummary.
-func (in *OSUpgradeSummary) DeepCopy() *OSUpgradeSummary {
- if in == nil {
- return nil
- }
- out := new(OSUpgradeSummary)
- in.DeepCopyInto(out)
- return out
-}
-
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *VersionCatalogSource) DeepCopyInto(out *VersionCatalogSource) {
*out = *in
diff --git a/clitools/pkg/cmd/controller/controller.go b/clitools/pkg/cmd/controller/controller.go
index 88b57d6..fb4bb72 100644
--- a/clitools/pkg/cmd/controller/controller.go
+++ b/clitools/pkg/cmd/controller/controller.go
@@ -2,6 +2,7 @@ package controller
import (
"context"
+ "errors"
"net"
"net/http"
"os"
@@ -11,44 +12,78 @@ import (
"k8s.io/cli-runtime/pkg/genericclioptions"
"k8s.io/klog/v2"
- mksadmission "example.com/monok8s/pkg/controller/admission"
+ mkscontroller "example.com/monok8s/pkg/controller"
+ osupgradectrl "example.com/monok8s/pkg/controller/osupgrade"
"example.com/monok8s/pkg/kube"
"example.com/monok8s/pkg/templates"
)
type ServerConfig struct {
+ Namespace string `json:"namespace,omitempty"`
TLSCertFile string `json:"tlsCertFile,omitempty"`
TLSPrivateKeyFile string `json:"tlsPrivateKeyFile,omitempty"`
}
func NewCmdController(flags *genericclioptions.ConfigFlags) *cobra.Command {
- var namespace string = templates.DefaultNamespace
var conf ServerConfig
cmd := &cobra.Command{
Use: "controller",
- Short: "Admission controller that handles OSUpgrade resources",
+ Short: "Start a controller that handles OSUpgrade resources",
RunE: func(cmd *cobra.Command, _ []string) error {
-
ctx := cmd.Context()
- klog.InfoS("starting controller",
- "namespace", namespace,
- )
+ klog.InfoS("starting controller", "namespace", conf.Namespace)
clients, err := kube.NewClients(flags)
if err != nil {
return err
}
- return httpListen(ctx, clients, conf)
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+
+ httpErrCh := make(chan error, 1)
+ watchErrCh := make(chan error, 1)
+
+ go func() {
+ klog.InfoS("starting OSUpgrade watch loop", "namespace", conf.Namespace)
+ watchErrCh <- osupgradectrl.Watch(ctx, clients, conf.Namespace)
+ }()
+
+ go func() {
+ httpErrCh <- httpListen(ctx, clients, conf)
+ }()
+
+ select {
+ case <-ctx.Done():
+ klog.InfoS("controller context canceled")
+ return ctx.Err()
+
+ case err := <-watchErrCh:
+ if err != nil && !errors.Is(err, context.Canceled) {
+ cancel()
+ return err
+ }
+ cancel()
+ return nil
+
+ case err := <-httpErrCh:
+ if err != nil && !errors.Is(err, context.Canceled) {
+ cancel()
+ return err
+ }
+ cancel()
+ return nil
+ }
},
}
- cmd.Flags().StringVar(&namespace, "namespace", templates.DefaultNamespace, "namespace to watch")
+ cmd.Flags().StringVar(&conf.Namespace, "namespace", templates.DefaultNamespace, "namespace to watch")
cmd.Flags().StringVar(&conf.TLSCertFile, "tls-cert-file", conf.TLSCertFile,
"File containing x509 Certificate used for serving HTTPS (with intermediate certs, if any, concatenated after server cert).")
- cmd.Flags().StringVar(&conf.TLSPrivateKeyFile, "tls-private-key-file", conf.TLSPrivateKeyFile, "File containing x509 private key matching --tls-cert-file.")
+ cmd.Flags().StringVar(&conf.TLSPrivateKeyFile, "tls-private-key-file", conf.TLSPrivateKeyFile,
+ "File containing x509 private key matching --tls-cert-file.")
return cmd
}
@@ -57,41 +92,59 @@ func httpListen(ctx context.Context, clients *kube.Clients, conf ServerConfig) e
address, port := "", "8443"
addr := net.JoinHostPort(address, port)
- ns := os.Getenv("POD_NAMESPACE")
nodeName := os.Getenv("NODE_NAME")
- server := mksadmission.NewServer(ctx, clients, ns, nodeName)
+ server := mkscontroller.NewServer(ctx, clients, conf.Namespace, nodeName)
s := &http.Server{
Addr: addr,
Handler: server,
IdleTimeout: 90 * time.Second,
- ReadTimeout: 4 * 60 * time.Minute,
- WriteTimeout: 4 * 60 * time.Minute,
+ ReadTimeout: 4 * time.Minute,
+ WriteTimeout: 4 * time.Minute,
MaxHeaderBytes: 1 << 20,
}
- if conf.TLSCertFile != "" {
- klog.InfoS("starting HTTPS server",
- "addr", addr,
- "certFile", conf.TLSCertFile,
- "keyFile", conf.TLSPrivateKeyFile,
- )
+ serverErrCh := make(chan error, 1)
+
+ go func() {
+ if conf.TLSCertFile != "" {
+ klog.InfoS("starting HTTPS server",
+ "addr", addr,
+ "certFile", conf.TLSCertFile,
+ "keyFile", conf.TLSPrivateKeyFile,
+ )
+ serverErrCh <- s.ListenAndServeTLS(conf.TLSCertFile, conf.TLSPrivateKeyFile)
+ return
+ }
+
+ klog.InfoS("starting HTTP server", "addr", addr)
+ serverErrCh <- s.ListenAndServe()
+ }()
+
+ select {
+ case <-ctx.Done():
+ klog.InfoS("shutting down HTTP server", "addr", addr)
+
+ shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
- if err := s.ListenAndServeTLS(conf.TLSCertFile, conf.TLSPrivateKeyFile); err != nil {
- klog.ErrorS(err, "HTTPS server failed")
- os.Exit(1)
+ err := s.Shutdown(shutdownCtx)
+ if err != nil {
+ return err
}
- } else {
- klog.InfoS("starting HTTP server",
- "addr", addr,
- )
- if err := s.ListenAndServe(); err != nil {
+ err = <-serverErrCh
+ if err != nil && !errors.Is(err, http.ErrServerClosed) {
+ return err
+ }
+ return context.Canceled
+
+ case err := <-serverErrCh:
+ if err != nil && !errors.Is(err, http.ErrServerClosed) {
klog.ErrorS(err, "HTTP server failed")
- os.Exit(1)
+ return err
}
+ return nil
}
-
- return nil
}
diff --git a/clitools/pkg/controller/admission/server.go b/clitools/pkg/controller/admission/server.go
deleted file mode 100644
index 84ab259..0000000
--- a/clitools/pkg/controller/admission/server.go
+++ /dev/null
@@ -1,157 +0,0 @@
-package admission
-
-import (
- "context"
- "io"
- "net/http"
-
- admissionv1 "k8s.io/api/admission/v1"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
- monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1"
- "example.com/monok8s/pkg/controller/osupgrade"
- "example.com/monok8s/pkg/kube"
-
- "github.com/emicklei/go-restful/v3"
- "k8s.io/apimachinery/pkg/runtime"
- "k8s.io/apimachinery/pkg/runtime/serializer"
- "k8s.io/apiserver/pkg/server/httplog"
- "k8s.io/klog/v2"
-)
-
-var (
- scheme = runtime.NewScheme()
- codecs = serializer.NewCodecFactory(scheme)
- deserializer = codecs.UniversalDeserializer()
-)
-
-var statusesNoTracePred = httplog.StatusIsNot(
- http.StatusOK,
- http.StatusFound,
- http.StatusMovedPermanently,
- http.StatusTemporaryRedirect,
- http.StatusBadRequest,
- http.StatusNotFound,
- http.StatusSwitchingProtocols,
-)
-
-func init() {
- _ = admissionv1.AddToScheme(scheme)
- _ = monov1alpha1.AddToScheme(scheme)
-}
-
-type Server struct {
- restfulCont *restful.Container
-
- ctx context.Context
- clients *kube.Clients
- namespace string
- nodeName string
-}
-
-func NewServer(ctx context.Context, clients *kube.Clients, namespace, nodeName string) *Server {
- s := &Server{
- ctx: ctx,
- clients: clients,
- namespace: namespace,
- nodeName: nodeName,
- }
- s.Initialize()
- return s
-}
-
-func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
- if s == nil {
- http.Error(w, "admission server is nil", http.StatusInternalServerError)
- return
- }
- if s.restfulCont == nil {
- http.Error(w, "admission server not initialized", http.StatusInternalServerError)
- return
- }
-
- handler := httplog.WithLogging(s.restfulCont, statusesNoTracePred)
- handler.ServeHTTP(w, req)
-}
-
-func (s *Server) Initialize() {
- s.restfulCont = restful.NewContainer()
-
- ws := new(restful.WebService)
-
- ws.Path("/admission").
- Consumes(restful.MIME_JSON).
- Produces(restful.MIME_JSON)
-
- ws.Route(ws.POST("").To(s.triggerAdmission).
- Reads(admissionv1.AdmissionReview{}).
- Writes(admissionv1.AdmissionReview{}))
-
- s.restfulCont.Add(ws)
-}
-
-func (s *Server) triggerAdmission(request *restful.Request, response *restful.Response) {
- body, err := io.ReadAll(request.Request.Body)
- if err != nil {
- _ = response.WriteError(http.StatusBadRequest, err)
- return
- }
-
- var reviewReq admissionv1.AdmissionReview
- if _, _, err := deserializer.Decode(body, nil, &reviewReq); err != nil {
- _ = response.WriteError(http.StatusBadRequest, err)
- return
- }
- if reviewReq.Request == nil {
- _ = response.WriteErrorString(http.StatusBadRequest, "missing admission request")
- return
- }
-
- resp := admissionv1.AdmissionReview{
- TypeMeta: metav1.TypeMeta{
- Kind: "AdmissionReview",
- APIVersion: "admission.k8s.io/v1",
- },
- Response: &admissionv1.AdmissionResponse{
- UID: reviewReq.Request.UID,
- Allowed: true,
- Result: &metav1.Status{Message: "OK"},
- },
- }
-
- var osu monov1alpha1.OSUpgrade
- if _, _, err := deserializer.Decode(reviewReq.Request.Object.Raw, nil, &osu); err != nil {
- klog.V(1).InfoS("Skipping non-OSUpgrade resource",
- "uid", reviewReq.Request.UID,
- "kind", reviewReq.Request.Kind.Kind,
- "operation", reviewReq.Request.Operation,
- "err", err,
- )
- _ = response.WriteEntity(resp)
- return
- }
-
- klog.InfoS("Received OSUpgrade admission",
- "uid", reviewReq.Request.UID,
- "operation", reviewReq.Request.Operation,
- "name", osu.Name,
- "namespace", osu.Namespace,
- "node", s.nodeName,
- )
-
- // Resolve every node name
- if err := osupgrade.EnsureOSUpgradeProgressForNode(
- s.ctx,
- s.clients,
- s.namespace,
- s.nodeName,
- &osu,
- ); err != nil {
- klog.ErrorS(err, "ensure OSUpgradeProgress for node failed",
- "osupgrade", osu.Name,
- "node", s.nodeName,
- )
- }
-
- _ = response.WriteEntity(resp)
-}
diff --git a/clitools/pkg/controller/osupgrade/watch.go b/clitools/pkg/controller/osupgrade/watch.go
new file mode 100644
index 0000000..8ff8bf4
--- /dev/null
+++ b/clitools/pkg/controller/osupgrade/watch.go
@@ -0,0 +1,346 @@
+package osupgrade
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ corev1 "k8s.io/api/core/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/labels"
+ "k8s.io/apimachinery/pkg/watch"
+ "k8s.io/klog/v2"
+
+ monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1"
+ "example.com/monok8s/pkg/kube"
+)
+
+func Watch(ctx context.Context, clients *kube.Clients, namespace string) error {
+ var resourceVersion string
+
+ for {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+
+ err := watchOnce(ctx, clients, namespace, &resourceVersion)
+ if err != nil {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+
+ // Expired RV is normal enough; clear it and relist.
+ if apierrors.IsResourceExpired(err) {
+ klog.InfoS("OSUpgrade watch resourceVersion expired; resetting",
+ "namespace", namespace,
+ "resourceVersion", resourceVersion,
+ )
+ resourceVersion = ""
+ } else {
+ klog.ErrorS(err, "OSUpgrade watch failed; retrying",
+ "namespace", namespace,
+ "resourceVersion", resourceVersion,
+ )
+ }
+
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-time.After(2 * time.Second):
+ }
+ continue
+ }
+ }
+}
+
+func watchOnce(
+ ctx context.Context,
+ clients *kube.Clients,
+ namespace string,
+ resourceVersion *string,
+) error {
+ // Cold start: list existing objects once, handle them, then watch from list RV.
+ if *resourceVersion == "" {
+ list, err := clients.MonoKS.
+ Monok8sV1alpha1().
+ OSUpgrades(namespace).
+ List(ctx, metav1.ListOptions{})
+ if err != nil {
+ return fmt.Errorf("list OSUpgrades: %w", err)
+ }
+
+ for i := range list.Items {
+ osu := &list.Items[i]
+ handled, err := handleOSUpgrade(ctx, clients, namespace, osu)
+ if err != nil {
+ klog.ErrorS(err, "reconcile existing OSUpgrade failed",
+ "name", osu.Name,
+ "resourceVersion", osu.ResourceVersion,
+ )
+ continue
+ }
+ if !handled {
+ klog.V(2).InfoS("skipping existing OSUpgrade",
+ "name", osu.Name,
+ "phase", osu.StatusPhase(),
+ )
+ }
+ }
+
+ *resourceVersion = list.ResourceVersion
+ klog.InfoS("initial OSUpgrade sync complete",
+ "namespace", namespace,
+ "resourceVersion", *resourceVersion,
+ "count", len(list.Items),
+ )
+ }
+
+ w, err := clients.MonoKS.
+ Monok8sV1alpha1().
+ OSUpgrades(namespace).
+ Watch(ctx, metav1.ListOptions{
+ ResourceVersion: *resourceVersion,
+ AllowWatchBookmarks: true,
+ })
+ if err != nil {
+ return fmt.Errorf("watch OSUpgrades: %w", err)
+ }
+ defer w.Stop()
+
+ klog.InfoS("watching OSUpgrades",
+ "namespace", namespace,
+ "resourceVersion", *resourceVersion,
+ )
+
+ for {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+
+ case evt, ok := <-w.ResultChan():
+ if !ok {
+ return fmt.Errorf("watch channel closed")
+ }
+
+ switch evt.Type {
+ case watch.Bookmark:
+ if rv := extractResourceVersion(evt.Object); rv != "" {
+ *resourceVersion = rv
+ }
+ continue
+
+ case watch.Error:
+ // Let outer loop retry / relist.
+ return fmt.Errorf("watch returned error event")
+
+ case watch.Deleted:
+ // Top-level delete does not require action here.
+ continue
+
+ case watch.Added, watch.Modified:
+ // handled below
+
+ default:
+ klog.V(1).InfoS("skipping unexpected watch event type",
+ "eventType", evt.Type,
+ )
+ continue
+ }
+
+ osu, ok := evt.Object.(*monov1alpha1.OSUpgrade)
+ if !ok {
+ klog.V(1).InfoS("skipping unexpected watch object type",
+ "type", fmt.Sprintf("%T", evt.Object),
+ )
+ continue
+ }
+
+ if osu.ResourceVersion != "" {
+ *resourceVersion = osu.ResourceVersion
+ }
+
+ handled, err := handleOSUpgrade(ctx, clients, namespace, osu)
+ if err != nil {
+ klog.ErrorS(err, "reconcile OSUpgrade failed",
+ "name", osu.Name,
+ "eventType", evt.Type,
+ "resourceVersion", osu.ResourceVersion,
+ )
+ continue
+ }
+ if !handled {
+ klog.V(2).InfoS("skipping OSUpgrade",
+ "name", osu.Name,
+ "eventType", evt.Type,
+ "phase", osu.StatusPhase(),
+ )
+ }
+ }
+ }
+}
+
+func handleOSUpgrade(
+ ctx context.Context,
+ clients *kube.Clients,
+ namespace string,
+ osu *monov1alpha1.OSUpgrade,
+) (bool, error) {
+ if !shouldHandle(osu) {
+ return false, nil
+ }
+
+ if osu.Status == nil || osu.Status.ObservedGeneration != osu.Generation {
+ return true, reconcileSpec(ctx, clients, namespace, osu)
+ }
+
+ if osu.Status.Phase == monov1alpha1.OSUpgradePhaseAccepted {
+ return true, reconcileFanout(ctx, clients, namespace, osu)
+ }
+
+ return false, nil
+}
+
+func reconcileSpec(
+ ctx context.Context,
+ clients *kube.Clients,
+ namespace string,
+ osu *monov1alpha1.OSUpgrade,
+) error {
+ osu = osu.DeepCopy()
+
+ osu.Status = &monov1alpha1.OSUpgradeStatus{
+ Phase: monov1alpha1.OSUpgradePhaseAccepted,
+ ResolvedVersion: osu.Spec.DesiredVersion,
+ ObservedGeneration: osu.Generation,
+ }
+
+ _, err := clients.MonoKS.
+ Monok8sV1alpha1().
+ OSUpgrades(namespace).
+ UpdateStatus(ctx, osu, metav1.UpdateOptions{})
+
+ return err
+}
+
+func reconcileFanout(
+ ctx context.Context,
+ clients *kube.Clients,
+ namespace string,
+ osu *monov1alpha1.OSUpgrade,
+) error {
+ nodeNames, err := listTargetNodeNames(ctx, clients, osu)
+ if err != nil {
+ return fmt.Errorf("list target nodes for %s: %w", osu.Name, err)
+ }
+
+ if len(nodeNames) == 0 {
+ klog.InfoS("no targets", "osupgrade", osu.Name)
+ return nil
+ }
+
+ klog.InfoS("ensuring OSUpgradeProgress for target nodes",
+ "osupgrade", osu.Name,
+ "targets", len(nodeNames),
+ )
+
+ for _, nodeName := range nodeNames {
+ if err := EnsureOSUpgradeProgressForNode(
+ ctx,
+ clients,
+ namespace,
+ nodeName,
+ osu,
+ ); err != nil {
+ klog.ErrorS(err, "ensure OSUpgradeProgress for node failed",
+ "osupgrade", osu.Name,
+ "node", nodeName,
+ )
+ }
+ }
+
+ return nil
+}
+
+func listTargetNodeNames(
+ ctx context.Context,
+ clients *kube.Clients,
+ osu *monov1alpha1.OSUpgrade,
+) ([]string, error) {
+ selector := labels.SelectorFromSet(labels.Set{
+ monov1alpha1.ControlAgentKey: "true",
+ })
+
+ if osu.Spec.NodeSelector != nil {
+ sel, err := metav1.LabelSelectorAsSelector(osu.Spec.NodeSelector)
+ if err != nil {
+ return nil, fmt.Errorf("invalid nodeSelector: %w", err)
+ }
+
+ reqs, selectable := sel.Requirements()
+ if !selectable {
+ return nil, fmt.Errorf("nodeSelector is not selectable")
+ }
+
+ selector = selector.Add(reqs...)
+ }
+
+ list, err := clients.Kubernetes.CoreV1().
+ Nodes().
+ List(ctx, metav1.ListOptions{
+ LabelSelector: selector.String(),
+ })
+ if err != nil {
+ return nil, fmt.Errorf("list nodes: %w", err)
+ }
+
+ out := make([]string, 0, len(list.Items))
+ for i := range list.Items {
+ node := &list.Items[i]
+ if shouldUseNode(node) {
+ out = append(out, node.Name)
+ }
+ }
+
+ return out, nil
+}
+
+func shouldUseNode(node *corev1.Node) bool {
+ // Keep this conservative for now. Tighten if you want only Ready nodes.
+ return node != nil && node.Name != ""
+}
+
+func shouldHandle(osu *monov1alpha1.OSUpgrade) bool {
+ if osu == nil || osu.DeletionTimestamp != nil {
+ return false
+ }
+
+ if osu.Spec.DesiredVersion == "" {
+ return false
+ }
+
+ // NEW: initial processing stage
+ if osu.Status == nil {
+ return true
+ }
+
+ // Reconcile if spec changed
+ if osu.Status.ObservedGeneration != osu.Generation {
+ return true
+ }
+
+ // Fanout stage
+ return osu.Status.Phase == monov1alpha1.OSUpgradePhaseAccepted
+}
+
+func extractResourceVersion(obj interface{}) string {
+ type hasRV interface {
+ GetResourceVersion() string
+ }
+
+ if o, ok := obj.(hasRV); ok {
+ return o.GetResourceVersion()
+ }
+
+ return ""
+}
diff --git a/clitools/pkg/controller/server.go b/clitools/pkg/controller/server.go
new file mode 100644
index 0000000..9feb297
--- /dev/null
+++ b/clitools/pkg/controller/server.go
@@ -0,0 +1,92 @@
+package controller
+
+import (
+ "context"
+ "net/http"
+ "time"
+
+ "example.com/monok8s/pkg/kube"
+
+ "github.com/emicklei/go-restful/v3"
+ "k8s.io/apiserver/pkg/server/httplog"
+)
+
+var statusesNoTracePred = httplog.StatusIsNot(
+ http.StatusOK,
+ http.StatusFound,
+ http.StatusMovedPermanently,
+ http.StatusTemporaryRedirect,
+ http.StatusBadRequest,
+ http.StatusNotFound,
+ http.StatusSwitchingProtocols,
+)
+
+type Server struct {
+ restfulCont *restful.Container
+
+ ctx context.Context
+ clients *kube.Clients
+ namespace string
+ nodeName string
+ startedAt time.Time
+}
+
+type StatusResponse struct {
+ OK bool `json:"ok"`
+ Service string `json:"service"`
+ Namespace string `json:"namespace,omitempty"`
+ NodeName string `json:"nodeName,omitempty"`
+ UptimeSec int64 `json:"uptimeSec"`
+}
+
+func NewServer(ctx context.Context, clients *kube.Clients, namespace, nodeName string) *Server {
+ s := &Server{
+ ctx: ctx,
+ clients: clients,
+ namespace: namespace,
+ nodeName: nodeName,
+ startedAt: time.Now(),
+ }
+ s.Initialize()
+ return s
+}
+
+func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+ if s == nil {
+ http.Error(w, "server is nil", http.StatusInternalServerError)
+ return
+ }
+ if s.restfulCont == nil {
+ http.Error(w, "server not initialized", http.StatusInternalServerError)
+ return
+ }
+
+ handler := httplog.WithLogging(s.restfulCont, statusesNoTracePred)
+ handler.ServeHTTP(w, req)
+}
+
+func (s *Server) Initialize() {
+ s.restfulCont = restful.NewContainer()
+
+ ws := new(restful.WebService)
+ ws.Path("/")
+ ws.Consumes(restful.MIME_JSON)
+ ws.Produces(restful.MIME_JSON)
+
+ ws.Route(ws.GET("/status").To(s.queryStatus).
+ Doc("Return basic controller status"))
+
+ s.restfulCont.Add(ws)
+}
+
+func (s *Server) queryStatus(request *restful.Request, response *restful.Response) {
+ resp := StatusResponse{
+ OK: true,
+ Service: "monok8s-controller",
+ Namespace: s.namespace,
+ NodeName: s.nodeName,
+ UptimeSec: int64(time.Since(s.startedAt).Seconds()),
+ }
+
+ _ = response.WriteHeaderAndEntity(http.StatusOK, resp)
+}
diff --git a/devtools/test-upgrade.sh b/devtools/test-upgrade.sh
index a1dd04e..11e4eec 100755
--- a/devtools/test-upgrade.sh
+++ b/devtools/test-upgrade.sh
@@ -16,7 +16,6 @@ echo "metadata:"
echo " name: \"$NAME\""
echo "spec:"
echo " desiredVersion: \"$TARGET_VERSION\""
-echo " imageURL: \"$BASE_URL/monok8s-$TARGET_VERSION-dev.ext4.zst\""
echo " nodeSelector: {}"
echo " catalog:"
echo " inline: |"