commit 65c643d7a2542286e6970915829e2c3254a6ab43305ffd8f6c69c4d76960552c
| author | 斟酌 鵬兄 <tgckpg@gmail.com> |
|---|
| date | 2026-04-16T02:44:42Z |
|---|
| subject | Added migrations.d |
|---|
commit 65c643d7a2542286e6970915829e2c3254a6ab43305ffd8f6c69c4d76960552c
Author: 斟酌 鵬兄 <tgckpg@gmail.com>
Date: 2026-04-16T02:44:42Z
Added migrations.d
---
README.md | 8 +-
alpine/build-rootfs.sh | 10 ++
.../k8s/1.35/10-drop-removed-kubelet-pause-flag.sh | 35 +++++
alpine/rootfs-extra/etc/local.d/monok8s.start | 35 +++--
.../rootfs-extra/usr/lib/monok8s/lib/migrations.sh | 58 ++++++++
clitools/pkg/cmd/version/version.go | 64 ++++++++-
.../pkg/controller/osimage/write_throttle_other.go | 3 +-
clitools/pkg/controller/osupgrade/planner.go | 21 ++-
clitools/pkg/controller/osupgrade/planner_test.go | 149 +++++++++++++++++++++
clitools/pkg/node/fs.go | 6 +-
docker/alpine.Dockerfile | 1 +
11 files changed, 366 insertions(+), 24 deletions(-)
diff --git a/README.md b/README.md
index 9c643a8..4c736c5 100644
--- a/README.md
+++ b/README.md
@@ -26,13 +26,11 @@ https://docs.mono.si/gateway-development-kit/getting-started
### Kubernetes
* OSUpgrade
* [x] Control Plane - kubeadm upgrade apply
- * [ ] Worker node - kubeadm upgrade node
* Upgrade chain
* [x] 1.33.3 -> 1.33.10
- * [ ] 1.33.10 -> 1.34.1
- * [ ] 1.34.1 -> 1.34.6
- * [ ] 1.34.6 -> 1.35.1
- * [ ] 1.35.1 -> 1.35.3
+ * [x] 1.33.10 -> 1.34.6
+ * [x] 1.34.6 -> 1.35.3
+ * [ ] Worker node - kubeadm upgrade node
* CNI
* [x] default bridge-cni
* [ ] Cilium
diff --git a/alpine/build-rootfs.sh b/alpine/build-rootfs.sh
index 2978a99..f9d4a3f 100755
--- a/alpine/build-rootfs.sh
+++ b/alpine/build-rootfs.sh
@@ -17,6 +17,7 @@ mkdir -p \
"$ROOTFS/build" \
"$ROOTFS/var/cache/apk" \
"$ROOTFS/usr/lib/monok8s/crds" \
+ "$ROOTFS/usr/lib/monok8s/migrations.d/k8s" \
"$ROOTFS/opt/monok8s/config"
mount --bind /var/cache/apk "$ROOTFS/var/cache/apk"
@@ -30,6 +31,15 @@ cp /etc/resolv.conf "$ROOTFS/etc/resolv.conf"
cp /build/crio.tar.gz "$ROOTFS/build/"
cp /build/crds/*.yaml "$ROOTFS/usr/lib/monok8s/crds"
+KUBE_MINOR=$(printf '%s\n' "$KUBE_VERSION" | sed -E 's/^v?([0-9]+\.[0-9]+).*/\1/')
+MIG_SRC="/build/migrations/k8s/$KUBE_MINOR"
+MIG_DST="$ROOTFS/usr/lib/monok8s/migrations.d/k8s/$KUBE_MINOR"
+
+if [ -d "$MIG_SRC" ]; then
+ mkdir -p "$MIG_DST"
+ cp -a "$MIG_SRC"/. "$MIG_DST"/
+fi
+
chroot "$ROOTFS" /bin/sh -c "ln -s /var/cache/apk /etc/apk/cache"
# chroot "$ROOTFS" /bin/sh -c "apk update"
chroot "$ROOTFS" /bin/sh -c "apk add bash curl"
diff --git a/alpine/migrations/k8s/1.35/10-drop-removed-kubelet-pause-flag.sh b/alpine/migrations/k8s/1.35/10-drop-removed-kubelet-pause-flag.sh
new file mode 100755
index 0000000..c4d73c2
--- /dev/null
+++ b/alpine/migrations/k8s/1.35/10-drop-removed-kubelet-pause-flag.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+set -eu
+
+# Kubernetes removed the kubelet flag:
+# --pod-infra-container-image
+#
+# Timeline:
+# - Deprecated before v1.27
+# - Removed in newer kubelet versions (>=1.27+)
+# - kubeadm may still write it into:
+# /var/lib/kubelet/kubeadm-flags.env
+#
+# This causes kubelet to fail with:
+# "unknown flag: --pod-infra-container-image"
+#
+# References:
+# - https://github.com/kubernetes/kubeadm/issues/3281
+# - https://github.com/kubernetes/kubernetes/pull/122739
+# - https://kubernetes.io/blog/2022/04/07/upcoming-changes-in-kubernetes-1-24/
+#
+# Root cause:
+# - Sandbox (pause) image is now managed by CRI (containerd/CRI-O),
+# not kubelet flags.
+#
+# Fix:
+# - Strip the flag from kubeadm-flags.env during upgrade
+
+FILE=/var/lib/kubelet/kubeadm-flags.env
+
+[ -f "$FILE" ] || exit 0
+grep -q -- '--pod-infra-container-image=' "$FILE" || exit 0
+
+sed -i 's/ --pod-infra-container-image=[^"]*//g' "$FILE"
+
+echo "Removed deprecated kubelet flag --pod-infra-container-image from $FILE"
diff --git a/alpine/rootfs-extra/etc/local.d/monok8s.start b/alpine/rootfs-extra/etc/local.d/monok8s.start
index 2f88bd9..4e76376 100755
--- a/alpine/rootfs-extra/etc/local.d/monok8s.start
+++ b/alpine/rootfs-extra/etc/local.d/monok8s.start
@@ -5,22 +5,37 @@ exec >>/var/log/monok8s/boot.log 2>&1
export PATH="/usr/local/bin:/usr/local/sbin:$PATH"
+MIGRATIONS_LIB=/usr/lib/monok8s/lib/migrations.sh
+CONFIG_DIR=/opt/monok8s/config
+BOOT_STATE=/run/monok8s/boot-state.env
+BOOTPART_FILE="$CONFIG_DIR/.bootpart"
+MIGRATION_STATE_DIR="$CONFIG_DIR/migration-state"
+
mkdir -p /dev/hugepages
mountpoint -q /dev/hugepages || mount -t hugetlbfs none /dev/hugepages
-echo 640 > /proc/sys/vm/nr_hugepages
+echo 256 > /proc/sys/vm/nr_hugepages
-CUR=$(grep '^BOOT_PART=' /run/monok8s/boot-state.env | cut -d= -f2-)
-LAST=$(cat /opt/monok8s/config/.bootpart 2>/dev/null || true)
+CUR=$(grep '^BOOT_PART=' "$BOOT_STATE" | cut -d= -f2-)
+LAST=$(cat "$BOOTPART_FILE" 2>/dev/null || true)
+slot_changed=0
if [ "$CUR" != "$LAST" ]; then
- echo "Slot changed ($LAST -> $CUR), cleaning runtime state"
+ slot_changed=1
+ echo "Slot changed ($LAST -> $CUR)"
+fi
- rm -rf /var/lib/containers \
- /var/lib/kubelet/pods \
- /var/lib/kubelet/plugins \
- /var/lib/kubelet/device-plugins
+# shellcheck source=/dev/null
+. "$MIGRATIONS_LIB"
+
+if [ "$slot_changed" -eq 1 ]; then
+ monok8s_cleanup_runtime_state
+fi
- mkdir -p /var/lib/containers /var/lib/kubelet
+K8S_MINOR="$(monok8s_detect_k8s_minor || true)"
+if [ -n "$K8S_MINOR" ]; then
+ monok8s_run_migration_dir \
+ "/usr/lib/monok8s/migrations.d/k8s/$K8S_MINOR" \
+ "$MIGRATION_STATE_DIR/k8s/$K8S_MINOR"
fi
-/usr/local/bin/ctl init --env-file /opt/monok8s/config/cluster.env >>/var/log/monok8s/bootstrap.log 2>&1 &
+/usr/local/bin/ctl init --env-file "$CONFIG_DIR/cluster.env" >>/var/log/monok8s/bootstrap.log 2>&1 &
diff --git a/alpine/rootfs-extra/usr/lib/monok8s/lib/migrations.sh b/alpine/rootfs-extra/usr/lib/monok8s/lib/migrations.sh
new file mode 100755
index 0000000..2758439
--- /dev/null
+++ b/alpine/rootfs-extra/usr/lib/monok8s/lib/migrations.sh
@@ -0,0 +1,58 @@
+#!/bin/sh
+
+monok8s_detect_k8s_minor() {
+ local env_file=${1:-}
+ local version major_minor
+
+ version=$(/usr/local/bin/ctl version -k 2>/dev/null || true)
+
+ [ -n "$version" ] || return 1
+
+ version=${version#v}
+ major_minor=$(printf '%s\n' "$version" | cut -d. -f1,2)
+
+ [ -n "$major_minor" ] || return 1
+ printf '%s\n' "$major_minor"
+}
+
+monok8s_cleanup_runtime_state() {
+ echo "Cleaning runtime state"
+
+ rm -rf \
+ /var/lib/containers \
+ /var/lib/cni \
+ /var/lib/kubelet/pods \
+ /var/lib/kubelet/plugins \
+ /var/lib/kubelet/plugins_registry \
+ /var/lib/kubelet/device-plugins \
+ /run/containers \
+ /run/netns
+
+ mkdir -p \
+ /var/lib/containers \
+ /var/lib/kubelet \
+ /var/lib/cni
+}
+
+monok8s_run_migration_dir() {
+ dir=$1
+ state_dir=$2
+
+ [ -d "$dir" ] || return 0
+ mkdir -p "$state_dir"
+
+ for script in "$dir"/*.sh; do
+ [ -e "$script" ] || continue
+
+ name=$(basename "$script")
+ stamp="$state_dir/$name.done"
+
+ if [ -e "$stamp" ]; then
+ continue
+ fi
+
+ echo "Running migration: $script"
+ sh "$script"
+ : > "$stamp"
+ done
+}
diff --git a/clitools/pkg/cmd/version/version.go b/clitools/pkg/cmd/version/version.go
index 82406c5..f5ef2ec 100644
--- a/clitools/pkg/cmd/version/version.go
+++ b/clitools/pkg/cmd/version/version.go
@@ -1,22 +1,76 @@
-package apply
+package version
import (
+ "encoding/json"
"fmt"
"github.com/spf13/cobra"
- buildInfo "example.com/monok8s/pkg/buildinfo"
+ buildinfo "example.com/monok8s/pkg/buildinfo"
)
+type versionInfo struct {
+ Version string `json:"version"`
+ GitRevision string `json:"gitRevision"`
+ Timestamp string `json:"timestamp"`
+ KubeVersion string `json:"kubernetesVersion"`
+}
+
func NewCmdVersion() *cobra.Command {
+ var (
+ shortOutput bool
+ jsonOutput bool
+ kubernetesOutput bool
+ )
+
cmd := &cobra.Command{
Use: "version",
- Short: "Print the version information",
+ Short: "Print version information",
+ Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
+ info := versionInfo{
+ Version: buildinfo.Version,
+ GitRevision: buildinfo.GitRevision,
+ Timestamp: buildinfo.Timestamp,
+ KubeVersion: buildinfo.KubeVersion,
+ }
+
+ out := cmd.OutOrStdout()
+
+ switch {
+ case jsonOutput:
+ enc := json.NewEncoder(out)
+ enc.SetIndent("", " ")
+ return enc.Encode(info)
- _, err := fmt.Fprintln(cmd.OutOrStdout(), fmt.Sprintf("%s %s(%s)", buildInfo.Version, buildInfo.GitRevision, buildInfo.Timestamp))
- return err
+ case kubernetesOutput:
+ _, err := fmt.Fprintln(out, info.KubeVersion)
+ return err
+
+ case shortOutput:
+ _, err := fmt.Fprintln(out, info.Version)
+ return err
+
+ default:
+ _, err := fmt.Fprintf(
+ out,
+ "Version: %s\nGit commit: %s\nBuilt at: %s\nKubernetes: %s\n",
+ info.Version,
+ info.GitRevision,
+ info.Timestamp,
+ info.KubeVersion,
+ )
+ return err
+ }
},
}
+
+ flags := cmd.Flags()
+ flags.BoolVar(&shortOutput, "short", false, "Show only the application version")
+ flags.BoolVar(&jsonOutput, "json", false, "Show version information as JSON")
+ flags.BoolVarP(&kubernetesOutput, "kubernetes", "k", false, "Show only the Kubernetes version this binary was built for")
+
+ cmd.MarkFlagsMutuallyExclusive("short", "json", "kubernetes")
+
return cmd
}
diff --git a/clitools/pkg/controller/osimage/write_throttle_other.go b/clitools/pkg/controller/osimage/write_throttle_other.go
index 3c3feea..57dd3df 100644
--- a/clitools/pkg/controller/osimage/write_throttle_other.go
+++ b/clitools/pkg/controller/osimage/write_throttle_other.go
@@ -15,4 +15,5 @@ func newNoopAdaptiveWriteController() *adaptiveWriteController {
}
func (c *adaptiveWriteController) Wait(ctx context.Context, n int) error { return nil }
-func (c *adaptiveWriteController) ObserveWrite(n int, dur interface{}) {}
+func (c *adaptiveWriteController) ObserveWrite(n int) {}
+func (c *adaptiveWriteController) ObserveSync() {}
diff --git a/clitools/pkg/controller/osupgrade/planner.go b/clitools/pkg/controller/osupgrade/planner.go
index 29b52a4..7111fd2 100644
--- a/clitools/pkg/controller/osupgrade/planner.go
+++ b/clitools/pkg/controller/osupgrade/planner.go
@@ -224,9 +224,9 @@ func calculatePath(current, target string, available []string) ([]string, error)
add(latestCurMinor)
}
- // Step 2: walk each intermediate minor using the lowest available patch in that minor.
+ // Step 2: walk each intermediate minor using the latest available patch in that minor.
for minor := cur.Minor + 1; minor < tgt.Minor; minor++ {
- bridge, ok := lowestPatchInMinor(versions, cur.Major, minor)
+ bridge, ok := latestAnyPatchInMinor(versions, cur.Major, minor)
if !ok {
return nil, fmt.Errorf("no available bridge version for v%d.%d.x", cur.Major, minor)
}
@@ -239,6 +239,23 @@ func calculatePath(current, target string, available []string) ([]string, error)
return versionsToStrings(path), nil
}
+func latestAnyPatchInMinor(versions []Version, major, minor int) (Version, bool) {
+ var found Version
+ ok := false
+
+ for _, v := range versions {
+ if v.Major != major || v.Minor != minor {
+ continue
+ }
+ if !ok || found.Compare(v) < 0 {
+ found = v
+ ok = true
+ }
+ }
+
+ return found, ok
+}
+
func parseAndSortVersions(raw []string) ([]Version, error) {
out := make([]Version, 0, len(raw))
seen := map[string]struct{}{}
diff --git a/clitools/pkg/controller/osupgrade/planner_test.go b/clitools/pkg/controller/osupgrade/planner_test.go
new file mode 100644
index 0000000..b6b668d
--- /dev/null
+++ b/clitools/pkg/controller/osupgrade/planner_test.go
@@ -0,0 +1,149 @@
+package osupgrade
+
+import (
+ "reflect"
+ "testing"
+)
+
+func TestCalculatePath(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ current string
+ target string
+ available []string
+ want []string
+ wantErr bool
+ }{
+ {
+ name: "same version returns nil path",
+ current: "v1.34.6",
+ target: "v1.34.6",
+ available: []string{"v1.34.6"},
+ want: nil,
+ wantErr: false,
+ },
+ {
+ name: "same minor jumps directly to target",
+ current: "v1.34.1",
+ target: "v1.34.6",
+ available: []string{"v1.34.1", "v1.34.3", "v1.34.6"},
+ want: []string{"v1.34.6"},
+ wantErr: false,
+ },
+ {
+ name: "next minor direct jump when no current minor patch available",
+ current: "v1.34.6",
+ target: "v1.35.3",
+ available: []string{"v1.34.6", "v1.35.1", "v1.35.3"},
+ want: []string{"v1.35.3"},
+ wantErr: false,
+ },
+ {
+ name: "finish current minor then target",
+ current: "v1.34.1",
+ target: "v1.35.3",
+ available: []string{"v1.34.1", "v1.34.6", "v1.35.1", "v1.35.3"},
+ want: []string{"v1.34.6", "v1.35.3"},
+ wantErr: false,
+ },
+ {
+ name: "multi minor path uses latest bridge patch",
+ current: "v1.33.10",
+ target: "v1.35.3",
+ available: []string{"v1.34.1", "v1.34.6", "v1.35.1", "v1.35.3"},
+ want: []string{"v1.34.6", "v1.35.3"},
+ wantErr: false,
+ },
+ {
+ name: "multi minor path finishes current minor and latest bridge patch",
+ current: "v1.33.1",
+ target: "v1.35.3",
+ available: []string{"v1.33.5", "v1.33.9", "v1.34.1", "v1.34.6", "v1.35.3"},
+ want: []string{"v1.33.9", "v1.34.6", "v1.35.3"},
+ wantErr: false,
+ },
+ {
+ name: "duplicates in available are ignored",
+ current: "v1.33.10",
+ target: "v1.35.3",
+ available: []string{"v1.34.6", "v1.34.6", "v1.35.3", "v1.35.3"},
+ want: []string{"v1.34.6", "v1.35.3"},
+ wantErr: false,
+ },
+ {
+ name: "target missing returns error",
+ current: "v1.34.6",
+ target: "v1.35.3",
+ available: []string{"v1.34.6", "v1.35.1"},
+ wantErr: true,
+ },
+ {
+ name: "missing bridge minor returns error",
+ current: "v1.33.10",
+ target: "v1.35.3",
+ available: []string{"v1.35.3"},
+ wantErr: true,
+ },
+ {
+ name: "downgrade not supported",
+ current: "v1.35.3",
+ target: "v1.34.6",
+ available: []string{"v1.34.6", "v1.35.3"},
+ wantErr: true,
+ },
+ {
+ name: "cross major not supported",
+ current: "v1.35.3",
+ target: "v2.0.0",
+ available: []string{"v1.35.3", "v2.0.0"},
+ wantErr: true,
+ },
+ {
+ name: "invalid current version returns error",
+ current: "garbage",
+ target: "v1.35.3",
+ available: []string{"v1.35.3"},
+ wantErr: true,
+ },
+ {
+ name: "invalid target version returns error",
+ current: "v1.34.6",
+ target: "wat",
+ available: []string{"v1.34.6", "v1.35.3"},
+ wantErr: true,
+ },
+ {
+ name: "invalid available version returns error",
+ current: "v1.34.6",
+ target: "v1.35.3",
+ available: []string{"v1.34.6", "broken", "v1.35.3"},
+ wantErr: true,
+ },
+ }
+
+ for _, tt := range tests {
+ tt := tt
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ got, err := calculatePath(tt.current, tt.target, tt.available)
+ if tt.wantErr {
+ if err == nil {
+ t.Fatalf("expected error, got nil; path=%v", got)
+ }
+ return
+ }
+
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+
+ if !reflect.DeepEqual(got, tt.want) {
+ t.Fatalf("calculatePath(%q, %q, %v)\n got: %v\n want: %v",
+ tt.current, tt.target, tt.available, got, tt.want)
+ }
+ })
+ }
+}
diff --git a/clitools/pkg/node/fs.go b/clitools/pkg/node/fs.go
index 8eab227..79ff8b1 100644
--- a/clitools/pkg/node/fs.go
+++ b/clitools/pkg/node/fs.go
@@ -6,6 +6,8 @@ import (
"os"
"path/filepath"
+ "k8s.io/klog/v2"
+
monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1"
"example.com/monok8s/pkg/controller/osimage"
)
@@ -26,7 +28,7 @@ func ReleaseControlGate(ctx context.Context, nctx *NodeContext) error {
gateFile := filepath.Join(monov1alpha1.EnvConfigDir, ".control-gate")
if err := os.Remove(gateFile); err != nil {
- return fmt.Errorf("relate control gate: %w", err)
+ return fmt.Errorf("release control gate: %w", err)
}
return WriteLastState(ctx, nctx)
@@ -46,6 +48,8 @@ func WriteLastState(ctx context.Context, nctx *NodeContext) error {
return fmt.Errorf("BOOT_PART missing")
}
+ klog.Infof("Writing last state: %+v", bootPart)
+
tmp := stBootPart + ".tmp"
if err := os.WriteFile(tmp, []byte(bootPart+"\n"), 0o644); err != nil {
return err
diff --git a/docker/alpine.Dockerfile b/docker/alpine.Dockerfile
index fbee291..6378bd9 100644
--- a/docker/alpine.Dockerfile
+++ b/docker/alpine.Dockerfile
@@ -31,6 +31,7 @@ COPY packages/kubernetes/kubectl-${KUBE_VERSION} /out/rootfs/usr/local/bin/kubec
# COPY clitools/out/dpdk/usr/local/lib/*.so* /out/rootfs/usr/local/lib/
# COPY clitools/out/dpdk/usr/local/lib/dpdk/pmds-23.0/*.so* /out/rootfs/usr/local/lib/dpdk/pmds-23.0/
COPY alpine/rootfs-extra ./rootfs-extra
+COPY alpine/migrations ./migrations
COPY out/build-info ./rootfs-extra/etc/profile.d/build-info.sh
COPY alpine/*.sh /
RUN chmod +x /out/rootfs/usr/local/bin/*