commit 578b3e6a6f8950253c3d75146656562a311194e0e498b4567ef4144006201b8b
| author | 斟酌 鵬兄 <tgckpg@gmail.com> |
|---|
| date | 2026-04-06T15:49:39Z |
|---|
| subject | Initial upgrade flow for control-plane |
|---|
commit 578b3e6a6f8950253c3d75146656562a311194e0e498b4567ef4144006201b8b
Author: 斟酌 鵬兄 <tgckpg@gmail.com>
Date: 2026-04-06T15:49:39Z
Initial upgrade flow for control-plane
---
README.md | 2 +-
alpine/rootfs-extra/etc/containers/storage.conf | 10 +-
build.env | 4 +-
clitools/makefile | 2 +-
.../pkg/apis/monok8s/v1alpha1/groupversion_info.go | 22 ++-
clitools/pkg/bootstrap/registry.go | 6 +-
clitools/pkg/bootstrap/runner.go | 20 +++
clitools/pkg/cmd/agent/agent.go | 38 +++-
clitools/pkg/controller/osimage/helpers.go | 46 +++++
clitools/pkg/controller/osupgrade/handler.go | 4 +-
clitools/pkg/node/crio.go | 60 ++++++-
clitools/pkg/node/fs.go | 32 ++++
clitools/pkg/node/kubeadm.go | 5 -
clitools/pkg/node/kubeadm_upgrade.go | 49 ++++++
clitools/pkg/node/mounts.go | 194 +++++++++++++++++++++
clitools/pkg/node/uboot/ab.go | 69 +++++++-
clitools/pkg/node/uboot/env_writer.go | 11 +-
docs/flashing-usb.md | 20 ++-
initramfs/rootfs-extra/init | 11 +-
19 files changed, 563 insertions(+), 42 deletions(-)
diff --git a/README.md b/README.md
index ae7a42b..dc4fc4e 100644
--- a/README.md
+++ b/README.md
@@ -16,7 +16,7 @@ https://docs.mono.si/gateway-development-kit/getting-started
* USE AT YOUR OWN RISKS. I leverage ChatGPT heavily for this.
-### Table of Contents
+## Table of Contents
1. Flashing
- [USB](docs/flashing-usb.md)
- [Over network (tftp)](docs/flashing-network.md)
diff --git a/alpine/rootfs-extra/etc/containers/storage.conf b/alpine/rootfs-extra/etc/containers/storage.conf
index aae2d5e..ceab0b3 100644
--- a/alpine/rootfs-extra/etc/containers/storage.conf
+++ b/alpine/rootfs-extra/etc/containers/storage.conf
@@ -1,9 +1 @@
-[storage]
-driver = "overlay"
-runroot = "/run/containers/storage"
-graphroot = "/var/lib/containers/storage"
-
-[storage.options]
-additionalimagestores = [
- "/usr/lib/monok8s/imagestore"
-]
+# Generated file. DO NOT MODIFY.
diff --git a/build.env b/build.env
index ecffdcb..49905dd 100644
--- a/build.env
+++ b/build.env
@@ -5,8 +5,8 @@ TAG=dev
# The Linux kernel, from NXP
NXP_VERSION=lf-6.18.2-1.0.0
-CRIO_VERSION=cri-o.arm64.v1.33.3
-KUBE_VERSION=v1.33.3
+CRIO_VERSION=cri-o.arm64.v1.33.10
+KUBE_VERSION=v1.33.10
# Mono's tutorial said fsl-ls1046a-rdb.dtb but our shipped board is not that one
# We need fsl-ls1046a-rdb-sdk.dtb here
diff --git a/clitools/makefile b/clitools/makefile
index d353f3b..4d3fdbb 100644
--- a/clitools/makefile
+++ b/clitools/makefile
@@ -3,7 +3,7 @@ VERSION ?= dev
UBOOT_VERSION=v2026.01
# Target kube version
-KUBE_VERSION ?= v1.35.0
+KUBE_VERSION ?= v1.33.3
GIT_REV := $(shell git rev-parse HEAD)
diff --git a/clitools/pkg/apis/monok8s/v1alpha1/groupversion_info.go b/clitools/pkg/apis/monok8s/v1alpha1/groupversion_info.go
index 5966af6..948cef2 100644
--- a/clitools/pkg/apis/monok8s/v1alpha1/groupversion_info.go
+++ b/clitools/pkg/apis/monok8s/v1alpha1/groupversion_info.go
@@ -9,15 +9,19 @@ import (
)
var (
- Group = "monok8s.io"
- Version = "v1alpha1"
- MonoKSConfigCRD = "monoksconfigs.monok8s.io"
- OSUpgradeCRD = "osupgrades.monok8s.io"
- APIVersion = "monok8s.io/v1alpha1"
- Label = "monok8s.io/label"
- Annotation = "monok8s.io/annotation"
- ControlAgentKey = "monok8s.io/control-agent"
- CatalogURL = "https://example.com/monok8s.io/v1alpha1/catalog.yaml"
+ Group = "monok8s.io"
+ Version = "v1alpha1"
+ APIVersion = "monok8s.io/v1alpha1"
+
+ AltPartDeviceLink = "/dev/mksaltpart"
+ Annotation = "monok8s.io/annotation"
+ BootStateFile = "/run/monok8s/boot-state.env"
+ CatalogURL = "https://example.com/monok8s.io/v1alpha1/catalog.yaml"
+ ControlAgentKey = "monok8s.io/control-agent"
+ EnvConfigDir = "/opt/monok8s/config"
+ Label = "monok8s.io/label"
+ MonoKSConfigCRD = "monoksconfigs.monok8s.io"
+ OSUpgradeCRD = "osupgrades.monok8s.io"
)
var (
diff --git a/clitools/pkg/bootstrap/registry.go b/clitools/pkg/bootstrap/registry.go
index 5c4a52b..06b05e6 100644
--- a/clitools/pkg/bootstrap/registry.go
+++ b/clitools/pkg/bootstrap/registry.go
@@ -27,21 +27,25 @@ func NewRegistry(ctx *node.NodeContext) *Registry {
"ApplyLocalNodeMetadataIfPossible": node.ApplyLocalNodeMetadataIfPossible,
"CheckForVersionSkew": node.CheckForVersionSkew,
"ClassifyBootstrapAction": node.ClassifyBootstrapAction,
+ "ConfigureABBoot": uboot.ConfigureABBoot,
"ConfigureDNS": node.ConfigureDNS(netCfg),
"ConfigureDefaultCNI": node.ConfigureDefaultCNI,
"ConfigureHostname": node.ConfigureHostname(netCfg),
"ConfigureMgmtInterface": node.ConfigureMgmtInterface(netCfg),
- "ConfigureABBoot": uboot.ConfigureABBoot,
"ConfigureUBootCommands": uboot.ConfigureUBootCommands,
"DetectLocalClusterState": node.DetectLocalClusterState,
+ "EngageControlGate": node.EngageControlGate,
"EnsureIPForward": node.EnsureIPForward,
+ "MountAltImageStore": node.MountAltImageStore,
"ReconcileControlPlane": node.ReconcileControlPlane,
"ReconcileWorker": node.ReconcileWorker,
+ "ReleaseControlGate": node.ReleaseControlGate,
"RunKubeadmInit": node.RunKubeadmInit,
"RunKubeadmJoin": node.RunKubeadmJoin,
"RunKubeadmUpgradeApply": node.RunKubeadmUpgradeApply,
"RunKubeadmUpgradeNode": node.RunKubeadmUpgradeNode,
"StartCRIO": node.StartCRIO,
+ "UnmountAltImageStore": node.UnmountAltImageStore,
"ValidateNodeIPAndAPIServerReachability": node.ValidateNodeIPAndAPIServerReachability,
"ValidateRequiredImagesPresent": node.ValidateRequiredImagesPresent,
"WaitForExistingClusterIfNeeded": node.WaitForExistingClusterIfNeeded,
diff --git a/clitools/pkg/bootstrap/runner.go b/clitools/pkg/bootstrap/runner.go
index d951046..c780596 100644
--- a/clitools/pkg/bootstrap/runner.go
+++ b/clitools/pkg/bootstrap/runner.go
@@ -62,6 +62,16 @@ func NewRunner(cfg *monov1alpha1.MonoKSConfig) *Runner {
Name: "Configure default CNI",
Desc: "Install or configure default container networking (CNI bridge, IPAM, etc.)",
},
+ {
+ RegKey: "MountAltImageStore",
+ Name: "Mount alt image store for CRI-O. Needed for upgrade.",
+ Desc: "Will be unmount after kubeadm upgrade apply",
+ },
+ {
+ RegKey: "EngageControlGate",
+ Name: "Engage the control gate",
+ Desc: "Prevents agent polling resources prematurely",
+ },
{
RegKey: "StartCRIO",
Name: "Start CRI-O runtime",
@@ -127,6 +137,11 @@ func NewRunner(cfg *monov1alpha1.MonoKSConfig) *Runner {
Name: "Run kubeadm upgrade node",
Desc: "Upgrade node components (kubelet, config) to match control plane",
},
+ {
+ RegKey: "UnmountAltImageStore",
+ Name: "Unmount alt image store",
+ Desc: "Rewrite CRIO storage.conf. Then restart CRIO. Then unmount.",
+ },
{
RegKey: "ApplyLocalNodeMetadataIfPossible",
Name: "Apply node metadata",
@@ -147,6 +162,11 @@ func NewRunner(cfg *monov1alpha1.MonoKSConfig) *Runner {
Name: "Apply daemonset for control agent",
Desc: "Control agent handles OSUpgrade resources",
},
+ {
+ RegKey: "ReleaseControlGate",
+ Name: "Release the control gate",
+ Desc: "Allow agent to start polling resources",
+ },
},
}
}
diff --git a/clitools/pkg/cmd/agent/agent.go b/clitools/pkg/cmd/agent/agent.go
index cc5e676..47a2f4b 100644
--- a/clitools/pkg/cmd/agent/agent.go
+++ b/clitools/pkg/cmd/agent/agent.go
@@ -3,6 +3,8 @@ package agent
import (
"context"
"fmt"
+ "os"
+ "path/filepath"
"time"
"github.com/spf13/cobra"
@@ -50,6 +52,11 @@ func NewCmdAgent(flags *genericclioptions.ConfigFlags) *cobra.Command {
return fmt.Errorf("node name is empty in rendered config")
}
+ ctx := cmd.Context()
+ if err := waitForControlGate(ctx, envFile, 2*time.Second); err != nil {
+ return fmt.Errorf("wait for control gate to release: %w", err)
+ }
+
klog.InfoS("starting agent",
"node", cfg.Spec.NodeName,
"namespace", namespace,
@@ -62,7 +69,6 @@ func NewCmdAgent(flags *genericclioptions.ConfigFlags) *cobra.Command {
return fmt.Errorf("create kube clients: %w", err)
}
- ctx := cmd.Context()
return runPollLoop(ctx, clients, namespace, cfg.Spec.NodeName, pollInterval)
},
}
@@ -74,6 +80,36 @@ func NewCmdAgent(flags *genericclioptions.ConfigFlags) *cobra.Command {
return cmd
}
+func waitForControlGate(ctx context.Context, envFile string, pollInterval time.Duration) error {
+ dir := filepath.Dir(envFile)
+ marker := filepath.Join(dir, ".control-gate")
+
+ if pollInterval <= 0 {
+ pollInterval = 2 * time.Second
+ }
+
+ ticker := time.NewTicker(pollInterval)
+ defer ticker.Stop()
+
+ for {
+ _, err := os.Stat(marker)
+ if err == nil {
+ klog.InfoS("Control gate is present; waiting before starting poll loop", "path", marker)
+ } else if os.IsNotExist(err) {
+ klog.InfoS("Control gate not present; starting poll loop", "path", marker)
+ return nil
+ } else {
+ return fmt.Errorf("stat upgrade marker %s: %w", marker, err)
+ }
+
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-ticker.C:
+ }
+ }
+}
+
func runPollLoop(ctx context.Context, clients *kube.Clients, namespace, nodeName string, interval time.Duration) error {
gvr := schema.GroupVersionResource{
Group: monov1alpha1.Group,
diff --git a/clitools/pkg/controller/osimage/helpers.go b/clitools/pkg/controller/osimage/helpers.go
index 505c573..efd5270 100644
--- a/clitools/pkg/controller/osimage/helpers.go
+++ b/clitools/pkg/controller/osimage/helpers.go
@@ -1,5 +1,20 @@
package osimage
+import (
+ "fmt"
+ "os"
+ "strings"
+ "sync"
+
+ monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1"
+)
+
+var (
+ bootStateOnce sync.Once
+ bootState map[string]string
+ bootStateErr error
+)
+
func PercentOf(done, total int64) int64 {
if total <= 0 {
return 0
@@ -13,3 +28,34 @@ func PercentOf(done, total int64) int64 {
}
return p
}
+
+func ReadBootState() (map[string]string, error) {
+ bootStateOnce.Do(func() {
+ data, err := os.ReadFile(monov1alpha1.BootStateFile)
+ if err != nil {
+ bootStateErr = err
+ return
+ }
+
+ out := make(map[string]string)
+
+ for _, line := range strings.Split(string(data), "\n") {
+ line = strings.TrimSpace(line)
+ if line == "" || strings.HasPrefix(line, "#") {
+ continue
+ }
+
+ k, v, ok := strings.Cut(line, "=")
+ if !ok {
+ bootStateErr = fmt.Errorf("invalid line: %q", line)
+ return
+ }
+
+ out[strings.TrimSpace(k)] = strings.TrimSpace(v)
+ }
+
+ bootState = out
+ })
+
+ return bootState, bootStateErr
+}
diff --git a/clitools/pkg/controller/osupgrade/handler.go b/clitools/pkg/controller/osupgrade/handler.go
index 76fec0a..fc1231d 100644
--- a/clitools/pkg/controller/osupgrade/handler.go
+++ b/clitools/pkg/controller/osupgrade/handler.go
@@ -111,7 +111,7 @@ func handleOSUpgradeLocked(ctx context.Context, clients *kube.Clients,
"resolvedTarget", plan.ResolvedTarget,
"steps", len(plan.Path),
"currentVersion", buildinfo.KubeVersion,
- "fSHA256irstVersion", first.Version,
+ "firstVersion", first.Version,
"firstURL", first.URL,
"size", first.Size,
)
@@ -126,7 +126,7 @@ func handleOSUpgradeLocked(ctx context.Context, clients *kube.Clients,
imageOptions := osimage.ApplyOptions{
URL: first.URL,
- TargetPath: "/dev/mksaltpart",
+ TargetPath: monov1alpha1.AltPartDeviceLink,
ExpectedRawSHA256: imageSHA,
ExpectedRawSize: first.Size,
BufferSize: 6 * 1024 * 1024,
diff --git a/clitools/pkg/node/crio.go b/clitools/pkg/node/crio.go
index 1988eb3..a614476 100644
--- a/clitools/pkg/node/crio.go
+++ b/clitools/pkg/node/crio.go
@@ -5,9 +5,14 @@ import (
"fmt"
"os"
"strings"
+ "time"
- "k8s.io/klog/v2"
system "example.com/monok8s/pkg/system"
+ "k8s.io/klog/v2"
+)
+
+const (
+ storageConfPath = "/etc/containers/storage.conf"
)
func ConfigureDefaultCNI(ctx context.Context, n *NodeContext) error {
@@ -60,3 +65,56 @@ func ConfigureDefaultCNI(ctx context.Context, n *NodeContext) error {
func StartCRIO(ctx context.Context, n *NodeContext) error {
return system.EnsureServiceRunning(ctx, n.SystemRunner, "crio")
}
+
+func RestartCRIO(ctx context.Context, nctx *NodeContext) error {
+ _, err := nctx.SystemRunner.RunWithOptions(
+ ctx,
+ "rc-service",
+ []string{"crio", "restart"},
+ system.RunOptions{
+ Timeout: 60 * time.Second,
+ OnStdoutLine: func(line string) { klog.Infof("[crio] %s", line) },
+ OnStderrLine: func(line string) { klog.Infof("[crio] %s", line) },
+ },
+ )
+ return err
+}
+
+func writeCRIOStorageConfig(ctx context.Context, nctx *NodeContext, altSource string) error {
+
+ additionalStores := []string{
+ "/usr/lib/monok8s/imagestore",
+ }
+
+ if altSource != "" {
+ additionalStores = append(additionalStores, altSource)
+ }
+
+ var b strings.Builder
+ b.WriteString("# Generated File. DO NOT MODIFY.\n")
+ b.WriteString("[storage]\n")
+ b.WriteString("driver = \"overlay\"\n")
+ b.WriteString("runroot = \"/run/containers/storage\"\n")
+ b.WriteString("graphroot = \"/var/lib/containers/storage\"\n\n")
+ b.WriteString("[storage.options]\n")
+ b.WriteString("additionalimagestores = [\n")
+ for _, s := range additionalStores {
+ b.WriteString(fmt.Sprintf(" %q,\n", s))
+ }
+ b.WriteString("]\n")
+
+ content := b.String()
+
+ path := storageConfPath
+ tmp := path + ".tmp"
+ if err := os.WriteFile(tmp, []byte(content), 0o644); err != nil {
+ return fmt.Errorf("write temp storage.conf: %w", err)
+ }
+ if err := os.Rename(tmp, path); err != nil {
+ _ = os.Remove(tmp)
+ return fmt.Errorf("replace storage.conf: %w", err)
+ }
+
+ klog.InfoS("wrote CRI-O storage config", "path", path, "additionalImageStores", additionalStores)
+ return nil
+}
diff --git a/clitools/pkg/node/fs.go b/clitools/pkg/node/fs.go
new file mode 100644
index 0000000..196422a
--- /dev/null
+++ b/clitools/pkg/node/fs.go
@@ -0,0 +1,32 @@
+package node
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+
+ monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1"
+)
+
+func EngageControlGate(ctx context.Context, nctx *NodeContext) error {
+ gateFile := filepath.Join(monov1alpha1.EnvConfigDir, ".control-gate")
+
+ f, err := os.OpenFile(gateFile, os.O_CREATE|os.O_WRONLY, 0o644)
+ if err != nil {
+ return fmt.Errorf("touch %s: %w", gateFile, err)
+ }
+ defer f.Close()
+
+ return nil
+}
+
+func ReleaseControlGate(ctx context.Context, nctx *NodeContext) error {
+ gateFile := filepath.Join(monov1alpha1.EnvConfigDir, ".control-gate")
+
+ if err := os.Remove(gateFile); err != nil {
+ return fmt.Errorf("relate control gate: %w", err)
+ }
+
+ return nil
+}
diff --git a/clitools/pkg/node/kubeadm.go b/clitools/pkg/node/kubeadm.go
index cc01781..dcd08f5 100644
--- a/clitools/pkg/node/kubeadm.go
+++ b/clitools/pkg/node/kubeadm.go
@@ -732,11 +732,6 @@ func RunKubeadmJoin(ctx context.Context, nctx *NodeContext) error {
return nil
}
-func RunKubeadmUpgradeApply(context.Context, *NodeContext) error {
- klog.Info("run_kubeadm_upgrade_apply: TODO implement kubeadm upgrade apply")
- return nil
-}
-
func RunKubeadmUpgradeNode(context.Context, *NodeContext) error {
klog.Info("run_kubeadm_upgrade_node: TODO implement kubeadm upgrade node")
return nil
diff --git a/clitools/pkg/node/kubeadm_upgrade.go b/clitools/pkg/node/kubeadm_upgrade.go
new file mode 100644
index 0000000..ef47477
--- /dev/null
+++ b/clitools/pkg/node/kubeadm_upgrade.go
@@ -0,0 +1,49 @@
+package node
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+ "time"
+
+ "k8s.io/klog/v2"
+
+ system "example.com/monok8s/pkg/system"
+)
+
+func RunKubeadmUpgradeApply(ctx context.Context, nctx *NodeContext) error {
+
+ if nctx.BootstrapState == nil {
+ return errors.New("BootstrapState is nil. Please run earlier steps first")
+ }
+
+ if nctx.BootstrapState.Action != BootstrapActionUpgradeControlPlane {
+ klog.V(4).Infof("skipped for %s", nctx.BootstrapState.Action)
+ return nil
+ }
+
+ if strings.TrimSpace(tmpKubeadmInitConf) == "" {
+ return fmt.Errorf("tmp kubeadm config path is empty")
+ }
+
+ _, err := nctx.SystemRunner.RunWithOptions(
+ ctx,
+ "kubeadm",
+ []string{"upgrade", "apply", "-y", nctx.Config.Spec.KubernetesVersion},
+ system.RunOptions{
+ Timeout: 15 * time.Minute,
+ OnStdoutLine: func(line string) {
+ klog.Infof("[kubeadm] %s", line)
+ },
+ OnStderrLine: func(line string) {
+ klog.Infof("[kubeadm] %s", line)
+ },
+ },
+ )
+ if err != nil {
+ return fmt.Errorf("run kubeadm upgrade apply: %w", err)
+ }
+
+ return nil
+}
diff --git a/clitools/pkg/node/mounts.go b/clitools/pkg/node/mounts.go
new file mode 100644
index 0000000..a99dc65
--- /dev/null
+++ b/clitools/pkg/node/mounts.go
@@ -0,0 +1,194 @@
+package node
+
+import (
+ "bufio"
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1"
+ system "example.com/monok8s/pkg/system"
+ "k8s.io/klog/v2"
+)
+
+const (
+ altRootMount = "/run/altrootfs"
+ altImageStoreSrc = "/run/altrootfs/usr/lib/monok8s/imagestore"
+)
+
+func MountAltImageStore(ctx context.Context, nctx *NodeContext) error {
+ // Make mountpoints first.
+ if err := os.MkdirAll(altRootMount, 0o755); err != nil {
+ return fmt.Errorf("mkdir %s: %w", altRootMount, err)
+ }
+
+ altDev := monov1alpha1.AltPartDeviceLink
+ if mounted, err := isMounted(altRootMount); err != nil {
+ return fmt.Errorf("check mount %s: %w", altRootMount, err)
+ } else if !mounted {
+ klog.InfoS("mounting alt rootfs", "device", altDev, "target", altRootMount)
+ if _, err := nctx.SystemRunner.RunWithOptions(
+ ctx,
+ "mount",
+ []string{"-o", "ro", altDev, altRootMount},
+ system.RunOptions{Timeout: 30 * time.Second},
+ ); err != nil {
+ return fmt.Errorf("mount alt rootfs %s on %s: %w", altDev, altRootMount, err)
+ }
+ }
+
+ // If the alt imagestore doesn't exist, don't fail hard unless you want strict behavior.
+ st, err := os.Stat(altImageStoreSrc)
+ if err != nil {
+
+ // Unmount immediately
+ _ = safeUnmount(ctx, nctx, altRootMount)
+
+ if os.IsNotExist(err) {
+ klog.InfoS(
+ "alt imagestore not found; proceeding without it",
+ "path", altImageStoreSrc,
+ )
+ err = writeCRIOStorageConfig(ctx, nctx, "")
+ if err != nil {
+ return err
+ }
+ return nil
+ }
+
+ return fmt.Errorf("stat alt imagestore %s: %w", altImageStoreSrc, err)
+ }
+
+ if !st.IsDir() {
+ return fmt.Errorf("alt imagestore exists but is not a directory: %s", altImageStoreSrc)
+ }
+
+ err = writeCRIOStorageConfig(ctx, nctx, altImageStoreSrc)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func UnmountAltImageStore(ctx context.Context, nctx *NodeContext) error {
+
+ mounted, err := isMounted(altRootMount)
+ if err != nil {
+ return err
+ }
+
+ if mounted {
+ if err := writeCRIOStorageConfig(ctx, nctx, ""); err != nil {
+ return err
+ }
+ err := RestartCRIO(ctx, nctx)
+ if err != nil {
+ return err
+ }
+ }
+
+ var errs []string
+
+ if err := safeUnmount(ctx, nctx, altRootMount); err != nil {
+ errs = append(errs, err.Error())
+ }
+
+ if len(errs) > 0 {
+ return fmt.Errorf(strings.Join(errs, "; "))
+ }
+ return nil
+}
+
+func isMounted(path string) (bool, error) {
+ f, err := os.Open("/proc/self/mountinfo")
+ if err != nil {
+ return false, fmt.Errorf("open /proc/self/mountinfo: %w", err)
+ }
+ defer f.Close()
+
+ target := filepath.Clean(path)
+
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ line := scanner.Text()
+ fields := strings.Fields(line)
+ if len(fields) < 5 {
+ continue
+ }
+
+ mountPoint, err := unescapeMountInfo(fields[4])
+ if err != nil {
+ return false, fmt.Errorf("decode mountpoint %q: %w", fields[4], err)
+ }
+
+ if filepath.Clean(mountPoint) == target {
+ return true, nil
+ }
+ }
+
+ if err := scanner.Err(); err != nil {
+ return false, fmt.Errorf("scan /proc/self/mountinfo: %w", err)
+ }
+
+ return false, nil
+}
+
+func safeUnmount(ctx context.Context, nctx *NodeContext, path string) error {
+ mounted, err := isMounted(path)
+ if err != nil {
+ return fmt.Errorf("check mountpoint %s: %w", path, err)
+ }
+ if !mounted {
+ return nil
+ }
+
+ klog.InfoS("unmounting", "target", path)
+ _, err = nctx.SystemRunner.RunWithOptions(
+ ctx,
+ "umount",
+ []string{path},
+ system.RunOptions{Timeout: 30 * time.Second},
+ )
+ if err != nil {
+ return fmt.Errorf("umount %s: %w", path, err)
+ }
+ return nil
+}
+
+func unescapeMountInfo(s string) (string, error) {
+ var b strings.Builder
+ b.Grow(len(s))
+
+ for i := 0; i < len(s); i++ {
+ if s[i] != '\\' {
+ b.WriteByte(s[i])
+ continue
+ }
+
+ if i+3 >= len(s) {
+ return "", fmt.Errorf("truncated escape sequence")
+ }
+
+ esc := s[i+1 : i+4]
+ switch esc {
+ case "040":
+ b.WriteByte(' ')
+ case "011":
+ b.WriteByte('\t')
+ case "012":
+ b.WriteByte('\n')
+ case "134":
+ b.WriteByte('\\')
+ default:
+ return "", fmt.Errorf("unsupported escape sequence \\%s", esc)
+ }
+
+ i += 3
+ }
+
+ return b.String(), nil
+}
diff --git a/clitools/pkg/node/uboot/ab.go b/clitools/pkg/node/uboot/ab.go
index cde8680..b610dc1 100644
--- a/clitools/pkg/node/uboot/ab.go
+++ b/clitools/pkg/node/uboot/ab.go
@@ -2,9 +2,14 @@ package uboot
import (
"context"
+ "errors"
"fmt"
"os"
+ "path/filepath"
+ "strings"
+ monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1"
+ "example.com/monok8s/pkg/controller/osimage"
"example.com/monok8s/pkg/node"
)
@@ -15,13 +20,67 @@ func ConfigureABBoot(ctx context.Context, nctx *node.NodeContext) error {
return fmt.Errorf("get current executable path: %w", err)
}
+ doWrite := false
writer := NewFWEnvWriter(HostFWEnvCfgPath, exePath)
- // TODO: configurable from cluster.env
- return writer.EnsureBootEnv(ctx, BootEnvConfig{
- BootSource: "usb",
- BootPart: "A",
- })
+ bootPart, err := writer.GetEnv(ctx, "boot_part")
+ if err != nil {
+ if errors.Is(err, ErrEnvNotFound) {
+
+ state, err := osimage.ReadBootState()
+ if err != nil {
+ return fmt.Errorf("read boot state: %w", err)
+ }
+
+ bootPart := state["BOOT_PART"]
+ if bootPart == "" {
+ return fmt.Errorf("BOOT_PART missing")
+ }
+
+ doWrite = true
+ } else {
+ return fmt.Errorf("get boot_part: %w", err)
+ }
+ }
+
+ bootSource, err := writer.GetEnv(ctx, "boot_source")
+ if err != nil {
+ if errors.Is(err, ErrEnvNotFound) {
+ target, err := os.Readlink(monov1alpha1.AltPartDeviceLink)
+ if err != nil {
+ return fmt.Errorf("readlink %q: %w", monov1alpha1.AltPartDeviceLink, err)
+ }
+
+ if !filepath.IsAbs(target) {
+ target = filepath.Join(filepath.Dir(monov1alpha1.AltPartDeviceLink), target)
+ }
+
+ resolved, err := filepath.EvalSymlinks(target)
+ if err != nil {
+ return fmt.Errorf("resolve %q: %w", target, err)
+ }
+
+ base := filepath.Base(resolved)
+
+ if strings.HasPrefix(base, "mmcblk") {
+ bootSource = "emmc"
+ } else {
+ bootSource = "usb"
+ }
+ doWrite = true
+ } else {
+ return fmt.Errorf("get boot_source: %w", err)
+ }
+ }
+
+ if doWrite {
+ return writer.EnsureBootEnv(ctx, BootEnvConfig{
+ BootSource: bootSource,
+ BootPart: bootPart,
+ })
+ }
+
+ return nil
}
// This is called by the agent controller/osupgrade/handler.go
diff --git a/clitools/pkg/node/uboot/env_writer.go b/clitools/pkg/node/uboot/env_writer.go
index 8cb9e32..fef4c24 100644
--- a/clitools/pkg/node/uboot/env_writer.go
+++ b/clitools/pkg/node/uboot/env_writer.go
@@ -2,6 +2,7 @@ package uboot
import (
"context"
+ "errors"
"fmt"
"strings"
"time"
@@ -11,6 +12,8 @@ import (
"example.com/monok8s/pkg/system"
)
+var ErrEnvNotFound = errors.New("fw env not found")
+
func NewFWEnvWriter(configPath string, ctlPath string) *FWEnvWriter {
return &FWEnvWriter{
Runner: system.NewRunner(system.RunnerConfig{
@@ -38,8 +41,12 @@ func (w *FWEnvWriter) GetEnv(ctx context.Context, key string) (string, error) {
res, err := w.Runner.RunWithOptions(ctx, w.CtlPath, args, system.RunOptions{Quiet: true})
if err != nil {
if res != nil {
- return "", fmt.Errorf("fw-printenv %q: %w (stdout=%q stderr=%q)",
- key, err, strings.TrimSpace(res.Stdout), strings.TrimSpace(res.Stderr))
+ if strings.Contains(res.Stderr, "not defined") {
+ return "", ErrEnvNotFound
+ } else {
+ return "", fmt.Errorf("fw-printenv %q: %w (stdout=%q stderr=%q)",
+ key, err, strings.TrimSpace(res.Stdout), strings.TrimSpace(res.Stderr))
+ }
}
return "", fmt.Errorf("fw-printenv %q: %w", key, err)
}
diff --git a/docs/flashing-usb.md b/docs/flashing-usb.md
index 1b1e782..9d62b6c 100644
--- a/docs/flashing-usb.md
+++ b/docs/flashing-usb.md
@@ -23,8 +23,24 @@ bootm 0x80000000
flash-emmc.sh
```
-4. If it boots, create the A/B deployment scheme
- - (WORK IN PROGRESS)
+6. Reboot into uboot, boot using the following commands
+```
+setenv boot_source emmc
+setenv boot_part A
+
+setenv rootpart 2;
+
+setenv bootdev 0:${rootpart};
+setenv rootdev emmc:2;
+
+setenv kernel_addr_r 0xa0000000;
+
+setenv bootargs "${bootargs_console} root=${rootdev} bootpart=${boot_part} rw rootwait rootfstype=ext4";
+ext4load mmc ${bootdev} ${kernel_addr_r} /boot/kernel.itb && bootm ${kernel_addr_r};'
+```
+
+7. tail /var/log/monok8s/bootstrap.log
+
## Flashing into USB
On MacOS
diff --git a/initramfs/rootfs-extra/init b/initramfs/rootfs-extra/init
index 1483568..c3c9275 100755
--- a/initramfs/rootfs-extra/init
+++ b/initramfs/rootfs-extra/init
@@ -221,7 +221,7 @@ log "Booting kernel took $(cut -d' ' -f1 /proc/uptime) seconds."
. /etc/build-info || panic "failed to source /etc/build-info"
-wait_for_partnames 30 rootfsA rootfsB data || panic "failed to wait for fs"
+wait_for_partnames 5 rootfsA rootfsB data || panic "failed to wait for fs"
ROOT_CMD="$(get_cmdline_arg root || true)"
BOOT_PART="$(get_cmdline_arg bootpart || true)"
@@ -283,6 +283,15 @@ if [ -n "$ALT_PART" ]; then
ln -sf "$ALT_PART" /dev/mksaltpart
fi
+mkdir -p /run/monok8s
+cat > /run/monok8s/boot-state.env <<EOF
+BOOT_PART=$BOOT_PART
+ROOT_DEV=$ROOT_DEV
+DATA_DEV=$DATA_DEV
+EOF
+
+mkdir -p /run/altrootfs
+
mount_or_panic --move /dev /newroot/dev
mount_or_panic --move /proc /newroot/proc
mount_or_panic --move /sys /newroot/sys