commit 517cc2e01d47e03293e4f843a4994b216bca6b74b925645703d67fccee809c7f
| author | 斟酌 鵬兄 <tgckpg@gmail.com> |
|---|
| date | 2026-04-04T12:19:25Z |
|---|
| subject | control agent can now uboot commands |
|---|
commit 517cc2e01d47e03293e4f843a4994b216bca6b74b925645703d67fccee809c7f
Author: 斟酌 鵬兄 <tgckpg@gmail.com>
Date: 2026-04-04T12:19:25Z
control agent can now uboot commands
---
alpine/configure-system.sh | 1 +
alpine/install-packages.sh | 3 +-
.../etc/crio/crio.conf.d/11-coredns.conf | 4 +
build.env | 5 +-
.../{Dockerfile => docker/ctl-agent.Dockerfile} | 2 +
clitools/docker/uboot-tools.Dockerfile | 41 +++++
clitools/makefile | 29 ++-
clitools/pkg/bootstrap/runner.go | 7 +-
clitools/pkg/cmd/internal/fwprintenv.go | 89 +++++++++
clitools/pkg/cmd/internal/fwsetenv.go | 115 ++++++++++++
clitools/pkg/cmd/internal/internal.go | 5 +-
clitools/pkg/controller/osupgrade/bootenv.go | 37 ++++
clitools/pkg/controller/osupgrade/handler.go | 17 +-
clitools/pkg/node/agent.go | 47 ++---
clitools/pkg/node/network.go | 2 +-
docs/coredns.md | 198 +++++++++++++++++++++
docs/uboot.md | 12 +-
kernel-extra.config | 8 +
makefile | 14 --
19 files changed, 573 insertions(+), 63 deletions(-)
diff --git a/alpine/configure-system.sh b/alpine/configure-system.sh
index b6edf64..d84279c 100755
--- a/alpine/configure-system.sh
+++ b/alpine/configure-system.sh
@@ -7,6 +7,7 @@ rc-update add devfs sysinit
rc-update add procfs sysinit
rc-update add sysfs sysinit
rc-update add cgroups sysinit
+rc-update add hwclock boot
rc-update add fancontrol boot
rc-update add loopback boot
rc-update add hostname boot
diff --git a/alpine/install-packages.sh b/alpine/install-packages.sh
index 9b26dc6..21469e8 100755
--- a/alpine/install-packages.sh
+++ b/alpine/install-packages.sh
@@ -31,7 +31,8 @@ if [ $? -ne 0 ]; then
exit $?
fi
-sed -i "s/default_runtime = \"crun\"/\0\ncgroup_manager = \"cgroupfs\"/g" /etc/crio/crio.conf.d/10-crio.conf
+sed -i -e "s/default_runtime = \"crun\"/\0\ncgroup_manager = \"cgroupfs\"/g" \
+ /etc/crio/crio.conf.d/10-crio.conf
grep cgroup_manager /etc/crio/crio.conf.d/10-crio.conf || exit 1
echo "##################################################### Installing Control Agent"
diff --git a/alpine/rootfs-extra/etc/crio/crio.conf.d/11-coredns.conf b/alpine/rootfs-extra/etc/crio/crio.conf.d/11-coredns.conf
new file mode 100644
index 0000000..52a3811
--- /dev/null
+++ b/alpine/rootfs-extra/etc/crio/crio.conf.d/11-coredns.conf
@@ -0,0 +1,4 @@
+[crio.runtime]
+default_sysctls = [
+ "net.ipv4.ip_unprivileged_port_start=20"
+]
diff --git a/build.env b/build.env
index 979202f..872115c 100644
--- a/build.env
+++ b/build.env
@@ -5,8 +5,8 @@ TAG=dev
# The Linux kernel, from NXP
NXP_VERSION=lf-6.18.2-1.0.0
-CRIO_VERSION=cri-o.arm64.v1.35.1
-KUBE_VERSION=v1.35.1
+CRIO_VERSION=cri-o.arm64.v1.35.2
+KUBE_VERSION=v1.35.3
# Mono's tutorial said fsl-ls1046a-rdb.dtb but our shipped board is not that one
# We need fsl-ls1046a-rdb-sdk.dtb here
@@ -21,6 +21,7 @@ CROSS_COMPILE=aarch64-linux-gnu-
# Tools for initramfs
BUSYBOX_VERSION=1_36_1
E2FSPROGS_VERSION=1.47.4
+UBOOT_VERSION=v2026.01
## Alpine Linux
ALPINE_VER=3.23.3
diff --git a/clitools/Dockerfile b/clitools/docker/ctl-agent.Dockerfile
similarity index 83%
rename from clitools/Dockerfile
rename to clitools/docker/ctl-agent.Dockerfile
index d6d9765..7387bf2 100644
--- a/clitools/Dockerfile
+++ b/clitools/docker/ctl-agent.Dockerfile
@@ -7,6 +7,8 @@ ENV VERSION=${VERSION}
WORKDIR /
COPY bin/ctl-linux-aarch64-${VERSION} ./ctl
+COPY out/fw_printenv ./
+COPY out/fw_setenv ./
COPY --from=cacerts /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
ENTRYPOINT ["/ctl"]
diff --git a/clitools/docker/uboot-tools.Dockerfile b/clitools/docker/uboot-tools.Dockerfile
new file mode 100644
index 0000000..67c5b64
--- /dev/null
+++ b/clitools/docker/uboot-tools.Dockerfile
@@ -0,0 +1,41 @@
+FROM alpine:3.22 AS build
+
+RUN apk add --no-cache \
+ build-base \
+ bison \
+ flex \
+ linux-headers \
+ file \
+ binutils \
+ tar
+
+WORKDIR /src
+
+ARG UBOOT_TAR
+ARG UBOOT_VERSION
+
+COPY ${UBOOT_TAR} /tmp/
+RUN tar zxf "/tmp/$(basename "${UBOOT_TAR}")"
+
+RUN make tools-only_defconfig
+
+# Build the env tools using the supported target.
+RUN make -j"$(nproc)" \
+ HOSTCC=gcc \
+ HOSTLD=gcc \
+ HOSTCFLAGS='-O2' \
+ HOSTLDFLAGS='-static' \
+ envtools
+
+# fw_setenv is the same program; create the link ourselves.
+RUN ln -sf fw_printenv tools/env/fw_setenv
+
+RUN file tools/env/fw_printenv tools/env/fw_setenv
+
+RUN readelf -d tools/env/fw_printenv || true
+
+RUN ! readelf -d tools/env/fw_printenv | grep -q '(NEEDED)'
+
+FROM scratch AS export
+COPY --from=build /src/tools/env/fw_printenv /fw_printenv
+COPY --from=build /src/tools/env/fw_setenv /fw_setenv
diff --git a/clitools/makefile b/clitools/makefile
index d55fce5..d353f3b 100644
--- a/clitools/makefile
+++ b/clitools/makefile
@@ -1,17 +1,24 @@
# Should be the same as upstream version in production
VERSION ?= dev
+UBOOT_VERSION=v2026.01
# Target kube version
KUBE_VERSION ?= v1.35.0
GIT_REV := $(shell git rev-parse HEAD)
-BIN_DIR := bin
-OUT_DIR := out
+PACKAGES_DIR := packages
+BIN_DIR := bin
+OUT_DIR := out
+
+UBOOT_TAR := $(PACKAGES_DIR)/uboot-$(UBOOT_VERSION).tar.gz
BUILDINFO_FILE := pkg/buildinfo/buildinfo_gen.go
CRD_PATHS := ./pkg/apis/...
+$(PACKAGES_DIR):
+ mkdir -p $@
+
# Never cache this
.buildinfo:
@mkdir -p $(dir $(BUILDINFO_FILE))
@@ -26,13 +33,27 @@ CRD_PATHS := ./pkg/apis/...
')' \
> $(BUILDINFO_FILE)
+$(UBOOT_TAR): | $(PACKAGES_DIR)
+ git clone --depth 1 --branch v2026.01 --filter=blob:none https://github.com/u-boot/u-boot.git $(OUT_DIR)/u-boot-$(UBOOT_VERSION)
+ tar -C "$(OUT_DIR)/u-boot-$(UBOOT_VERSION)" -zcf "$@" .
+ rm -rf $(OUT_DIR)/u-boot-$(UBOOT_VERSION)
+ test -f $@
+
+uboot-tools: $(UBOOT_TAR)
+ docker buildx build --platform linux/arm64 \
+ -f docker/uboot-tools.Dockerfile \
+ --build-arg UBOOT_VERSION=$(UBOOT_VERSION) \
+ --build-arg UBOOT_TAR=$(UBOOT_TAR) \
+ --output type=local,dest=./$(OUT_DIR) .
+
build: .buildinfo
mkdir -p $(BIN_DIR) $(OUT_DIR)/crds
controller-gen crd paths=$(CRD_PATHS) output:crd:dir=$(OUT_DIR)/crds
GOOS=linux GOARCH=arm64 go build -o $(BIN_DIR)/ctl-linux-aarch64-$(VERSION) ./cmd/ctl
-build-agent: build
+build-agent: build uboot-tools
docker build \
+ -f docker/ctl-agent.Dockerfile \
--platform=linux/arm64 \
--build-arg VERSION=$(VERSION) \
-t localhost/monok8s/control-agent:$(VERSION) .
@@ -52,4 +73,4 @@ clean:
all: build build-agent build-local
-.PHONY: clean all run .buildinfo build build-local build-agent
+.PHONY: clean all run .buildinfo build build-local build-agent uboot-tools
diff --git a/clitools/pkg/bootstrap/runner.go b/clitools/pkg/bootstrap/runner.go
index 8ccb172..e57d461 100644
--- a/clitools/pkg/bootstrap/runner.go
+++ b/clitools/pkg/bootstrap/runner.go
@@ -4,10 +4,10 @@ import (
"context"
"fmt"
- "k8s.io/klog/v2"
monov1alpha1 "example.com/monok8s/pkg/apis/monok8s/v1alpha1"
"example.com/monok8s/pkg/node"
"example.com/monok8s/pkg/system"
+ "k8s.io/klog/v2"
)
type Runner struct {
@@ -132,6 +132,11 @@ func NewRunner(cfg *monov1alpha1.MonoKSConfig) *Runner {
Name: "Apply node metadata",
Desc: "Apply labels/annotations to the local node if API server is reachable",
},
+ {
+ RegKey: "ConfigureUBootCommands",
+ Name: "Ensure fw_env config and u-boot-tools availablilty",
+ Desc: "Install or generate /etc/fw_env.config for U-Boot environment access",
+ },
{
RegKey: "ApplyControlAgentDaemonSetResources",
Name: "Apply daemonset for control agent",
diff --git a/clitools/pkg/cmd/internal/fwprintenv.go b/clitools/pkg/cmd/internal/fwprintenv.go
new file mode 100644
index 0000000..9bc6fde
--- /dev/null
+++ b/clitools/pkg/cmd/internal/fwprintenv.go
@@ -0,0 +1,89 @@
+package internal
+
+import (
+ "fmt"
+ "os"
+ "strings"
+ "time"
+
+ "github.com/spf13/cobra"
+ "k8s.io/klog/v2"
+
+ "example.com/monok8s/pkg/system"
+)
+
+func newInternalFWPrintEnvCmd() *cobra.Command {
+ var key string
+ var configPath string
+
+ cmd := &cobra.Command{
+ Use: "fw-printenv",
+ Short: "Run fw_printenv",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ ctx := cmd.Context()
+
+ key = strings.TrimSpace(key)
+ configPath = strings.TrimSpace(configPath)
+
+ if configPath == "" {
+ configPath = defaultFWEnvConfigPath
+ }
+
+ if _, err := os.Stat(configPath); err != nil {
+ return fmt.Errorf("stat fw env config %q: %w", configPath, err)
+ }
+
+ runner := system.NewRunner(system.RunnerConfig{
+ DefaultTimeout: 15 * time.Second,
+ StreamOutput: false,
+ Logger: &system.StdLogger{},
+ })
+
+ runArgs := []string{"-c", configPath}
+ if key != "" {
+ runArgs = append(runArgs, key)
+ }
+
+ res, err := runner.RunWithOptions(
+ ctx,
+ "/fw_printenv",
+ runArgs,
+ system.RunOptions{
+ Quiet: true,
+ },
+ )
+ if err != nil {
+ if res != nil {
+ klog.ErrorS(err, "fw_printenv failed",
+ "key", key,
+ "stdout", strings.TrimSpace(res.Stdout),
+ "stderr", strings.TrimSpace(res.Stderr),
+ )
+ }
+ return err
+ }
+
+ stdout := strings.TrimSpace(res.Stdout)
+ stderr := strings.TrimSpace(res.Stderr)
+
+ if stdout != "" {
+ fmt.Println(stdout)
+ }
+ if stderr != "" {
+ klog.InfoS("fw_printenv stderr", "output", stderr)
+ }
+
+ klog.InfoS("fw_printenv succeeded",
+ "key", key,
+ "configPath", configPath,
+ )
+
+ return nil
+ },
+ }
+
+ cmd.Flags().StringVar(&key, "key", "", "U-Boot environment variable name to print")
+ cmd.Flags().StringVar(&configPath, "config", defaultFWEnvConfigPath, "Path to fw_env.config")
+
+ return cmd
+}
diff --git a/clitools/pkg/cmd/internal/fwsetenv.go b/clitools/pkg/cmd/internal/fwsetenv.go
new file mode 100644
index 0000000..3537e33
--- /dev/null
+++ b/clitools/pkg/cmd/internal/fwsetenv.go
@@ -0,0 +1,115 @@
+package internal
+
+import (
+ "fmt"
+ "os"
+ "strings"
+ "time"
+
+ "github.com/spf13/cobra"
+ "k8s.io/klog/v2"
+
+ "example.com/monok8s/pkg/system"
+)
+
+const defaultFWEnvConfigPath = "/host/etc/fw_env.config"
+
+func newInternalFWSetEnvCmd() *cobra.Command {
+ var key string
+ var value string
+ var configPath string
+
+ cmd := &cobra.Command{
+ Use: "fw-setenv",
+ Short: "Run fw_setenv",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ ctx := cmd.Context()
+
+ key = strings.TrimSpace(key)
+ value = strings.TrimSpace(value)
+ configPath = strings.TrimSpace(configPath)
+
+ if key == "" {
+ return fmt.Errorf("--key is required")
+ }
+ if value == "" {
+ return fmt.Errorf("--value is required")
+ }
+ if configPath == "" {
+ configPath = defaultFWEnvConfigPath
+ }
+
+ if _, err := os.Stat(configPath); err != nil {
+ return fmt.Errorf("stat fw env config %q: %w", configPath, err)
+ }
+
+ runner := system.NewRunner(system.RunnerConfig{
+ DefaultTimeout: 15 * time.Second,
+ StreamOutput: false,
+ Logger: &system.StdLogger{},
+ })
+
+ // Preflight first so failure is clearer than blindly writing.
+ preflightRes, err := runner.RunWithOptions(
+ ctx,
+ "/fw_printenv",
+ []string{"-c", configPath},
+ system.RunOptions{
+ Quiet: true,
+ },
+ )
+ if err != nil {
+ if preflightRes != nil {
+ klog.ErrorS(err, "fw_printenv preflight failed",
+ "stdout", strings.TrimSpace(preflightRes.Stdout),
+ "stderr", strings.TrimSpace(preflightRes.Stderr),
+ )
+ }
+ return fmt.Errorf("fw_printenv preflight: %w", err)
+ }
+
+ res, err := runner.RunWithOptions(
+ ctx,
+ "/fw_setenv",
+ []string{
+ "-c", configPath,
+ key, value,
+ },
+ system.RunOptions{
+ Quiet: true,
+ },
+ )
+ if err != nil {
+ if res != nil {
+ klog.ErrorS(err, "fw_setenv failed",
+ "key", key,
+ "value", value,
+ "stdout", strings.TrimSpace(res.Stdout),
+ "stderr", strings.TrimSpace(res.Stderr),
+ )
+ }
+ return err
+ }
+
+ if strings.TrimSpace(res.Stdout) != "" {
+ klog.InfoS("fw_setenv stdout", "output", strings.TrimSpace(res.Stdout))
+ }
+ if strings.TrimSpace(res.Stderr) != "" {
+ klog.InfoS("fw_setenv stderr", "output", strings.TrimSpace(res.Stderr))
+ }
+
+ klog.InfoS("fw_setenv succeeded",
+ "key", key,
+ "value", value,
+ "configPath", configPath,
+ )
+ return nil
+ },
+ }
+
+ cmd.Flags().StringVar(&key, "key", "", "U-Boot environment variable name")
+ cmd.Flags().StringVar(&value, "value", "", "U-Boot environment variable value")
+ cmd.Flags().StringVar(&configPath, "config", defaultFWEnvConfigPath, "Path to fw_env.config")
+
+ return cmd
+}
diff --git a/clitools/pkg/cmd/internal/internal.go b/clitools/pkg/cmd/internal/internal.go
index 7394077..de8410c 100644
--- a/clitools/pkg/cmd/internal/internal.go
+++ b/clitools/pkg/cmd/internal/internal.go
@@ -1,9 +1,9 @@
package internal
import (
- "github.com/spf13/cobra"
"example.com/monok8s/pkg/bootstrap"
"example.com/monok8s/pkg/config"
+ "github.com/spf13/cobra"
)
func NewCmdInternal() *cobra.Command {
@@ -33,6 +33,9 @@ func NewCmdInternal() *cobra.Command {
return nil
},
})
+ cmd.AddCommand(newInternalFWSetEnvCmd())
+ cmd.AddCommand(newInternalFWPrintEnvCmd())
+
cmd.PersistentFlags().StringVarP(&configPath, "config", "c", "", "path to MonoKSConfig yaml")
return cmd
}
diff --git a/clitools/pkg/controller/osupgrade/bootenv.go b/clitools/pkg/controller/osupgrade/bootenv.go
new file mode 100644
index 0000000..b632694
--- /dev/null
+++ b/clitools/pkg/controller/osupgrade/bootenv.go
@@ -0,0 +1,37 @@
+package osupgrade
+
+import (
+ "context"
+ "fmt"
+ "os/exec"
+)
+
+type NextBootConfig struct {
+ Key string
+ Value string
+}
+
+func SetNextBootEnv(ctx context.Context, cfg NextBootConfig) error {
+ if cfg.Key == "" {
+ return fmt.Errorf("boot env key is empty")
+ }
+ if cfg.Value == "" {
+ return fmt.Errorf("boot env value is empty")
+ }
+
+ cmd := exec.CommandContext(
+ ctx,
+ "/proc/self/exe",
+ "internal",
+ "fw-setenv",
+ "--key", cfg.Key,
+ "--value", cfg.Value,
+ )
+
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ return fmt.Errorf("invoke internal fw-setenv: %w: %s", err, string(out))
+ }
+
+ return nil
+}
diff --git a/clitools/pkg/controller/osupgrade/handler.go b/clitools/pkg/controller/osupgrade/handler.go
index 57f587e..51a519a 100644
--- a/clitools/pkg/controller/osupgrade/handler.go
+++ b/clitools/pkg/controller/osupgrade/handler.go
@@ -128,7 +128,22 @@ func HandleOSUpgrade(ctx context.Context, clients *kube.Clients,
}
klog.Info(result)
- // TODO: fw_setenv
+ if err := SetNextBootEnv(ctx, NextBootConfig{
+ Key: "boot_part",
+ Value: "B",
+ }); err != nil {
+ return failProgress(ctx, clients, osup, "set boot env", err)
+ }
+
+ now = metav1.Now()
+ osup.Status.LastUpdatedAt = &now
+ osup.Status.Message = "image applied, verified, and next boot environment updated"
+ osup.Status.Phase = monov1alpha1.OSUpgradeProgressPhaseRebooting
+
+ osup, err = updateProgressStatus(ctx, clients, osup_gvr, osup)
+ if err != nil {
+ return fmt.Errorf("update progress status: %w", err)
+ }
return nil
}
diff --git a/clitools/pkg/node/agent.go b/clitools/pkg/node/agent.go
index 976fb9f..d52d63f 100644
--- a/clitools/pkg/node/agent.go
+++ b/clitools/pkg/node/agent.go
@@ -273,20 +273,8 @@ func applyControlAgentDaemonSet(ctx context.Context, kubeClient kubernetes.Inter
Value: "/host/opt/monok8s/config/cluster.env",
},
{
- Name: "HOST_MOUNT_ROOT",
- Value: "/host/mnt/control-agent",
- },
- {
- Name: "HOST_DEV_DIR",
- Value: "/host/dev",
- },
- {
- Name: "HOST_PROC_DIR",
- Value: "/host/proc",
- },
- {
- Name: "HOST_RUN_DIR",
- Value: "/host/run",
+ Name: "FW_ENV_CONFIG_FILE",
+ Value: "/host/etc/fw_env.config",
},
},
SecurityContext: &corev1.SecurityContext{
@@ -295,20 +283,16 @@ func applyControlAgentDaemonSet(ctx context.Context, kubeClient kubernetes.Inter
VolumeMounts: []corev1.VolumeMount{
{
Name: "host-dev",
- MountPath: "/host/dev",
+ MountPath: "/dev",
},
{
- Name: "host-config",
- MountPath: "/host/opt/monok8s/config",
+ Name: "host-etc",
+ MountPath: "/host/etc",
ReadOnly: true,
},
{
- Name: "host-run",
- MountPath: "/host/run",
- },
- {
- Name: "host-proc",
- MountPath: "/host/proc",
+ Name: "host-config",
+ MountPath: "/host/opt/monok8s/config",
ReadOnly: true,
},
},
@@ -325,28 +309,19 @@ func applyControlAgentDaemonSet(ctx context.Context, kubeClient kubernetes.Inter
},
},
{
- Name: "host-config",
+ Name: "host-etc",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
- Path: "/opt/monok8s/config",
+ Path: "/etc",
Type: hostPathType(corev1.HostPathDirectory),
},
},
},
{
- Name: "host-run",
- VolumeSource: corev1.VolumeSource{
- HostPath: &corev1.HostPathVolumeSource{
- Path: "/run",
- Type: hostPathType(corev1.HostPathDirectory),
- },
- },
- },
- {
- Name: "host-proc",
+ Name: "host-config",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
- Path: "/proc",
+ Path: "/opt/monok8s/config",
Type: hostPathType(corev1.HostPathDirectory),
},
},
diff --git a/clitools/pkg/node/network.go b/clitools/pkg/node/network.go
index b3ee032..7b45b57 100644
--- a/clitools/pkg/node/network.go
+++ b/clitools/pkg/node/network.go
@@ -7,8 +7,8 @@ import (
"os"
"strings"
- "k8s.io/klog/v2"
system "example.com/monok8s/pkg/system"
+ "k8s.io/klog/v2"
)
type NetworkConfig struct {
diff --git a/docs/coredns.md b/docs/coredns.md
new file mode 100644
index 0000000..c3155b3
--- /dev/null
+++ b/docs/coredns.md
@@ -0,0 +1,198 @@
+## About /etc/crio/crio.conf.d/11-coredns.conf
+
+CoreDNS fails with permission denied. Probably because of readonly RootFS.
+The current solution is to just allow port 20 up to bind without root
+
+well this is just brute force allowing any ports 20 < to be yeah go ahead
+```
+[crio.runtime]
+default_sysctls = [
+ "net.ipv4.ip_unprivileged_port_start=20"
+]
+```
+
+CoreDNS wants 53. And users may want to have ssh servers
+
+So just allow all the ports!
+
+
+## Diagnostics
+
+I'll just put some of my findings in here.
+
+Catching effective cap being 0x0000 (this is the issue?)
+```
+kubectl rollout restart deployment -n kube-system coredns
+while true; do
+ PID=$(pgrep coredns)
+ if [ ! -z "$PID" ]; then
+ echo "Found CoreDNS PID: $PID"
+ grep Cap /proc/$PID/status
+ break
+ fi
+done
+
+CapInh: 0000000000000000
+CapPrm: 0000000000000000
+CapEff: 0000000000000000
+CapBnd: 0000000000000400
+CapAmb: 0000000000000000
+```
+
+# find /var/lib/containers/storage/overlay-containers -name config.json | xargs grep "noNewPrivileges"
+
+Current kernel config
+```
+gunzip -c /proc/config.gz | grep -e SECURITY -e LSM -e SECCOMP
+CONFIG_HAVE_ARCH_SECCOMP=y
+CONFIG_HAVE_ARCH_SECCOMP_FILTER=y
+CONFIG_SECCOMP=y
+CONFIG_SECCOMP_FILTER=y
+# CONFIG_SECCOMP_CACHE_DEBUG is not set
+CONFIG_IIO_ST_LSM6DSX=m
+CONFIG_IIO_ST_LSM6DSX_I2C=m
+CONFIG_IIO_ST_LSM6DSX_SPI=m
+CONFIG_IIO_ST_LSM6DSX_I3C=m
+# CONFIG_IIO_ST_LSM9DS0 is not set
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_UBIFS_FS_SECURITY=y
+CONFIG_NFS_V4_SECURITY_LABEL=y
+# CONFIG_9P_FS_SECURITY is not set
+# CONFIG_SECURITY_DMESG_RESTRICT is not set
+CONFIG_SECURITY=y
+CONFIG_HAS_SECURITY_AUDIT=y
+CONFIG_SECURITYFS=y
+CONFIG_SECURITY_NETWORK=y
+CONFIG_SECURITY_NETWORK_XFRM=y
+CONFIG_SECURITY_PATH=y
+# CONFIG_SECURITY_SELINUX is not set
+# CONFIG_SECURITY_SMACK is not set
+# CONFIG_SECURITY_TOMOYO is not set
+# CONFIG_SECURITY_APPARMOR is not set
+# CONFIG_SECURITY_LOADPIN is not set
+# CONFIG_SECURITY_YAMA is not set
+# CONFIG_SECURITY_SAFESETID is not set
+# CONFIG_SECURITY_LOCKDOWN_LSM is not set
+# CONFIG_SECURITY_LANDLOCK is not set
+# CONFIG_SECURITY_IPE is not set
+CONFIG_DEFAULT_SECURITY_DAC=y
+CONFIG_LSM="landlock,lockdown,yama,loadpin,safesetid,ipe,bpf"
+```
+
+`CONFIG_LSM` does not need `capability`. It works when testing CAP_NET_BIND manually.
+```
+# capsh --keep=1 --user=nobody --inh=cap_net_bind_service --addamb=cap_net_bind_service -- -c "grep Cap /proc/self/status"
+CapInh: 0000000000000400
+CapPrm: 0000000000000400
+CapEff: 0000000000000400
+CapBnd: 000001ffffffffff
+CapAmb: 0000000000000400
+```
+
+And this also aligns with debian's `CONFIG_LSM`. They don't have `capability` hardcoded in.
+
+Debian's working kernel config
+```
+grep -e SECURITY -e LSM -e SECCOMP /boot/config-$(uname -r)
+CONFIG_BPF_LSM=y
+# CONFIG_NFIT_SECURITY_DEBUG is not set
+CONFIG_HAVE_ARCH_SECCOMP=y
+CONFIG_HAVE_ARCH_SECCOMP_FILTER=y
+CONFIG_SECCOMP=y
+CONFIG_SECCOMP_FILTER=y
+# CONFIG_SECCOMP_CACHE_DEBUG is not set
+CONFIG_IP_NF_SECURITY=m
+CONFIG_IP6_NF_SECURITY=m
+CONFIG_IIO_ST_LSM6DSX=m
+CONFIG_IIO_ST_LSM6DSX_I2C=m
+CONFIG_IIO_ST_LSM6DSX_SPI=m
+# CONFIG_IIO_ST_LSM9DS0 is not set
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_REISERFS_FS_SECURITY=y
+CONFIG_JFS_SECURITY=y
+CONFIG_F2FS_FS_SECURITY=y
+CONFIG_JFFS2_FS_SECURITY=y
+CONFIG_UBIFS_FS_SECURITY=y
+CONFIG_EROFS_FS_SECURITY=y
+CONFIG_NFS_V4_SECURITY_LABEL=y
+CONFIG_NFSD_V4_SECURITY_LABEL=y
+# CONFIG_CEPH_FS_SECURITY_LABEL is not set
+CONFIG_9P_FS_SECURITY=y
+CONFIG_SECURITY_DMESG_RESTRICT=y
+CONFIG_SECURITY_PERF_EVENTS_RESTRICT=y
+CONFIG_SECURITY=y
+CONFIG_SECURITYFS=y
+CONFIG_SECURITY_NETWORK=y
+# CONFIG_SECURITY_INFINIBAND is not set
+CONFIG_SECURITY_NETWORK_XFRM=y
+CONFIG_SECURITY_PATH=y
+CONFIG_LSM_MMAP_MIN_ADDR=65536
+CONFIG_SECURITY_SELINUX=y
+# CONFIG_SECURITY_SELINUX_BOOTPARAM is not set
+# CONFIG_SECURITY_SELINUX_DISABLE is not set
+CONFIG_SECURITY_SELINUX_DEVELOP=y
+CONFIG_SECURITY_SELINUX_AVC_STATS=y
+CONFIG_SECURITY_SELINUX_CHECKREQPROT_VALUE=0
+CONFIG_SECURITY_SELINUX_SIDTAB_HASH_BITS=9
+CONFIG_SECURITY_SELINUX_SID2STR_CACHE_SIZE=256
+# CONFIG_SECURITY_SMACK is not set
+CONFIG_SECURITY_TOMOYO=y
+CONFIG_SECURITY_TOMOYO_MAX_ACCEPT_ENTRY=2048
+CONFIG_SECURITY_TOMOYO_MAX_AUDIT_LOG=1024
+# CONFIG_SECURITY_TOMOYO_OMIT_USERSPACE_LOADER is not set
+CONFIG_SECURITY_TOMOYO_POLICY_LOADER="/sbin/tomoyo-init"
+CONFIG_SECURITY_TOMOYO_ACTIVATION_TRIGGER="/sbin/init"
+# CONFIG_SECURITY_TOMOYO_INSECURE_BUILTIN_SETTING is not set
+CONFIG_SECURITY_APPARMOR=y
+# CONFIG_SECURITY_APPARMOR_DEBUG is not set
+CONFIG_SECURITY_APPARMOR_INTROSPECT_POLICY=y
+CONFIG_SECURITY_APPARMOR_HASH=y
+CONFIG_SECURITY_APPARMOR_HASH_DEFAULT=y
+CONFIG_SECURITY_APPARMOR_EXPORT_BINARY=y
+CONFIG_SECURITY_APPARMOR_PARANOID_LOAD=y
+# CONFIG_SECURITY_LOADPIN is not set
+CONFIG_SECURITY_YAMA=y
+# CONFIG_SECURITY_SAFESETID is not set
+CONFIG_SECURITY_LOCKDOWN_LSM=y
+CONFIG_SECURITY_LOCKDOWN_LSM_EARLY=y
+CONFIG_SECURITY_LANDLOCK=y
+CONFIG_IMA_LSM_RULES=y
+# CONFIG_DEFAULT_SECURITY_SELINUX is not set
+# CONFIG_DEFAULT_SECURITY_TOMOYO is not set
+CONFIG_DEFAULT_SECURITY_APPARMOR=y
+# CONFIG_DEFAULT_SECURITY_DAC is not set
+CONFIG_LSM="landlock,lockdown,yama,loadpin,safesetid,integrity,apparmor,selinux,smack,tomoyo,bpf"
+```
+
+Debian's CoreDNS cap
+```
+# grep Cap /proc/1911/status
+CapInh: 0000000000000000
+CapPrm: 0000000000000400
+CapEff: 0000000000000400
+CapBnd: 0000000000000400
+CapAmb: 0000000000000000
+```
+
+Debian's CRIO config
+```
+# cat /etc/crio/crio.conf.d/10-crio.conf
+[crio.image]
+signature_policy = "/etc/crio/policy.json"
+
+[crio.runtime]
+default_runtime = "crun"
+
+[crio.runtime.runtimes.crun]
+runtime_path = "/usr/libexec/crio/crun"
+runtime_root = "/run/crun"
+monitor_path = "/usr/libexec/crio/conmon"
+allowed_annotations = [
+ "io.containers.trace-syscall",
+]
+
+[crio.runtime.runtimes.runc]
+runtime_path = "/usr/libexec/crio/runc"
+runtime_root = "/run/runc"
+monitor_path = "/usr/libexec/crio/conmon"
+```
diff --git a/docs/uboot.md b/docs/uboot.md
index 7b29601..e393d43 100644
--- a/docs/uboot.md
+++ b/docs/uboot.md
@@ -1,4 +1,4 @@
-Booting release image with bootusb
+## Booting release image with bootusb
- Use this if you are flashing image from macos/flashusb.sh
```
setenv bootusb '
@@ -18,8 +18,16 @@ setenv bootusb 'usb start; ext4load usb 0:2 ${mks_loadaddr} /boot/kernel.itb; bo
run bootusb
```
+## Run fw_printenv and fw_setenv from kubectl
+```
+# Avoid using daemonset/control-agent if you have multiple nodes
+kubectl exec -n kube-system control-agent-abcdef1 -- /ctl internal fw-setenv --key foo --value bar
+
+# fw_printenv
+kubectl exec -n kube-system ds/control-agent -- /ctl internal fw-printenv --key foo
+```
-Original uboot env from mono
+## Original uboot env from mono
```
arch=arm
baudrate=115200
diff --git a/kernel-extra.config b/kernel-extra.config
index efd4499..833cb56 100644
--- a/kernel-extra.config
+++ b/kernel-extra.config
@@ -293,6 +293,14 @@ CONFIG_SECCOMP_FILTER=y
# CONFIG_SECURITY=y
# CONFIG_SECURITYFS=y
+CONFIG_SECURITY_NETWORK=y
+# enables Security Module (LSM) hooks for network operations. CoreDNS needs this
+
+CONFIG_SECURITY_PATH=y
+# Recommended for container isolation
+
+CONFIG_SECURITY_NETWORK_XFRM=y
+
###############################################################################
# Misc userspace/container compatibility
diff --git a/makefile b/makefile
index 48a64ae..911a542 100644
--- a/makefile
+++ b/makefile
@@ -231,20 +231,6 @@ $(RELEASE_IMAGE): $(RELEASE_DEPS) | $(OUT_DIR)
test -f $@
-check-functions:
- @echo "Checking function calls vs definitions..."
- @awk '/^main\(\)/ { in_main=1; next } \
- in_main && /^\}/ { in_main=0 } \
- in_main && /^[[:space:]]*[a-zA-Z_][a-zA-Z0-9_]*[[:space:]]*$$/ { \
- gsub(/^[ \t]+/, "", $$0); \
- print $$0 \
- }' ./alpine/rootfs-extra/opt/scripts/bootstrap-cluster.sh \
- | sort -u > /tmp/called.txt
- @grep -E '^[a-zA-Z_][a-zA-Z0-9_]*\(\)' ./alpine/rootfs-extra/opt/scripts/bootstrap-cluster.sh \
- | sed 's/().*//' \
- | sort -u > /tmp/defined.txt
- @echo "Missing functions:"
- @comm -23 /tmp/called.txt /tmp/defined.txt || true
# ---- config targets ------------------------------------------------------------