Skip to content

Vpp-manager refactoring #853

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
sknat wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Vpp-manager refactoring #853

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 42 additions & 16 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,13 +162,13 @@ func RunHook(hookScript *string, hookName string, params *VppManagerParams, log
if *hookScript == "" {
return
}
template, err := TemplateScriptReplace(*hookScript, params, nil)
template, err := TemplateScriptReplace(*hookScript, params)
if err != nil {
log.Warnf("Running hook %s errored with %s", hookName, err)
return
}

cmd := exec.Command("/bin/bash", "-c", template, hookName, params.UplinksSpecs[0].InterfaceName)
cmd := exec.Command("/bin/bash", "-c", template, hookName, params.InterfacesById[0].Spec.InterfaceName)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err = cmd.Run()
Expand Down Expand Up @@ -428,8 +428,6 @@ type CalicoVppInitialConfigConfigType struct { //out of agent and vppmanager
// CorePattern is the pattern to use for VPP corefiles.
// Usually "/var/lib/vpp/vppcore.%e.%p"
CorePattern string `json:"corePattern"`
ExtraAddrCount int `json:"extraAddrCount"`
IfConfigSavePath string `json:"ifConfigSavePath"`
// DefaultGWs Comma separated list of IPs to be
// configured in VPP as default GW
DefaultGWs string `json:"defaultGWs"`
Expand Down Expand Up @@ -619,9 +617,35 @@ const (
VfioUnsafeNoIommuModeDISABLED UnsafeNoIommuMode = "disabled"
)

type UplinkDriver interface {
PreconfigureLinux() error
CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *UplinkInterfaceSpec) error
RestoreLinux()
IsSupported(warn bool) bool
Copy link
Collaborator

@aritrbas aritrbas Dec 8, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unrelated to this PR, but this function for the AF_XDP driver seems to be broken. It is always returning false!

GetName() string
UpdateVppConfigFile(template string) string
GetDefaultRxMode() types.RxMode
}

type VppManagerInterface struct {
Spec UplinkInterfaceSpec
State *LinuxInterfaceState
Driver UplinkDriver
}

func (p *VppManagerParams) AllInterfacesPhysical() bool {
for _, intf := range p.Interfaces {
if intf.State.IsTunTap || intf.State.IsVeth {
return false
}
}
return true
}

type VppManagerParams struct {
UplinksSpecs []UplinkInterfaceSpec
/* Capabilities */
Interfaces map[string]*VppManagerInterface
InterfacesById []*VppManagerInterface
// Capabilities
LoadedDrivers map[string]bool
KernelVersion *KernelVersion
AvailableHugePages int
Expand Down Expand Up @@ -744,23 +768,25 @@ func getCpusetCPU() (string, error) {
return regexp.MustCompile("[,-]").Split(cpusetCPU, 2)[0], nil
}

func TemplateScriptReplace(input string, params *VppManagerParams, conf []*LinuxInterfaceState) (template string, err error) {
func TemplateScriptReplace(input string, params *VppManagerParams) (template string, err error) {
template = input
if conf != nil {
/* We might template scripts before reading interface conf */
template = strings.ReplaceAll(template, "__PCI_DEVICE_ID__", conf[0].PciID)
for i, ifcConf := range conf {
template = strings.ReplaceAll(template, "__PCI_DEVICE_ID_"+strconv.Itoa(i)+"__", ifcConf.PciID)
}
if len(params.Interfaces) > 0 {
// We might template scripts before reading interface conf
template = strings.ReplaceAll(template, "__PCI_DEVICE_ID__", params.InterfacesById[0].State.PciID)
}
for idx, intf := range params.InterfacesById {
template = strings.ReplaceAll(template, "__PCI_DEVICE_ID_"+strconv.Itoa(idx)+"__", intf.State.PciID)
}
vppcpu, err := getCpusetCPU()
if err != nil {
return "", err
}
template = strings.ReplaceAll(template, "__CPUSET_CPUS_FIRST__", vppcpu)
template = strings.ReplaceAll(template, "__VPP_DATAPLANE_IF__", params.UplinksSpecs[0].InterfaceName)
for i, ifc := range params.UplinksSpecs {
template = strings.ReplaceAll(template, "__VPP_DATAPLANE_IF_"+fmt.Sprintf("%d", i)+"__", ifc.InterfaceName)
if len(params.Interfaces) > 0 {
template = strings.ReplaceAll(template, "__VPP_DATAPLANE_IF__", params.InterfacesById[0].Spec.InterfaceName)
}
for idx, intf := range params.InterfacesById {
template = strings.ReplaceAll(template, "__VPP_DATAPLANE_IF_"+fmt.Sprintf("%d", idx)+"__", intf.Spec.InterfaceName)
}
for key, value := range params.NodeAnnotations {
template = strings.ReplaceAll(template, fmt.Sprintf("__NODE_ANNOTATION:%s__", key), value)
Expand Down
32 changes: 13 additions & 19 deletions vpp-manager/hooks/network_manager_hook.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,9 @@ type SystemType struct {

// NetworkManagerHook manages network configuration during VPP lifecycle
type NetworkManagerHook struct {
interfaceNames []string
systemType SystemType
log *logrus.Logger
vppManagerParams *config.VppManagerParams
systemType SystemType
log *logrus.Logger
}

// chrootCommand creates a command that will be executed in the host namespace
Expand Down Expand Up @@ -151,21 +151,16 @@ func (h *NetworkManagerHook) restartService(serviceName string) error {
}

// NewNetworkManagerHook creates a new NetworkManagerHook instance
func NewNetworkManagerHook(log *logrus.Logger) *NetworkManagerHook {
func NewNetworkManagerHook(log *logrus.Logger, vppManagerParams *config.VppManagerParams) *NetworkManagerHook {
hook := &NetworkManagerHook{
log: log,
log: log,
vppManagerParams: vppManagerParams,
}

hook.detectSystem()
return hook
}

// SetInterfaceNames updates the interface names of NetworkManagerHook instance
func (h *NetworkManagerHook) SetInterfaceNames(interfaceNames []string) {
h.interfaceNames = interfaceNames
h.log.Infof("NetworkManagerHook: Interface names updated to %v", interfaceNames)
}

// fixDNS modifies NetworkManager configuration to disable DNS management
func (h *NetworkManagerHook) fixDNS() error {
if !h.systemType.HasNetworkManager {
Expand Down Expand Up @@ -500,7 +495,7 @@ func (h *NetworkManagerHook) beforeVppRun() error {
}

// Save network file for AWS systemd-networkd for each interface
for _, interfaceName := range h.interfaceNames {
for interfaceName := range h.vppManagerParams.Interfaces {
err = h.saveNetworkFile(interfaceName)
if err != nil {
return err
Expand All @@ -519,7 +514,7 @@ func (h *NetworkManagerHook) vppRunning() error {
}

// Tweak network file for AWS systemd-networkd for each interface
for _, interfaceName := range h.interfaceNames {
for interfaceName := range h.vppManagerParams.Interfaces {
err = h.tweakNetworkFile(interfaceName)
if err != nil {
return err
Expand All @@ -538,7 +533,7 @@ func (h *NetworkManagerHook) vppDoneOk() error {
}

// Remove the tweaked network file for AWS systemd-networkd for each interface
for _, interfaceName := range h.interfaceNames {
for interfaceName := range h.vppManagerParams.Interfaces {
err = h.removeTweakedNetworkFile(interfaceName)
if err != nil {
return err
Expand Down Expand Up @@ -566,7 +561,7 @@ func (h *NetworkManagerHook) Execute(hookPoint HookPoint) error {
return nil
}

h.log.Infof("NetworkManagerHook: Executing %s for interfaces %v", hookPoint, h.interfaceNames)
h.log.Infof("NetworkManagerHook: Executing %s for all interfaces", hookPoint)

var err error
switch hookPoint {
Expand Down Expand Up @@ -616,11 +611,10 @@ func (h *NetworkManagerHook) useDefaultHookFallback(userHook *string) bool {
// 2. If native hooks enabled -> run native Go hook
// 3. If native hooks disabled & no user script -> fallback to default_hook.sh
func (h *NetworkManagerHook) ExecuteWithUserScript(hookPoint HookPoint,
hookScript *string,
params *config.VppManagerParams) {
hookScript *string) {
// Check if user script is configured (highest priority - overrides everything)
if hookScript != nil && *hookScript != "" {
config.RunHook(hookScript, string(hookPoint), params, h.log)
config.RunHook(hookScript, string(hookPoint), h.vppManagerParams, h.log)
return
}

Expand All @@ -636,6 +630,6 @@ func (h *NetworkManagerHook) ExecuteWithUserScript(hookPoint HookPoint,
// Fallback to default_hook.sh when native hooks are disabled
if h.useDefaultHookFallback(hookScript) {
h.log.Infof("Native hooks disabled, falling back to default_hook.sh for %s", hookPoint)
config.RunHook(&config.DefaultHookScript, string(hookPoint), params, h.log)
config.RunHook(&config.DefaultHookScript, string(hookPoint), h.vppManagerParams, h.log)
}
}
44 changes: 13 additions & 31 deletions vpp-manager/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,8 +165,8 @@ func main() {

/* Initialize native Go NewNetworkManagerHook with empty interface names
* We will update this later once we have the actual interface names */
networkHook = hooks.NewNetworkManagerHook(log)
networkHook.ExecuteWithUserScript(hooks.HookBeforeIfRead, config.HookScriptBeforeIfRead, params)
networkHook = hooks.NewNetworkManagerHook(log, params)
networkHook.ExecuteWithUserScript(hooks.HookBeforeIfRead, config.HookScriptBeforeIfRead)

err = utils.ClearVppManagerFiles()
if err != nil {
Expand All @@ -183,44 +183,31 @@ func main() {
log.Errorf("Error raising memlock limit, VPP may fail to start: %v", err)
}

confs, err := startup.GetInterfaceConfig(params)
if err != nil {
log.Fatalf("Error getting initial interface configuration: %s", err)
}

/* Update native Go NewNetworkManagerHook with all interface names */
if len(params.UplinksSpecs) > 0 {
interfaceNames := make([]string, len(params.UplinksSpecs))
for i, spec := range params.UplinksSpecs {
interfaceNames[i] = spec.InterfaceName
}
networkHook.SetInterfaceNames(interfaceNames)
}

runningCond = sync.NewCond(&sync.Mutex{})
go handleSignals()

ctx, cancel := context.WithCancel(context.Background())
defer cancel()
config.HandleUsr2Signal(ctx, log.WithFields(logrus.Fields{"component": "sighdlr"}))

startup.PrintVppManagerConfig(params, confs)

runner := NewVPPRunner(params, confs)
startup.PrintVppManagerConfig(params)

makeNewVPPIndex()

if len(params.UplinksSpecs) == 1 && params.UplinksSpecs[0].VppDriver == "" {
for _, driver := range uplink.SupportedUplinkDrivers(params, confs[0], &params.UplinksSpecs[0]) {
if len(params.Interfaces) == 1 && params.InterfacesById[0].Spec.VppDriver == "" {
intf := params.InterfacesById[0]
for _, driver := range uplink.SupportedUplinkDrivers(params, intf) {
err := utils.CleanupCoreFiles(config.GetCalicoVppInitialConfig().CorePattern, maxCoreFiles)
if err != nil {
log.Errorf("CleanupCoreFiles errored %s", err)
}
intf.Driver = driver

internalKill = false
err = runner.Run([]uplink.UplinkDriver{driver})
runner := NewVPPRunner(params)
err = runner.Run()
if err != nil {
networkHook.ExecuteWithUserScript(hooks.HookVppErrored, config.HookScriptVppErrored, params)
networkHook.ExecuteWithUserScript(hooks.HookVppErrored, config.HookScriptVppErrored)
log.Errorf("VPP(%s) run failed with %s", driver.GetName(), err)
}
if vppProcess != nil && !internalKill {
Expand All @@ -236,15 +223,10 @@ func main() {
log.Errorf("CleanupCoreFiles errored %s", err)
}

var drivers []uplink.UplinkDriver
for idx := 0; idx < len(params.UplinksSpecs); idx++ {
drivers = append(drivers, uplink.NewUplinkDriver(params.UplinksSpecs[idx].VppDriver,
params, confs[idx], &params.UplinksSpecs[idx]))
}

err = runner.Run(drivers)
runner := NewVPPRunner(params)
err = runner.Run()
if err != nil {
networkHook.ExecuteWithUserScript(hooks.HookVppErrored, config.HookScriptVppErrored, params)
networkHook.ExecuteWithUserScript(hooks.HookVppErrored, config.HookScriptVppErrored)
log.Errorf("VPP run failed with %v", err)
}

Expand Down
103 changes: 0 additions & 103 deletions vpp-manager/startup/interface_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,7 @@
package startup

import (
"encoding/gob"
"fmt"
"net"
"os"

"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
Expand All @@ -30,56 +27,6 @@ import (
"github.com/projectcalico/vpp-dataplane/v3/vpplink"
)

func GetInterfaceConfig(params *config.VppManagerParams) (conf []*config.LinuxInterfaceState, err error) {
errs := []error{}
conf = []*config.LinuxInterfaceState{}
for _, ifSpec := range params.UplinksSpecs {
configuration, err := loadInterfaceConfigFromLinux(ifSpec)
errs = append(errs, err)
conf = append(conf, configuration)
}
allLoaded := true
for i := range errs {
if errs[i] != nil {
allLoaded = false
log.Warnf("Could not load config from linux (%v)", errs[i])
}
}
if allLoaded {
err = saveConfig(params, conf)
if err != nil {
log.Warnf("Could not save interface config: %v", err)
}
} else {
// Loading config failed, try loading from save file
confFile, err2 := loadInterfaceConfigFromFile(params)
if err2 != nil {
return nil, err2
}
// If loaded from file replace non loaded interface configs
for i := range conf {
if conf[i] == nil {
for j := range confFile {
if confFile[j].InterfaceName == params.UplinksSpecs[i].InterfaceName {
conf[i] = confFile[j]
}
}
}
}
for i := range conf {
if conf[i] == nil {
return nil, fmt.Errorf("interface configs not found")
}
}
log.Infof("Loaded config. Interfaces marked as down since loading config from linux failed.")
// This ensures we don't try to set the interface down in runVpp()
for _, config := range conf {
config.IsUp = false
}
}
return conf, nil
}

func loadInterfaceConfigFromLinux(ifSpec config.UplinkInterfaceSpec) (*config.LinuxInterfaceState, error) {
conf := config.LinuxInterfaceState{}
link, err := netlink.LinkByName(ifSpec.InterfaceName)
Expand Down Expand Up @@ -146,53 +93,3 @@ func getNodeAddress(conf *config.LinuxInterfaceState, isV6 bool) string {
}
return ""
}

func clearSavedConfig(params *config.VppManagerParams) {
if config.GetCalicoVppInitialConfig().IfConfigSavePath == "" {
return
}
err := os.Remove(config.GetCalicoVppInitialConfig().IfConfigSavePath)
if err != nil {
log.Warnf("could not delete saved interface config: %v", err)
}
}

func saveConfig(params *config.VppManagerParams, conf []*config.LinuxInterfaceState) error {
if config.GetCalicoVppInitialConfig().IfConfigSavePath == "" {
return nil
}
file, err := os.Create(config.GetCalicoVppInitialConfig().IfConfigSavePath)
if err != nil {
return errors.Wrap(err, "error opening save file")
}
enc := gob.NewEncoder(file)
err = enc.Encode(conf)
if err != nil {
file.Close()
clearSavedConfig(params)
return errors.Wrap(err, "error encoding data")
}
err = file.Close()
if err != nil {
return errors.Wrap(err, "error closing file")
}
return nil
}

func loadInterfaceConfigFromFile(params *config.VppManagerParams) ([]*config.LinuxInterfaceState, error) {
conf := []*config.LinuxInterfaceState{}
if config.GetCalicoVppInitialConfig().IfConfigSavePath == "" {
return nil, fmt.Errorf("interface config save file not configured")
}
file, err := os.Open(config.GetCalicoVppInitialConfig().IfConfigSavePath)
if err != nil {
return nil, errors.Wrap(err, "error opening save file")
}
dec := gob.NewDecoder(file)
err = dec.Decode(&conf)
if err != nil {
return nil, errors.Wrap(err, "decode error")
}
file.Close()
return conf, nil
}
Loading
Loading