Skip to content

Commit

Permalink
Implemented nic pool instead of specify via mac address. See also REA…
Browse files Browse the repository at this point in the history 
…DME.
  • Loading branch information
@Tydus
Tydus committed Oct 8, 2022
1 parent a570c25 commit 9182539
Show file tree
Hide file tree
Showing 2 changed files with 70 additions and 107 deletions.
18 changes: 12 additions & 6 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,24 +19,30 @@ sudo systemctl enable docker-plugin-hostnic
sudo systemctl start docker-plugin-hostnic
```

3. Create hostnic network，the subnet and gateway argument should be same as hostnic.
3. Create hostnic network with a list of candidate nics (specified by `-o niclist=eth1,eth3`).

```bash
docker network create -d hostnic \
-o niclist=eth1,eth3 \
--subnet=192.168.1.0/24 --gateway 192.168.1.1 --ip-range 192.168.1.128/25 \
--ipv6 --subnet 2001:db8::/64 --gateway 2001:db8:0::ffff \
network1
```

4. Run a container and binding a special host nic. We use `--mac-address` argument to identify the hostnic. Please ensure that the `--ip` argument do not conflict with other container in the same network.
4. Run a container and binding an nic in the pool.

```bash
docker run -it --ip 192.168.1.5 --mac-address 52:54:0e:e5:00:f7 --network network1 ubuntu:22.04 bash
docker run -it --mac-address 52:54:0e:e5:00:f7 --network network1 ubuntu:22.04 bash
```

If the `--mac-address` argument is specified, the plugin will try to find the specific nic in the pool that matches the mac address, and fail if it's not available.

```bash
docker run -it --ip 192.168.1.135 --mac-address 52:54:0e:e5:00:f7 --network network1 ubuntu:22.04 bash
```

## Additional Notes:
0. It is **strongly recommended** to run it directly on host (v.s. in a container).
Otherwise, it may cause dependency problem while restarting docker daemon and make it *very slow* to reboot a server.
1. If the `--ip` argument is not passed when running container, docker will assign an ip to the container, so please pass the `--ip` argument and ensure that the ip do not conflict with other containers.
2. Network config persistent is in `/etc/docker/hostnic/config.json`.
3. If your host only have one nic, please not use this plugin. If you bind the only nic to a container, your host will lose network connectivity.
1. Network config is stored in `/etc/docker/hostnic/config.json`.
2. If your host only have one nic, please not use this plugin. If you bind the only nic to a container, your host will lose network connectivity.
159 changes: 58 additions & 101 deletions driver/driver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"encoding/json"
"fmt"
"github.com/docker/go-plugins-helpers/network"
"github.com/vishvananda/netlink"
"github.com/Tydus/docker-plugin-hostnic/log"
"io/ioutil"
"net"
Expand Down Expand Up @@ -62,6 +61,7 @@ type Network struct {
ID string
Gateway4 string
Gateway6 string
NICPool []string
endpoints map[string]*Endpoint
}

Expand All @@ -72,21 +72,25 @@ type HostNicDriver struct {
lock sync.RWMutex
}

func (d *HostNicDriver) RegisterNetwork(networkID string, Gateway4 string, Gateway6 string) error {
func (d *HostNicDriver) RegisterNetwork(networkID string, NICPool []string, Gateway4 string, Gateway6 string) error {
if nw := d.getNetworkByGateway(Gateway4); nw != nil {
return fmt.Errorf("Exist network [%s] has the same ipv4 gateway [%s].", nw.ID, Gateway4)
}
if nw := d.getNetworkByGateway(Gateway6); nw != nil {
return fmt.Errorf("Exist network [%s] has the same ipv6 gateway [%s].", nw.ID, Gateway6)
}

// TODO: Check NICPool for overlapping.

nw := Network{
ID: networkID,
Gateway4: Gateway4,
Gateway6: Gateway6,
NICPool: NICPool,
endpoints: make(map[string]*Endpoint),
}
d.networks[networkID] = &nw
log.Info("RegisterNetwork [ %s ] Gateway4 : [ %v ] Gateway6 : [ %v ].", nw.ID, nw.Gateway4, nw.Gateway6)
log.Info("RegisterNetwork [ %s ] NICPool : %v Gateway4 : [ %v ] Gateway6 : [ %v ].", nw.ID, nw.NICPool, nw.Gateway4, nw.Gateway6)
return nil
}

Expand All @@ -101,6 +105,13 @@ func (d *HostNicDriver) CreateNetwork(r *network.CreateNetworkRequest) error {
d.lock.Lock()
defer d.lock.Unlock()

if r.Options["com.docker.network.generic"] == nil ||
r.Options["com.docker.network.generic"].(map[string]interface{})["niclist"] == nil ||
r.Options["com.docker.network.generic"].(map[string]interface{})["niclist"] == "" {
return fmt.Errorf("niclist option must be specified. E.g. '-o niclist=eth1,eth3')")
}
niclist := strings.Split(r.Options["com.docker.network.generic"].(map[string]interface{})["niclist"].(string), ",")

var gw4 string
if r.IPv4Data != nil && len(r.IPv4Data) != 0 && r.IPv4Data[0].Gateway != "" {
gw4 = strings.Split(r.IPv4Data[0].Gateway, "/")[0]
Expand All @@ -111,19 +122,14 @@ func (d *HostNicDriver) CreateNetwork(r *network.CreateNetworkRequest) error {
gw6 = strings.Split(r.IPv6Data[0].Gateway, "/")[0]
}

err := d.RegisterNetwork(r.NetworkID, gw4, gw6)
err := d.RegisterNetwork(r.NetworkID, niclist, gw4, gw6)
if err != nil {
return err
}
d.saveConfig()
return nil
}

func (d *HostNicDriver) AllocateNetwork(r *network.AllocateNetworkRequest) (*network.AllocateNetworkResponse, error) {
log.Debug("AllocateNetwork Called: [ %+v ]", r)
return nil, nil
}

func (d *HostNicDriver) DeleteNetwork(r *network.DeleteNetworkRequest) error {
log.Debug("DeleteNetwork Called: [ %+v ]", r)
d.lock.Lock()
Expand All @@ -132,42 +138,66 @@ func (d *HostNicDriver) DeleteNetwork(r *network.DeleteNetworkRequest) error {
d.saveConfig()
return nil
}

func (d *HostNicDriver) AllocateNetwork(r *network.AllocateNetworkRequest) (*network.AllocateNetworkResponse, error) {
log.Debug("AllocateNetwork Called: [ %+v ]", r)
return nil, nil
}

func (d *HostNicDriver) FreeNetwork(r *network.FreeNetworkRequest) error {
log.Debug("FreeNetwork Called: [ %+v ]", r)
return nil
}

func (d *HostNicDriver) FindFirstUsableNIC(nw *Network, macAddr string) (*HostNic) {
// Find first usable NIC in pool.
// If MacAddress is specified, try to match it. Otherwise, any available one will be returned.

for _, i := range nw.NICPool {
// The "net" package only queries from main network namespace, so the interfaces already in the container will not be counted.
nic, err := net.InterfaceByName(i)
if err != nil {
log.Debug("Fail to get NIC [ %s ] (%s), skip.", i, err.Error())
continue
}

if macAddr != "" && macAddr != nic.HardwareAddr.String() {
log.Debug("NIC [ %s ] exists but macaddr [ %s ] mismatch, skip.", nic.Name, nic.HardwareAddr)
continue
}

// TODO: checks

return &HostNic{Name: nic.Name, HardwareAddr: nic.HardwareAddr.String()}
}
log.Info("No available in network %v, abort.", nw.ID)
return nil
}

func (d *HostNicDriver) CreateEndpoint(r *network.CreateEndpointRequest) (*network.CreateEndpointResponse, error) {
d.lock.Lock()
defer d.lock.Unlock()

log.Debug("CreateEndpoint Called: [ %+v ]", r)
log.Debug("r.Interface: [ %+v ]", r.Interface)
nw := d.networks[r.NetworkID]
log.Debug("r.Interface: [ %+v ], r.Interface.MacAddress: [ %v ].", r.Interface, r.Interface.MacAddress)
log.Debug("r.Options: [ %+v ]", r.Options)

nw := d.networks[r.NetworkID]
if nw == nil {
return nil, fmt.Errorf("Can not find network [ %s ].", r.NetworkID)
}

var hostNic *HostNic

if r.Interface.MacAddress == "" {
if r.Interface.MacAddress == "" {
//Support parameters in driver-opt.
//It is used when the interface is connected to the container after container has been created.
r.Interface.MacAddress = r.Options["mac-address"].(string)
}

if r.Interface.MacAddress == "" {
return nil, fmt.Errorf("Missing --mac-address argument.")
if r.Options["mac-address"] != nil {
r.Interface.MacAddress = r.Options["mac-address"].(string)
}
}

hostNic = d.FindNicByHardwareAddr(r.Interface.MacAddress)

hostNic := d.FindFirstUsableNIC(nw, r.Interface.MacAddress)
if hostNic == nil {
return nil, fmt.Errorf("Can not find host nic by mac address [ %+v ].", r.Interface.MacAddress)
}

if hostNic.endpoint != nil {
return nil, fmt.Errorf("Host nic [%s] has already bound to endpoint [ %+v ].", hostNic.Name, hostNic.endpoint)
return nil, fmt.Errorf("No available NIC found in network [ %+v ].", nw.ID)
}

hostNic.Address = r.Interface.Address
Expand All @@ -184,15 +214,6 @@ func (d *HostNicDriver) CreateEndpoint(r *network.CreateEndpointRequest) (*netwo
hostNic.endpoint = endpoint

endpointInterface := &network.EndpointInterface{}
// if r.Interface.Address == "" {
// endpointInterface.Address = hostNic.Address
// }
// if r.Interface.AddressIPv6 == "" {
// endpointInterface.AddressIPv6 = hostNic.AddressIPv6
// }
// if r.Interface.MacAddress == "" {
// endpointInterface.MacAddress = hostNic.HardwareAddr
// }
resp := &network.CreateEndpointResponse{Interface: endpointInterface}
log.Debug("CreateEndpoint resp interface: [ %+v ] ", resp.Interface)
return resp, nil
Expand Down Expand Up @@ -255,6 +276,7 @@ func (d *HostNicDriver) Join(r *network.JoinRequest) (*network.JoinResponse, err
log.Debug("Join resp : [ %+v ]", resp)
return &resp, nil
}

func (d *HostNicDriver) Leave(r *network.LeaveRequest) error {
log.Debug("Leave Called: [ %+v ]", r)
d.lock.Lock()
Expand Down Expand Up @@ -318,71 +340,6 @@ func (d *HostNicDriver) getNetworkByGateway(gateway string) *Network {
return nil
}

func (d *HostNicDriver) findNicFromInterfaces(hardwareAddr string) *HostNic {
nics, err := net.Interfaces()
if err == nil {
for _, nic := range nics {
if nic.HardwareAddr.String() == hardwareAddr {
return &HostNic{Name: nic.Name, HardwareAddr: nic.HardwareAddr.String()}
}
}
} else {
log.Error("Get Interfaces error: %s", err.Error())
}
return nil
}

func (d *HostNicDriver) findNicFromLinks(hardwareAddr string) *HostNic {
links, err := netlink.LinkList()
if err == nil {
for _, link := range links {
attr := link.Attrs()
if attr.HardwareAddr.String() == hardwareAddr {
return &HostNic{Name: attr.Name, HardwareAddr: attr.HardwareAddr.String()}
}
}
} else {
log.Error("Get LinkList error: %s", err.Error())
}
return nil
}

func (d *HostNicDriver) FindNicByHardwareAddr(hardwareAddr string) *HostNic {
for _, nic := range d.nics {
//ensure nic in cache is exist on host.
if !d.ensureNic(nic) {
log.Info("Delete nic [%+v] from nic table.", nic)
delete(d.nics, nic.HardwareAddr)
continue
}
if nic.HardwareAddr == hardwareAddr {
return nic
}
}
nic := d.findNicFromInterfaces(hardwareAddr)
if nic == nil {
nic = d.findNicFromLinks(hardwareAddr)
}
if nic != nil {
log.Info("Add nic [%+v] to nic table.", nic)
d.nics[nic.HardwareAddr] = nic
}
return nic
}

// ensureNic ensure nic exist and info is update
func (d *HostNicDriver) ensureNic(nic *HostNic) bool {
existNic := d.findNicFromInterfaces(nic.HardwareAddr)
if existNic == nil {
existNic = d.findNicFromLinks(nic.HardwareAddr)
}
if existNic != nil {
// nic dev name may be changed by os, so ensure it is update.
nic.Name = existNic.Name
}
return existNic != nil
}

func (d *HostNicDriver) loadConfig() error {
configFile := fmt.Sprintf("%s/%s", configDir, "config.json")
exists, err := FileExists(configFile)
Expand All @@ -401,7 +358,7 @@ func (d *HostNicDriver) loadConfig() error {
}
log.Info("Load config from [%s].", configFile)
for _, nw := range networks {
d.RegisterNetwork(nw.ID, nw.Gateway4, nw.Gateway6)
d.RegisterNetwork(nw.ID, nw.NICPool, nw.Gateway4, nw.Gateway6)
}
}
return nil
Expand Down

0 comments on commit 9182539

Please sign in to comment.