From 66860e593dd4c238579f050db5da9e03293c2fb5 Mon Sep 17 00:00:00 2001 From: Dave Russell Date: Wed, 12 Aug 2015 21:33:12 -0700 Subject: [PATCH] libnetwork prep for docker port persistence policies. This is the libnetwork portion of work required to define port persistence policies (see docker/docker#12622) Remains backward-compatible with docker/docker master Signed-off-by: Dave Russell --- drivers/bridge/port_mapping.go | 13 +++---- libnetwork_test.go | 9 +++-- portallocator/portallocator.go | 48 +++++++++++++++++++------ portallocator/portallocator_test.go | 55 +++++++++++++++++++++++++++-- portmapper/mapper.go | 11 ++++-- types/types.go | 29 ++++++++------- 6 files changed, 125 insertions(+), 40 deletions(-) diff --git a/drivers/bridge/port_mapping.go b/drivers/bridge/port_mapping.go index 4dab8a0c89..e05ca08c3f 100644 --- a/drivers/bridge/port_mapping.go +++ b/drivers/bridge/port_mapping.go @@ -57,11 +57,6 @@ func (n *bridgeNetwork) allocatePort(bnd *types.PortBinding, containerIP, defHos bnd.HostIP = defHostIP } - // Adjust HostPortEnd if this is not a range. - if bnd.HostPortEnd == 0 { - bnd.HostPortEnd = bnd.HostPort - } - // Construct the container side transport address container, err := bnd.ContainerAddr() if err != nil { @@ -70,12 +65,12 @@ func (n *bridgeNetwork) allocatePort(bnd *types.PortBinding, containerIP, defHos // Try up to maxAllocatePortAttempts times to get a port that's not already allocated. for i := 0; i < maxAllocatePortAttempts; i++ { - if host, err = n.portMapper.MapRange(container, bnd.HostIP, int(bnd.HostPort), int(bnd.HostPortEnd), ulPxyEnabled); err == nil { + if host, err = n.portMapper.PreferredMapFromRange(container, bnd.HostIP, int(bnd.HostPort), int(bnd.HostPortStart), int(bnd.HostPortEnd), ulPxyEnabled); err == nil { break } - // There is no point in immediately retrying to map an explicitly chosen port. - if bnd.HostPort != 0 { - logrus.Warnf("Failed to allocate and map port %d-%d: %s", bnd.HostPort, bnd.HostPortEnd, err) + // There is no point in immediately retrying to map an explicitly chosen port without a range. + if bnd.HostPort != 0 && bnd.HostPortStart != 0 { + logrus.Warnf("Failed to allocate and map port %d (from range %d-%d): %s", bnd.HostPort, bnd.HostPortStart, bnd.HostPortEnd, err) break } logrus.Warnf("Failed to allocate and map port: %s, retry: %d", err, i+1) diff --git a/libnetwork_test.go b/libnetwork_test.go index f2f9407cf9..4f4504032a 100644 --- a/libnetwork_test.go +++ b/libnetwork_test.go @@ -92,8 +92,11 @@ func getPortMapping() []types.PortBinding { {Proto: types.TCP, Port: uint16(230), HostPort: uint16(23000)}, {Proto: types.UDP, Port: uint16(200), HostPort: uint16(22000)}, {Proto: types.TCP, Port: uint16(120), HostPort: uint16(12000)}, - {Proto: types.TCP, Port: uint16(320), HostPort: uint16(32000), HostPortEnd: uint16(32999)}, - {Proto: types.UDP, Port: uint16(420), HostPort: uint16(42000), HostPortEnd: uint16(42001)}, + {Proto: types.TCP, Port: uint16(320), HostPortStart: uint16(32000), HostPortEnd: uint16(32999)}, + {Proto: types.UDP, Port: uint16(420), HostPortStart: uint16(42000), HostPortEnd: uint16(42001)}, + {Proto: types.TCP, Port: uint16(330), HostPort: uint16(33998), HostPortStart: uint16(33000), HostPortEnd: uint16(33999)}, + {Proto: types.TCP, Port: uint16(331), HostPort: uint16(33998), HostPortStart: uint16(33000), HostPortEnd: uint16(33999)}, + {Proto: types.UDP, Port: uint16(430), HostPort: uint16(43000), HostPortStart: uint16(43000), HostPortEnd: uint16(43001)}, } } @@ -281,7 +284,7 @@ func TestBridge(t *testing.T) { if !ok { t.Fatalf("Unexpected format for port mapping in endpoint operational data") } - if len(pm) != 5 { + if len(pm) != 8 { t.Fatalf("Incomplete data for port mapping in endpoint operational data: %d", len(pm)) } diff --git a/portallocator/portallocator.go b/portallocator/portallocator.go index 240e94fc82..6be48e8a65 100644 --- a/portallocator/portallocator.go +++ b/portallocator/portallocator.go @@ -7,6 +7,8 @@ import ( "net" "os" "sync" + + "github.com/Sirupsen/logrus" ) const ( @@ -138,7 +140,16 @@ func (p *PortAllocator) RequestPort(ip net.IP, proto string, port int) (int, err // If portStart != portEnd it returns the first free port in the requested range. // Otherwise (portStart == portEnd) it checks port availability in the requested proto's port-pool // and returns that port or error if port is already busy. -func (p *PortAllocator) RequestPortInRange(ip net.IP, proto string, portStart, portEnd int) (int, error) { +func (p *PortAllocator) RequestPortInRange(ip net.IP, proto string, portStart int, portEnd int) (int, error) { + return p.RequestPreferredPortInRange(ip, proto, 0, portStart, portEnd) +} + +// RequestPreferredPortInRange allows caller to specify a preferred specific port and a fallback range. +// If port, portStart and portEnd are all 0 it returns the first free port in the default ephemeral range. +// If port is 0 and portStart < portEnd it returns the first free port in the requested range. +// If port != 0, we allocate the specified port if it is available, or the first free port in the specified custom range. +// When no range is specified, returns error if port is already busy. +func (p *PortAllocator) RequestPreferredPortInRange(ip net.IP, proto string, port int, portStart int, portEnd int) (int, error) { p.mutex.Lock() defer p.mutex.Unlock() @@ -160,12 +171,29 @@ func (p *PortAllocator) RequestPortInRange(ip net.IP, proto string, portStart, p p.ipMap[ipstr] = protomap } mapping := protomap[proto] - if portStart > 0 && portStart == portEnd { - if _, ok := mapping.p[portStart]; !ok { - mapping.p[portStart] = struct{}{} - return portStart, nil + + // Fixup request for dynamic port from range of size 1 + if port == 0 && portStart == portEnd { + port = portStart + } + // Catch a preferred port with an invalid range + if port != 0 && portStart != 0 && + (port < portStart || port > portEnd) { + return 0, fmt.Errorf("invalid port range %d-%d for requested port: %d", portStart, portEnd, port) + } + + if port > 0 { + if _, ok := mapping.p[port]; !ok { + mapping.p[port] = struct{}{} + return port, nil + } + // If a custom range is specified, we can try to auto-allocate again from the range. + if portStart != 0 && portStart != portEnd && port >= portStart && port <= portEnd { + warn := fmt.Sprintf("Port %d/%s is busy, re-allocating from specified range: %d-%d", port, proto, portStart, portEnd) + logrus.Warn(warn) + } else { + return 0, newErrPortAlreadyAllocated(ipstr, port) } - return 0, newErrPortAlreadyAllocated(ipstr, portStart) } port, err := mapping.findPort(portStart, portEnd) @@ -211,11 +239,11 @@ func (p *PortAllocator) ReleaseAll() error { return nil } -func getRangeKey(portStart, portEnd int) string { +func getRangeKey(portStart int, portEnd int) string { return fmt.Sprintf("%d-%d", portStart, portEnd) } -func newPortRange(portStart, portEnd int) *portRange { +func newPortRange(portStart int, portEnd int) *portRange { return &portRange{ begin: portStart, end: portEnd, @@ -223,7 +251,7 @@ func newPortRange(portStart, portEnd int) *portRange { } } -func (pm *portMap) getPortRange(portStart, portEnd int) (*portRange, error) { +func (pm *portMap) getPortRange(portStart int, portEnd int) (*portRange, error) { var key string if portStart == 0 && portEnd == 0 { key = pm.defaultRange @@ -247,7 +275,7 @@ func (pm *portMap) getPortRange(portStart, portEnd int) (*portRange, error) { return pr, nil } -func (pm *portMap) findPort(portStart, portEnd int) (int, error) { +func (pm *portMap) findPort(portStart int, portEnd int) (int, error) { pr, err := pm.getPortRange(portStart, portEnd) if err != nil { return 0, err diff --git a/portallocator/portallocator_test.go b/portallocator/portallocator_test.go index ffabb778ac..875505a18a 100644 --- a/portallocator/portallocator_test.go +++ b/portallocator/portallocator_test.go @@ -254,10 +254,10 @@ func TestPortAllocationWithCustomRange(t *testing.T) { t.Fatalf("Expected error for invalid range %d-%d", 0, end) } if _, err := p.RequestPortInRange(defaultIP, "tcp", start, 0); err == nil { - t.Fatalf("Expected error for invalid range %d-%d", 0, end) + t.Fatalf("Expected error for invalid range %d-%d", start, 0) } if _, err := p.RequestPortInRange(defaultIP, "tcp", 8081, 8080); err == nil { - t.Fatalf("Expected error for invalid range %d-%d", 0, end) + t.Fatalf("Expected error for invalid range %d-%d", 8081, 8080) } //request a single port @@ -302,6 +302,57 @@ func TestPortAllocationWithCustomRange(t *testing.T) { } } +func TestPreferredPortAllocationWithCustomRange(t *testing.T) { + p := Get() + defer resetPortAllocator() + + start, end := 8083, 8084 + specificPort := 8000 + + //get a preferred port from the range + port1, err := p.RequestPreferredPortInRange(defaultIP, "tcp", start, start, end) + if err != nil { + t.Fatal(err) + } + if port1 != start { + t.Fatalf("Expected preferred port %d, got %d", start, port1) + } + + //try for the same port again, should get a different port + port2, err := p.RequestPreferredPortInRange(defaultIP, "tcp", start, start, end) + if err != nil { + t.Fatal(err) + } + if port2 == start { + t.Fatalf("Expected preferred port %d to be busy, but got it allocated", start) + } + + //try to get a preferred port from an invalid range for that port + if _, err := p.RequestPreferredPortInRange(defaultIP, "tcp", 8100, 8200, 8300); err == nil { + t.Fatalf("Expected error for invalid range %d-%d for port %d", 8200, 8300, 8100) + } + + //request a single port with empty range + port3, err := p.RequestPreferredPortInRange(defaultIP, "tcp", specificPort, 0, 0) + if err != nil { + t.Fatal(err) + } + if port3 != specificPort { + t.Fatalf("Expected port %d, got %d", specificPort, port3) + } + + //request a single port with empty range again, should be busy + if port4, err := p.RequestPreferredPortInRange(defaultIP, "tcp", specificPort, 0, 0); err == nil { + t.Fatalf("Expected port allocation error, got port: %d", port4) + } else { + switch err.(type) { + case ErrPortAlreadyAllocated: + default: + t.Fatalf("Expected port allocation error got %s", err) + } + } +} + func TestNoDuplicateBPR(t *testing.T) { p := Get() defer resetPortAllocator() diff --git a/portmapper/mapper.go b/portmapper/mapper.go index bbdedaa3ca..54a8f20d42 100644 --- a/portmapper/mapper.go +++ b/portmapper/mapper.go @@ -66,7 +66,12 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int, usePr } // MapRange maps the specified container transport address to the host's network address and transport port range -func (pm *PortMapper) MapRange(container net.Addr, hostIP net.IP, hostPortStart, hostPortEnd int, useProxy bool) (host net.Addr, err error) { +func (pm *PortMapper) MapRange(container net.Addr, hostIP net.IP, hostPortStart int, hostPortEnd int, useProxy bool) (host net.Addr, err error) { + return pm.PreferredMapFromRange(container, hostIP, 0, hostPortStart, hostPortEnd, useProxy) +} + +// PreferredMapFromRange maps the specified container transport address to the host's network address and transport port range, trying the preferred port first +func (pm *PortMapper) PreferredMapFromRange(container net.Addr, hostIP net.IP, hostPort int, hostPortStart int, hostPortEnd int, useProxy bool) (host net.Addr, err error) { pm.lock.Lock() defer pm.lock.Unlock() @@ -79,7 +84,7 @@ func (pm *PortMapper) MapRange(container net.Addr, hostIP net.IP, hostPortStart, switch container.(type) { case *net.TCPAddr: proto = "tcp" - if allocatedHostPort, err = pm.Allocator.RequestPortInRange(hostIP, proto, hostPortStart, hostPortEnd); err != nil { + if allocatedHostPort, err = pm.Allocator.RequestPreferredPortInRange(hostIP, proto, hostPort, hostPortStart, hostPortEnd); err != nil { return nil, err } @@ -96,7 +101,7 @@ func (pm *PortMapper) MapRange(container net.Addr, hostIP net.IP, hostPortStart, } case *net.UDPAddr: proto = "udp" - if allocatedHostPort, err = pm.Allocator.RequestPortInRange(hostIP, proto, hostPortStart, hostPortEnd); err != nil { + if allocatedHostPort, err = pm.Allocator.RequestPreferredPortInRange(hostIP, proto, hostPort, hostPortStart, hostPortEnd); err != nil { return nil, err } diff --git a/types/types.go b/types/types.go index 176f40d0f1..22b0816b79 100644 --- a/types/types.go +++ b/types/types.go @@ -24,12 +24,13 @@ func (t *TransportPort) GetCopy() TransportPort { // PortBinding represent a port binding between the container and the host type PortBinding struct { - Proto Protocol - IP net.IP - Port uint16 - HostIP net.IP - HostPort uint16 - HostPortEnd uint16 + Proto Protocol + IP net.IP + Port uint16 + HostIP net.IP + HostPort uint16 + HostPortStart uint16 + HostPortEnd uint16 } // HostAddr returns the host side transport address @@ -59,12 +60,13 @@ func (p PortBinding) ContainerAddr() (net.Addr, error) { // GetCopy returns a copy of this PortBinding structure instance func (p *PortBinding) GetCopy() PortBinding { return PortBinding{ - Proto: p.Proto, - IP: GetIPCopy(p.IP), - Port: p.Port, - HostIP: GetIPCopy(p.HostIP), - HostPort: p.HostPort, - HostPortEnd: p.HostPortEnd, + Proto: p.Proto, + IP: GetIPCopy(p.IP), + Port: p.Port, + HostIP: GetIPCopy(p.HostIP), + HostPort: p.HostPort, + HostPortStart: p.HostPortStart, + HostPortEnd: p.HostPortEnd, } } @@ -79,7 +81,8 @@ func (p *PortBinding) Equal(o *PortBinding) bool { } if p.Proto != o.Proto || p.Port != o.Port || - p.HostPort != o.HostPort || p.HostPortEnd != o.HostPortEnd { + p.HostPort != o.HostPort || p.HostPortStart != o.HostPortStart || + p.HostPortEnd != o.HostPortEnd { return false }