Skip to content

Commit 8728c2a

Browse files
clintaclinta
committed
move locking into their own files
1 parent ade9d3c commit 8728c2a

File tree

4 files changed

+147
-121
lines changed

4 files changed

+147
-121
lines changed

‎docker/core/core.go

Lines changed: 2 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,6 @@ type Core struct {
3232
putNr chan *types.NetworkResource
3333
}
3434

35-
type getNr struct {
36-
s string
37-
rc chan<- *types.NetworkResource
38-
}
39-
4035
// New creates a new client
4136
func New(propTime, respTime time.Duration) (*Core, error) {
4237
dc, err := client.NewEnvClient()
@@ -57,42 +52,6 @@ func New(propTime, respTime time.Duration) (*Core, error) {
5752
return c, nil
5853
}
5954

60-
func nrCacheLoop(getNr <-chan *getNr, delNr <-chan string, putNr <-chan *types.NetworkResource) {
61-
nrCache := make(map[string]*types.NetworkResource)
62-
for {
63-
select {
64-
case rc := <-getNr:
65-
rc.rc <- nrCache[rc.s]
66-
case dn := <-delNr:
67-
nr := nrCache[dn]
68-
if nr == nil {
69-
break
70-
}
71-
delete(nrCache, nr.ID)
72-
pool, err := poolFromNR(nr)
73-
if err != nil {
74-
log.Debug("failed to get pool from network resource, not deleting")
75-
break
76-
}
77-
delete(nrCache, pool)
78-
case nr := <-putNr:
79-
nrCache[nr.ID] = nr
80-
pool, err := poolFromNR(nr)
81-
if err != nil {
82-
log.Debug("failed to get pool from network resource, not caching")
83-
break
84-
}
85-
nrCache[pool] = nr
86-
}
87-
}
88-
}
89-
90-
func (c *Core) getNrFromCache(s string) *types.NetworkResource {
91-
rc := make(chan *types.NetworkResource)
92-
c.getNr <- &getNr{s, rc}
93-
return <-rc
94-
}
95-
9655
// getNetworkResourceByID gets a network resource by ID (checks cache first)
9756
func (c *Core) getNetworkResourceByID(id string) (*types.NetworkResource, error) {
9857
log := log.WithField("net_id", id)
@@ -113,7 +72,7 @@ func (c *Core) getNetworkResourceByID(id string) (*types.NetworkResource, error)
11372
}
11473
nr = &nnr
11574

116-
c.putNr <- nr
75+
c.putNrInCache(nr)
11776

11877
return nr, nil
11978
}
@@ -155,7 +114,7 @@ func (c *Core) getNetworkResourceByPool(pool string) (*types.NetworkResource, er
155114
// Uncache uncaches the network resources
156115
func (c *Core) Uncache(poolid string) {
157116
pool := poolFromID(poolid)
158-
c.delNr <- pool
117+
c.delNrInCache(pool)
159118
}
160119

161120
// ConnectAndGetAddress connects the host to the network for the

‎docker/core/lock.go

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
package core
2+
3+
import (
4+
log "github.com/Sirupsen/logrus"
5+
"github.com/docker/docker/api/types"
6+
)
7+
8+
type getNr struct {
9+
s string
10+
rc chan<- *types.NetworkResource
11+
}
12+
13+
func nrCacheLoop(getNr <-chan *getNr, delNr <-chan string, putNr <-chan *types.NetworkResource) {
14+
nrCache := make(map[string]*types.NetworkResource)
15+
for {
16+
select {
17+
case rc := <-getNr:
18+
rc.rc <- nrCache[rc.s]
19+
case dn := <-delNr:
20+
nr := nrCache[dn]
21+
if nr == nil {
22+
break
23+
}
24+
delete(nrCache, nr.ID)
25+
pool, err := poolFromNR(nr)
26+
if err != nil {
27+
log.Debug("failed to get pool from network resource, not deleting")
28+
break
29+
}
30+
delete(nrCache, pool)
31+
case nr := <-putNr:
32+
nrCache[nr.ID] = nr
33+
pool, err := poolFromNR(nr)
34+
if err != nil {
35+
log.Debug("failed to get pool from network resource, not caching")
36+
break
37+
}
38+
nrCache[pool] = nr
39+
}
40+
}
41+
}
42+
43+
func (c *Core) getNrFromCache(s string) *types.NetworkResource {
44+
rc := make(chan *types.NetworkResource)
45+
c.getNr <- &getNr{s, rc}
46+
return <-rc
47+
}
48+
49+
func (c *Core) putNrInCache(nr *types.NetworkResource) {
50+
c.putNr <- nr
51+
}
52+
53+
func (c *Core) delNrInCache(s string) {
54+
c.delNr <- s
55+
}

‎host/interface.go

Lines changed: 1 addition & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package host
33
import (
44
"fmt"
55
"net"
6-
"sync"
76
"time"
87

98
log "github.com/Sirupsen/logrus"
@@ -18,84 +17,8 @@ import (
1817
var (
1918
routeProto = vxrouter.GetEnvIntWithDefault(vxrouter.EnvPrefix+"ROUTE_PROTO", "", vxrouter.DefaultRouteProto)
2019
reqAddrSleepTime = vxrouter.GetEnvDurWithDefault(vxrouter.EnvPrefix+"REQ_ADDR_SLEEP", "", vxrouter.DefaultReqAddrSleepTime)
21-
getHlc chan *getHlReq
22-
delHlc chan string
2320
)
2421

25-
func init() {
26-
getHlc = make(chan *getHlReq)
27-
delHlc = make(chan string)
28-
go hiLockLoop()
29-
}
30-
31-
// This is a special rwlock which does not cause rlock to block if a lock is waiting.
32-
// it allows new rlocks to be acquired, a full lock must wait for all of them to complete before it can run
33-
type hiLock struct {
34-
l sync.RWMutex
35-
wg sync.WaitGroup
36-
}
37-
38-
func (hl *hiLock) lock() {
39-
hl.wg.Wait()
40-
hl.l.Lock()
41-
42-
// In case wg.add called after we acquired a lock, that rlock is waiting, and new rlocks are being blocked.
43-
// Try to wait again to check. If this second wait blocks for more than 1ms, unlock and start over.
44-
c := make(chan struct{})
45-
go func() {
46-
hl.wg.Wait()
47-
close(c)
48-
}()
49-
t := time.NewTimer(1 * time.Millisecond)
50-
select {
51-
case <-c:
52-
case <-t.C:
53-
hl.l.Unlock()
54-
hl.lock()
55-
}
56-
t.Stop()
57-
}
58-
59-
func (hl *hiLock) unlock() {
60-
hl.l.Unlock()
61-
}
62-
63-
func (hl *hiLock) rlock() {
64-
hl.wg.Add(1)
65-
hl.l.RLock()
66-
}
67-
68-
func (hl *hiLock) runlock() {
69-
hl.wg.Done()
70-
hl.l.RUnlock()
71-
}
72-
73-
type getHlReq struct {
74-
s string
75-
rc chan<- *hiLock
76-
}
77-
78-
func hiLockLoop() {
79-
hlCache := make(map[string]*hiLock)
80-
for {
81-
select {
82-
case gh := <-getHlc:
83-
if _, ok := hlCache[gh.s]; !ok {
84-
hlCache[gh.s] = &hiLock{}
85-
}
86-
gh.rc <- hlCache[gh.s]
87-
case s := <-delHlc:
88-
delete(hlCache, s)
89-
}
90-
}
91-
}
92-
93-
func getHl(s string) *hiLock {
94-
rc := make(chan *hiLock)
95-
getHlc <- &getHlReq{s, rc}
96-
return <-rc
97-
}
98-
9922
// Interface holds a vxlan and a host macvlan interface used for the gateway interface on a container network
10023
type Interface struct {
10124
name string
@@ -267,7 +190,7 @@ func (hi *Interface) UnsafeDelete() error {
267190
return nil
268191
}
269192

270-
delHlc <- hi.name
193+
delHl(hi.name)
271194

272195
return hi.vxl.Delete()
273196
}

‎host/lock.go

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
package host
2+
3+
import (
4+
"sync"
5+
"time"
6+
)
7+
8+
var (
9+
getHlc chan *getHlReq
10+
delHlc chan string
11+
)
12+
13+
func init() {
14+
getHlc = make(chan *getHlReq)
15+
delHlc = make(chan string)
16+
go hiLockLoop()
17+
}
18+
19+
// This is a special rwlock which does not cause rlock to block if a lock is waiting.
20+
// it allows new rlocks to be acquired, a full lock must wait for all of them to complete before it can run
21+
type hiLock struct {
22+
l sync.RWMutex
23+
wg sync.WaitGroup
24+
}
25+
26+
func (hl *hiLock) lock() {
27+
hl.wg.Wait()
28+
hl.l.Lock()
29+
30+
// In case wg.add called after we acquired a lock, that rlock is waiting, and new rlocks are being blocked.
31+
// Try to wait again to check. If this second wait blocks for more than 1ms, unlock and start over.
32+
c := make(chan struct{})
33+
go func() {
34+
hl.wg.Wait()
35+
close(c)
36+
}()
37+
t := time.NewTimer(1 * time.Millisecond)
38+
select {
39+
case <-c:
40+
case <-t.C:
41+
hl.l.Unlock()
42+
hl.lock()
43+
}
44+
t.Stop()
45+
}
46+
47+
func (hl *hiLock) unlock() {
48+
hl.l.Unlock()
49+
}
50+
51+
func (hl *hiLock) rlock() {
52+
hl.wg.Add(1)
53+
hl.l.RLock()
54+
}
55+
56+
func (hl *hiLock) runlock() {
57+
hl.wg.Done()
58+
hl.l.RUnlock()
59+
}
60+
61+
type getHlReq struct {
62+
s string
63+
rc chan<- *hiLock
64+
}
65+
66+
func hiLockLoop() {
67+
hlCache := make(map[string]*hiLock)
68+
for {
69+
select {
70+
case gh := <-getHlc:
71+
if _, ok := hlCache[gh.s]; !ok {
72+
hlCache[gh.s] = &hiLock{}
73+
}
74+
gh.rc <- hlCache[gh.s]
75+
case s := <-delHlc:
76+
delete(hlCache, s)
77+
}
78+
}
79+
}
80+
81+
func getHl(s string) *hiLock {
82+
rc := make(chan *hiLock)
83+
getHlc <- &getHlReq{s, rc}
84+
return <-rc
85+
}
86+
87+
func delHl(s string) {
88+
delHlc <- s
89+
}

0 commit comments

Comments
 (0)