1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
package proxy
import (
"context"
"errors"
"net/http"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/armanlife/firefox-vpn-client/core/internal/logx"
)
const (
// serverFailoverStrikes is how many failures in a row of one server
// move the upstream to another server of the location. A rebuild
// counts once, and rebuilds are spaced (redials 0, 10 s, 30 s; new
// passes 10 s apart and more), so a dead server is left within about
// 10 to 40 seconds; one or two failures in a row are what a short
// network blip (a Wi-Fi roam, a mobile handover) does, and must not
// move the server. It matches the failure tracker's three distinct
// failing targets.
serverFailoverStrikes = 3
// serverAvoidTime is how long a server that reached
// serverFailoverStrikes is skipped. It outlasts several rounds of the
// longest rebuild spacing (2 minutes), so the location's other servers
// are tried before it, and is short enough that a server back from a
// restart or a short outage returns to the choice within a quarter of
// an hour.
serverAvoidTime = 15 * time.Minute
)
// ServerFailover moves the upstream to another server of the chosen
// location when one server keeps failing: after serverFailoverStrikes
// failures in a row to establish or keep a session on it (serverFault
// decides which failures are the server's), the server is avoided for
// serverAvoidTime and another server of the same city, else of the
// location's next city, is used and saved as the persisted choice. When
// every other server of the location is avoided, the upstream stays where
// it is and the usual retries go on; the location never changes.
//
// One per daemon: it outlives controllers and engines, so the wakes and
// the controllers of one location share the strikes and the avoided
// servers. A nil ServerFailover does nothing (a server set by hand).
type ServerFailover struct {
// selectionFile is where a new choice is saved (selection.json); empty
// saves nothing.
selectionFile string
// now is the clock; tests replace it.
now func() time.Time
// pending is set while a server has strikes, so Worked, called for
// every tunnel, takes no lock otherwise.
pending atomic.Bool
mu sync.Mutex
// failing is the canonical address of the server with strikes.
failing string
strikes int
// avoided maps canonical addresses to the end of their avoidance.
avoided map[string]time.Time
}
// NewServerFailover returns a failover that saves new choices in
// selectionFile.
func NewServerFailover(selectionFile string) *ServerFailover {
return &ServerFailover{selectionFile: selectionFile, now: time.Now}
}
// Avoided reports whether the server at addr failed recently and is
// skipped by selection.
func (f *ServerFailover) Avoided(addr string) bool {
if f == nil {
return false
}
key, err := canonicalProxyAddr(addr)
if err != nil {
return false
}
f.mu.Lock()
defer f.mu.Unlock()
return f.avoidedLocked(key, f.now())
}
func (f *ServerFailover) avoidedLocked(key string, now time.Time) bool {
until, ok := f.avoided[key]
if ok && !now.Before(until) {
delete(f.avoided, key)
return false
}
return ok
}
// Worked forgets the strikes of the server at addr: a tunnel opened
// through it.
func (f *ServerFailover) Worked(addr string) {
if f == nil || !f.pending.Load() {
return
}
key, err := canonicalProxyAddr(addr)
if err != nil {
return
}
f.mu.Lock()
if f.failing == key {
f.failing, f.strikes = "", 0
f.pending.Store(false)
}
f.mu.Unlock()
}
// Failed counts err against server, one of location's servers, when the
// server is to blame (serverFault). At its serverFailoverStrikes-th
// failure in a row the server is avoided for serverAvoidTime and Failed
// returns the server to use instead (nextServer), saved as the persisted
// choice. ok is false otherwise, also when every other server of the
// location is avoided: the caller then stays on server.
func (f *ServerFailover) Failed(location []Candidate, server Candidate, err error) (next Candidate, ok bool) {
if f == nil || !serverFault(err) {
return Candidate{}, false
}
key, keyErr := canonicalProxyAddr(server.Addr)
if keyErr != nil {
return Candidate{}, false
}
f.mu.Lock()
if f.failing != key {
f.failing, f.strikes = key, 0
}
f.strikes++
f.pending.Store(true)
if f.strikes < serverFailoverStrikes {
strikes := f.strikes
f.mu.Unlock()
logx.Debug("upstream server failed proxy=%s failures_in_a_row=%d err=%s", server.Addr, strikes, logx.Err(err))
return Candidate{}, false
}
now := f.now()
f.failing, f.strikes = "", 0
f.pending.Store(false)
if f.avoided == nil {
f.avoided = make(map[string]time.Time)
}
f.avoided[key] = now.Add(serverAvoidTime)
next, ok = nextServer(location, server, func(addr string) bool {
k, err := canonicalProxyAddr(addr)
return err == nil && f.avoidedLocked(k, now)
})
f.mu.Unlock()
if !ok {
logx.Warn("upstream server failed %d times in a row, and every other server of the location failed recently; staying on it proxy=%s err=%s",
serverFailoverStrikes, server.Addr, logx.Err(err))
return Candidate{}, false
}
logx.Info("upstream server failed %d times in a row; switching to another server of the location from=%s to=%s city=%q avoid_for=%s err=%s",
serverFailoverStrikes, server.Addr, next.Addr, next.CityNameOrUnknown(), serverAvoidTime, logx.Err(err))
if f.selectionFile != "" {
if err := SaveSelection(f.selectionFile, next); err != nil {
logx.Warn("saving proxy selection failed path=%s err=%s", f.selectionFile, logx.Err(err))
}
}
return next, true
}
// nextServer picks the server to use instead of current: another server
// of current's city, else one of the location's first other city that has
// one, at random within the city (as pickInFirstCity does), skipping
// avoided servers.
func nextServer(location []Candidate, current Candidate, avoided func(addr string) bool) (Candidate, bool) {
currentKey, _ := canonicalProxyAddr(current.Addr)
usable := func(c Candidate) bool {
key, err := canonicalProxyAddr(c.Addr)
return err == nil && key != currentKey && !avoided(c.Addr)
}
if next, ok := pickInFirstCity(location, func(c Candidate) bool { return sameCity(c, current) && usable(c) }); ok {
return next, true
}
return pickInFirstCity(location, usable)
}
// sameCity reports whether a and b are servers of one city.
func sameCity(a, b Candidate) bool {
return strings.EqualFold(a.CountryCode, b.CountryCode) && a.CityCode == b.CityCode && a.CityName == b.CityName
}
// serverFault reports whether err, a failure to establish or keep a
// session on a Mozilla proxy server, can be blamed on that server. It
// cannot when this device has no network, when the account or its pass is
// the problem (Guardian's answers; a CONNECT 401, 403, 407, or 429), when
// the attempt was cancelled or did not happen (a rebuild cooling down, no
// session to use), or for a port Mozilla's proxy refuses (such a 502 never
// gets here: it neither rebuilds nor marks a session unhealthy).
func serverFault(err error) bool {
if err == nil ||
errors.Is(err, context.Canceled) ||
errors.Is(err, errRebuildCoolingDown) ||
errors.Is(err, ErrNoUsableSession) ||
errors.Is(err, errSessionSaturated) {
return false
}
switch FailureCode(err) {
case FailureNoNetwork, FailureSignIn, FailureQuota, FailureNotEntitled, FailureRegion,
FailureNoAccount, FailureServer, FailureLocationRefused:
return false
}
var connectErr *proxyConnectHTTPError
if errors.As(err, &connectErr) {
switch connectErr.statusCode {
case http.StatusUnauthorized, http.StatusForbidden, http.StatusProxyAuthRequired, http.StatusTooManyRequests:
return false
}
}
return true
}