forked from Kong/lua-resty-healthcheck
-
Notifications
You must be signed in to change notification settings - Fork 8
/
healthcheck.lua
1869 lines (1555 loc) · 61.5 KB
/
healthcheck.lua
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
--------------------------------------------------------------------------
-- Healthcheck library for OpenResty.
--
-- Some notes on the usage of this library:
--
-- - Each target will have 4 counters, 1 success counter and 3 failure
-- counters ('http', 'tcp', and 'timeout'). Any failure will _only_ reset the
-- success counter, but a success will reset _all three_ failure counters.
--
-- - All targets are uniquely identified by their IP address and port number
-- combination, most functions take those as arguments.
--
-- - All keys in the SHM will be namespaced by the healthchecker name as
-- provided to the `new` function. Hence no collissions will occur on shm-keys
-- as long as the `name` is unique.
--
-- - Active healthchecks will be synchronized across workers, such that only
-- a single active healthcheck runs.
--
-- - Events will be raised in every worker, see [lua-resty-worker-events](https://github.com/Kong/lua-resty-worker-events)
-- for details.
--
-- @copyright 2017-2023 Kong Inc.
-- @author Hisham Muhammad, Thijs Schreijer
-- @license Apache 2.0
local ERR = ngx.ERR
local WARN = ngx.WARN
local DEBUG = ngx.DEBUG
local ngx_log = ngx.log
local tostring = tostring
local ipairs = ipairs
local table_insert = table.insert
local table_remove = table.remove
local string_format = string.format
local ssl = require("ngx.ssl")
local resty_timer = require "resty.timer"
local bit = require("bit")
local re_find = ngx.re.find
local ngx_now = ngx.now
local ngx_worker_id = ngx.worker.id
local ngx_worker_pid = ngx.worker.pid
local pcall = pcall
local get_phase = ngx.get_phase
local type = type
local assert = assert
local RESTY_EVENTS_VER = [[^0\.1\.\d+$]]
local RESTY_WORKER_EVENTS_VER = "0.3.3"
local new_tab
local nkeys
local is_array
local codec
local TESTING = _G.__TESTING_HEALTHCHECKER or false
do
local ok
ok, new_tab = pcall(require, "table.new")
if not ok then
new_tab = function () return {} end
end
-- OpenResty branch of LuaJIT New API
ok, nkeys = pcall(require, "table.nkeys")
if not ok then
nkeys = function (tab)
local count = 0
for _, v in pairs(tab) do
if v ~= nil then
count = count + 1
end
end
return count
end
end
ok, is_array = pcall(require, "table.isarray")
if not ok then
is_array = function(t)
for k in pairs(t) do
if type(k) ~= "number" or math.floor(k) ~= k then
return false
end
end
return true
end
end
ok, codec = pcall(require, "string.buffer")
if not ok then
codec = require("cjson.safe").new()
end
end
local worker_events
--- This function loads the worker events module received as arg. It will throw
-- error() if it is not possible to load the module.
local function load_events_module(self)
if self.events_module == "resty.worker.events" then
worker_events = require("resty.worker.events")
assert(worker_events, "could not load lua-resty-worker-events")
assert(worker_events._VERSION == RESTY_WORKER_EVENTS_VER,
"unsupported lua-resty-worker-events version")
elseif self.events_module == "resty.events" then
worker_events = require("resty.events.compat")
local version_match = ngx.re.match(worker_events._VERSION, RESTY_EVENTS_VER, "o")
assert(version_match, "unsupported lua-resty-events version")
else
error("unknown events module")
end
assert(worker_events.configured(), "please configure the '" ..
self.events_module .. "' module before using 'lua-resty-healthcheck'")
end
-- constants
local EVENT_SOURCE_PREFIX = "lua-resty-healthcheck"
local LOG_PREFIX = "[healthcheck] "
local SHM_PREFIX = "lua-resty-healthcheck:"
local EMPTY = setmetatable({},{
__newindex = function()
error("the EMPTY table is read only, check your code!", 2)
end
})
--- timer constants
-- evaluate active checks every 0.1s
local CHECK_INTERVAL = 0.1
-- use a 10% jitter to start each worker timer
local CHECK_JITTER = CHECK_INTERVAL * 0.1
-- lock valid period: the worker which acquires the lock owns it for 15 times
-- the check interval. If it does not update the shm during this period, we
-- consider that it is not able to continue checking (the worker probably was killed)
local LOCK_PERIOD = CHECK_INTERVAL * 15
-- interval between stale targets cleanup
local CLEANUP_INTERVAL = CHECK_INTERVAL * 25
-- Counters: a 32-bit shm integer can hold up to four 8-bit counters.
local CTR_SUCCESS = 0x00000001
local CTR_HTTP = 0x00000100
local CTR_TCP = 0x00010000
local CTR_TIMEOUT = 0x01000000
local MASK_FAILURE = 0xffffff00
local MASK_SUCCESS = 0x000000ff
local COUNTER_NAMES = {
[CTR_SUCCESS] = "SUCCESS",
[CTR_HTTP] = "HTTP",
[CTR_TCP] = "TCP",
[CTR_TIMEOUT] = "TIMEOUT",
}
--- The list of potential events generated.
-- The `checker.EVENT_SOURCE` field can be used to subscribe to the events, see the
-- example below. Each of the events will get a table passed containing
-- the target details `ip`, `port`, and `hostname`.
-- See [lua-resty-worker-events](https://github.com/Kong/lua-resty-worker-events).
-- @field remove Event raised when a target is removed from the checker.
-- @field healthy This event is raised when the target status changed to
-- healthy (and when a target is added as `healthy`).
-- @field unhealthy This event is raised when the target status changed to
-- unhealthy (and when a target is added as `unhealthy`).
-- @field mostly_healthy This event is raised when the target status is
-- still healthy but it started to receive "unhealthy" updates via active or
-- passive checks.
-- @field mostly_unhealthy This event is raised when the target status is
-- still unhealthy but it started to receive "healthy" updates via active or
-- passive checks.
-- @table checker.events
-- @usage -- Register for all events from `my_checker`
-- local event_callback = function(target, event, source, source_PID)
-- local t = target.ip .. ":" .. target.port .." by name '" ..
-- target.hostname .. "' ")
--
-- if event == my_checker.events.remove then
-- print(t .. "has been removed")
-- elseif event == my_checker.events.healthy then
-- print(t .. "is now healthy")
-- elseif event == my_checker.events.unhealthy then
-- print(t .. "is now unhealthy")
-- end
-- end
--
-- worker_events.register(event_callback, my_checker.EVENT_SOURCE)
local EVENTS = setmetatable({}, {
__index = function(self, key)
error(("'%s' is not a valid event name"):format(tostring(key)))
end
})
for _, event in ipairs({
"remove",
"healthy",
"unhealthy",
"mostly_healthy",
"mostly_unhealthy",
"clear",
}) do
EVENTS[event] = event
end
local INTERNAL_STATES = {}
for i, key in ipairs({
"healthy",
"unhealthy",
"mostly_healthy",
"mostly_unhealthy",
}) do
INTERNAL_STATES[i] = key
INTERNAL_STATES[key] = i
end
-- Some color for demo purposes
local use_color = false
local id = function(x) return x end
local worker_color = use_color and function(str) return ("\027["..tostring(31 + ngx_worker_pid() % 5).."m"..str.."\027[0m") end or id
-- Debug function
local function dump(...) print(require("pl.pretty").write({...})) end -- luacheck: ignore 211
local _M = {}
-- checker objects (weak) table
local hcs = setmetatable({}, {
__mode = "v",
})
local active_check_timer
local last_cleanup_check
-- serialize a table to a string
local serialize = codec.encode
-- deserialize a string to a table
local deserialize = codec.decode
local function key_for(key_prefix, ip, port, hostname)
return string_format("%s:%s:%s%s", key_prefix, ip, port, hostname and ":" .. hostname or "")
end
-- resty.lock timeout when yieldable
local LOCK_TIMEOUT = 5
local run_locked
do
-- resty_lock is restricted to this scope in order to keep sensitive
-- lock-handling code separate separate from all other business logic
--
-- If you need to use resty_lock in a way that is not covered by the
-- `run_locked` helper function defined below, it's strongly-advised to
-- define it fully within this scope unless you have a very good reason
--
-- (see https://github.com/Kong/lua-resty-healthcheck/pull/112)
local resty_lock = require "resty.lock"
local yieldable = {
rewrite = true,
access = true,
content = true,
timer = true,
}
local function run_in_timer(premature, self, key, fn, ...)
if premature then
return
end
local ok, err = run_locked(self, key, fn, ...)
if not ok then
self:log(ERR, "locked function for key '", key, "' failed in timer: ", err)
end
end
local function schedule(self, key, fn, ...)
local ok, err = ngx.timer.at(0, run_in_timer, self, key, fn, ...)
if not ok then
return nil, "failed scheduling locked function for key '" .. key ..
"', " .. err
end
return "scheduled"
end
-- resty.lock consumes these options immediately, so this table can be reused
local opts = {
exptime = 10, -- timeout after which lock is released anyway
timeout = LOCK_TIMEOUT, -- max wait time to acquire lock
}
---
-- Acquire a lock and run a function
--
-- The function call itself is wrapped with `pcall` to protect against
-- exception.
--
-- This function exhibits some special behavior when called during a
-- non-yieldable phase such as `init_worker` or `log`:
--
-- 1. The lock timeout is set to 0 to ensure that `resty.lock` does not
-- attempt to sleep/yield
-- 2. If acquiring the lock fails due to a timeout, `run_locked`
-- (this function) is re-scheduled to run in a timer. In this case,
-- the function returns `"scheduled"`
--
-- @param self The checker object
-- @param key the key/identifier to acquire a lock for
-- @param fn The function to execute
-- @param ... arguments that will be passed to fn
-- @return The results of the function; or nil and an error message
-- in case it fails locking.
function run_locked(self, key, fn, ...)
-- we're extra extra extra defensive in this code path
local typ = type(key)
-- XXX is a number key ever expected?
assert(typ == "string" or typ == "number",
"unexpected lock key type: " .. typ)
key = tostring(key)
-- first aqcuire a lock or conditionally re-schedule ourselves in a timer
local lock
do
local yield = yieldable[get_phase()]
if yield then
opts.timeout = LOCK_TIMEOUT
else
-- if yielding is not possible in the current phase, use a zero timeout
-- so that resty.lock will return `nil, "timeout"` immediately instead of
-- calling ngx.sleep()
opts.timeout = 0
end
local err
lock, err = resty_lock:new(self.shm_name, opts)
if not lock then
return nil, "failed creating lock for '" .. key .. "', " .. err
end
local elapsed
elapsed, err = lock:lock(key)
if not elapsed and err == "timeout" and not yield then
-- yielding is not possible in the current phase, so retry in a timer
return schedule(self, key, fn, ...)
elseif not elapsed then
return nil, "failed acquiring lock for '" .. key .. "', " .. err
end
end
local pok, perr, res = pcall(fn, ...)
local ok, err = lock:unlock()
if not ok then
self:log(ERR, "failed unlocking '", key, "', ", err)
end
if not pok then
return nil, "locked function threw an exception: " .. tostring(perr)
end
return perr, res
end
end
local deepcopy
do
local function _deepcopy(orig, copied)
-- prevent infinite loop when a field refers its parent
copied[orig] = true
-- If the array-like table contains nil in the middle,
-- the len might be smaller than the expected.
-- But it doesn't affect the correctness.
local len = #orig
local copy = table.new(len, table.nkeys(orig) - len)
for orig_key, orig_value in pairs(orig) do
if type(orig_value) == "table" and not copied[orig_value] then
copy[orig_key] = _deepcopy(orig_value, copied)
else
copy[orig_key] = orig_value
end
end
local mt = getmetatable(orig)
if mt ~= nil then
setmetatable(copy, mt)
end
return copy
end
local copied_recorder = {}
function deepcopy(orig)
local orig_type = type(orig)
if orig_type ~= 'table' then
return orig
end
local res = _deepcopy(orig, copied_recorder)
table.clear(copied_recorder)
return res
end
end
local checker = {}
------------------------------------------------------------------------------
-- Node management.
-- @section node-management
------------------------------------------------------------------------------
-- @return the target list from the shm, an empty table if not found, or
-- `nil + error` upon a failure
local function fetch_target_list(self)
local target_list, err = self.shm:get(self.TARGET_LIST)
if err then
return nil, "failed to fetch target_list from shm: " .. err
end
return target_list and deserialize(target_list) or {}
end
local function with_target_list(self, fn)
local targets, err = fetch_target_list(self)
if not targets then
return nil, err
end
-- this is only ever called in the context of `run_locked`,
-- so no pcall needed
return fn(targets)
end
--- Run the given function holding a lock on the target list.
-- @param self The checker object
-- @param fn The function to execute
-- @return The results of the function; or nil and an error message
-- in case it fails locking.
local function locking_target_list(self, fn)
local ok, err = run_locked(self, self.TARGET_LIST_LOCK, with_target_list, self, fn)
if ok == "scheduled" then
self:log(DEBUG, "target_list function re-scheduled in timer")
end
return ok, err
end
--- Get a target
local function get_target(self, ip, port, hostname)
hostname = hostname or ip
return ((self.targets[ip] or EMPTY)[port] or EMPTY)[hostname]
end
--- Add a target to the healthchecker.
-- When the ip + port + hostname combination already exists, it will simply
-- return success (without updating `is_healthy` status).
-- @param ip IP address of the target to check.
-- @param port the port to check against.
-- @param hostname (optional) hostname to set as the host header in the HTTP
-- probe request
-- @param is_healthy (optional) a boolean value indicating the initial state,
-- default is `true`.
-- @param hostheader (optional) a value to use for the Host header on
-- active healthchecks.
-- @param tbl_meta (optional) a lua table with custom info of business stuff
-- @return `true` on success, or `nil + error` on failure.
function checker:add_target(ip, port, hostname, is_healthy, hostheader, tbl_meta)
ip = tostring(assert(ip, "no ip address provided"))
port = assert(tonumber(port), "no port number provided")
hostname = hostname or ip
if is_healthy == nil then
is_healthy = true
end
local internal_health = is_healthy and "healthy" or "unhealthy"
local ok, err = locking_target_list(self, function(target_list)
local found = false
-- check whether we already have this target
for _, target in ipairs(target_list) do
if target.ip == ip and target.port == port and target.hostname == (hostname) then
if target.purge_time == nil then
self:log(DEBUG, "adding an existing target: ", hostname or "", " ", ip,
":", port, " (ignoring)")
return false
end
target.purge_time = nil
found = true
internal_health = self:get_target_status(ip, port, hostname) and
"healthy" or "unhealthy"
break
end
end
-- we first add the internal health, and only then the updated list.
-- this prevents a state where a target is in the list, but does not
-- have a key in the shm.
local ok, err = self.shm:set(key_for(self.TARGET_STATE, ip, port, hostname),
INTERNAL_STATES[internal_health])
if not ok then
self:log(ERR, "failed to set initial health status in shm: ", err)
end
-- target does not exist, go add it
if not found then
target_list[#target_list + 1] = {
ip = ip,
port = port,
hostname = hostname,
hostheader = hostheader,
meta = tbl_meta,
}
end
target_list = serialize(target_list)
ok, err = self.shm:set(self.TARGET_LIST, target_list)
if not ok then
return nil, "failed to store target_list in shm: " .. err
end
-- raise event for our newly added target
if not found then
self:raise_event(self.events[internal_health], ip, port, hostname)
end
return true
end)
if ok == false then
-- the target already existed, no event, but still success
return true
end
return ok, err
end
-- Remove health status entries from an individual target from shm
-- @param self The checker object
-- @param ip IP address of the target being checked.
-- @param port the port being checked against.
-- @param hostname hostname of the target being checked.
local function clear_target_data_from_shm(self, ip, port, hostname)
local ok, err = self.shm:set(key_for(self.TARGET_STATE, ip, port, hostname), nil)
if not ok then
self:log(ERR, "failed to remove health status from shm: ", err)
end
ok, err = self.shm:set(key_for(self.TARGET_COUNTER, ip, port, hostname), nil)
if not ok then
self:log(ERR, "failed to clear health counter from shm: ", err)
end
end
--- Remove a target from the healthchecker.
-- The target not existing is not considered an error.
-- @param ip IP address of the target being checked.
-- @param port the port being checked against.
-- @param hostname (optional) hostname of the target being checked.
-- @return `true` on success, or `nil + error` on failure.
function checker:remove_target(ip, port, hostname)
ip = tostring(assert(ip, "no ip address provided"))
port = assert(tonumber(port), "no port number provided")
return locking_target_list(self, function(target_list)
-- find the target
local target_found
for i, target in ipairs(target_list) do
if target.ip == ip and target.port == port and target.hostname == hostname then
target_found = target
table_remove(target_list, i)
break
end
end
if not target_found then
return true
end
-- go update the shm
target_list = serialize(target_list)
-- we first write the updated list, and only then remove the health
-- status; this prevents race conditions when a healthchecker gets the
-- initial state from the shm
local ok, err = self.shm:set(self.TARGET_LIST, target_list)
if not ok then
return nil, "failed to store target_list in shm: " .. err
end
clear_target_data_from_shm(self, ip, port, hostname)
-- raise event for our removed target
self:raise_event(self.events.remove, ip, port, hostname)
return true
end)
end
--- Clear all healthcheck data.
-- @return `true` on success, or `nil + error` on failure.
function checker:clear()
return locking_target_list(self, function(target_list)
local old_target_list = target_list
-- go update the shm
target_list = serialize({})
local ok, err = self.shm:set(self.TARGET_LIST, target_list)
if not ok then
return nil, "failed to store target_list in shm: " .. err
end
-- remove all individual statuses
for _, target in ipairs(old_target_list) do
local ip, port, hostname = target.ip, target.port, target.hostname
clear_target_data_from_shm(self, ip, port, hostname)
end
self.targets = {}
-- raise event for our removed target
self:raise_event(self.events.clear)
return true
end)
end
--- Clear all healthcheck data after a period of time.
-- Useful for keeping target status between configuration reloads.
-- @param delay delay in seconds before purging target state.
-- @return `true` on success, or `nil + error` on failure.
function checker:delayed_clear(delay)
assert(tonumber(delay), "no delay provided")
return locking_target_list(self, function(target_list)
local purge_time = ngx_now() + delay
-- add purge time to all targets
for _, target in ipairs(target_list) do
target.purge_time = purge_time
end
target_list = serialize(target_list)
local ok, err = self.shm:set(self.TARGET_LIST, target_list)
if not ok then
return nil, "failed to store target_list in shm: " .. err
end
return true
end)
end
--- Get the current status of the target.
-- @param ip IP address of the target being checked.
-- @param port the port being checked against.
-- @param hostname the hostname of the target being checked.
-- @return `true` if healthy, `false` if unhealthy, or `nil + error` on failure.
function checker:get_target_status(ip, port, hostname)
local target = get_target(self, ip, port, hostname)
if not target then
return nil, "target not found"
end
return target.internal_health == "healthy"
or target.internal_health == "mostly_healthy"
end
------------------------------------------------------------------------------
-- Health management.
-- Functions that allow reporting of failures/successes for passive checks.
-- @section health-management
------------------------------------------------------------------------------
-- Run the given function holding a lock on the target.
-- @param self The checker object
-- @param ip Target IP
-- @param port Target port
-- @param hostname Target hostname
-- @param fn The function to execute
-- @return The results of the function; or true in case it fails locking and
-- will retry asynchronously; or nil+err in case it fails to retry.
local function locking_target(self, ip, port, hostname, fn)
local key = key_for(self.TARGET_LOCK, ip, port, hostname)
local ok, err = run_locked(self, key, fn)
if ok == "scheduled" then
self:log(DEBUG, "target function for ", key, " was re-scheduled")
end
return ok, err
end
-- Extract the value of the counter at `idx` from multi-counter `multictr`.
-- @param multictr A 32-bit multi-counter holding 4 values.
-- @param idx The shift index specifying which counter to get.
-- @return The 8-bit value extracted from the 32-bit multi-counter.
local function ctr_get(multictr, idx)
return bit.band(multictr / idx, 0xff)
end
-- Increment the healthy or unhealthy counter. If the threshold of occurrences
-- is reached, it changes the status of the target in the shm and posts an
-- event.
-- @param self The checker object
-- @param health_report "healthy" for the success counter that drives a target
-- towards the healthy state; "unhealthy" for the failure counter.
-- @param ip Target IP
-- @param port Target port
-- @param hostname Target hostname
-- @param limit the limit after which target status is changed
-- @param ctr_type the counter to increment, see CTR_xxx constants
-- @return True if succeeded, or nil and an error message.
local function incr_counter(self, health_report, ip, port, hostname, limit, ctr_type)
-- fail fast on counters that are disabled by configuration
if limit == 0 then
return true
end
port = tonumber(port)
local target = get_target(self, ip, port, hostname)
if not target then
-- sync issue: warn, but return success
self:log(WARN, "trying to increment a target that is not in the list: ",
hostname and "(" .. hostname .. ") " or "", ip, ":", port)
return true
end
local current_health = target.internal_health
if health_report == current_health then
-- No need to count successes when internal health is fully "healthy"
-- or failures when internal health is fully "unhealthy"
return true
end
return locking_target(self, ip, port, hostname, function()
local counter_key = key_for(self.TARGET_COUNTER, ip, port, hostname)
local multictr, err = self.shm:incr(counter_key, ctr_type, 0)
if err then
return nil, err
end
local ctr = ctr_get(multictr, ctr_type)
self:log(WARN, health_report, " ", COUNTER_NAMES[ctr_type],
" increment (", ctr, "/", limit, ") for '", hostname or "",
"(", ip, ":", port, ")'")
local new_multictr
if ctr_type == CTR_SUCCESS then
new_multictr = bit.band(multictr, MASK_SUCCESS)
else
new_multictr = bit.band(multictr, MASK_FAILURE)
end
if new_multictr ~= multictr then
self.shm:set(counter_key, new_multictr)
end
local new_health
if ctr >= limit then
new_health = health_report
elseif current_health == "healthy" and bit.band(new_multictr, MASK_FAILURE) > 0 then
new_health = "mostly_healthy"
elseif current_health == "unhealthy" and bit.band(new_multictr, MASK_SUCCESS) > 0 then
new_health = "mostly_unhealthy"
end
if new_health and new_health ~= current_health then
local state_key = key_for(self.TARGET_STATE, ip, port, hostname)
self.shm:set(state_key, INTERNAL_STATES[new_health])
self:raise_event(self.events[new_health], ip, port, hostname)
end
return true
end)
end
--- Report a health failure.
-- Reports a health failure which will count against the number of occurrences
-- required to make a target "fall". The type of healthchecker,
-- "tcp" or "http" (see `new`) determines against which counter the occurence goes.
-- If `unhealthy.tcp_failures` (for TCP failures) or `unhealthy.http_failures`
-- is set to zero in the configuration, this function is a no-op
-- and returns `true`.
-- @param ip IP address of the target being checked.
-- @param port the port being checked against.
-- @param hostname (optional) hostname of the target being checked.
-- @param check (optional) the type of check, either "passive" or "active", default "passive".
-- @return `true` on success, or `nil + error` on failure.
function checker:report_failure(ip, port, hostname, check)
local checks = self.checks[check or "passive"]
local limit, ctr_type
if self.checks[check or "passive"].type == "tcp" then
limit = checks.unhealthy.tcp_failures
ctr_type = CTR_TCP
else
limit = checks.unhealthy.http_failures
ctr_type = CTR_HTTP
end
return incr_counter(self, "unhealthy", ip, port, hostname, limit, ctr_type)
end
--- Report a health success.
-- Reports a health success which will count against the number of occurrences
-- required to make a target "rise".
-- If `healthy.successes` is set to zero in the configuration,
-- this function is a no-op and returns `true`.
-- @param ip IP address of the target being checked.
-- @param port the port being checked against.
-- @param hostname (optional) hostname of the target being checked.
-- @param check (optional) the type of check, either "passive" or "active", default "passive".
-- @return `true` on success, or `nil + error` on failure.
function checker:report_success(ip, port, hostname, check)
local limit = self.checks[check or "passive"].healthy.successes
return incr_counter(self, "healthy", ip, port, hostname, limit, CTR_SUCCESS)
end
--- Report a http response code.
-- How the code is interpreted is based on the configuration for healthy and
-- unhealthy statuses. If it is in neither strategy, it will be ignored.
-- If `healthy.successes` (for healthy HTTP status codes)
-- or `unhealthy.http_failures` (fur unhealthy HTTP status codes)
-- is set to zero in the configuration, this function is a no-op
-- and returns `true`.
-- @param ip IP address of the target being checked.
-- @param port the port being checked against.
-- @param hostname (optional) hostname of the target being checked.
-- @param http_status the http statuscode, or nil to report an invalid http response.
-- @param check (optional) the type of check, either "passive" or "active", default "passive".
-- @return `true` on success, `nil` if the status was ignored (not in active or
-- passive health check lists) or `nil + error` on failure.
function checker:report_http_status(ip, port, hostname, http_status, check)
http_status = tonumber(http_status) or 0
local checks = self.checks[check or "passive"]
local status_type, limit, ctr
if checks.healthy.http_statuses[http_status] then
status_type = "healthy"
limit = checks.healthy.successes
ctr = CTR_SUCCESS
elseif checks.unhealthy.http_statuses[http_status]
or http_status == 0 then
status_type = "unhealthy"
limit = checks.unhealthy.http_failures
ctr = CTR_HTTP
else
return
end
return incr_counter(self, status_type, ip, port, hostname, limit, ctr)
end
--- Report a failure on TCP level.
-- If `unhealthy.tcp_failures` is set to zero in the configuration,
-- this function is a no-op and returns `true`.
-- @param ip IP address of the target being checked.
-- @param port the port being checked against.
-- @param hostname hostname of the target being checked.
-- @param operation The socket operation that failed:
-- "connect", "send" or "receive".
-- TODO check what kind of information we get from the OpenResty layer
-- in order to tell these error conditions apart
-- https://github.com/openresty/lua-resty-core/blob/master/lib/ngx/balancer.md#get_last_failure
-- @param check (optional) the type of check, either "passive" or "active", default "passive".
-- @return `true` on success, or `nil + error` on failure.
function checker:report_tcp_failure(ip, port, hostname, operation, check)
local limit = self.checks[check or "passive"].unhealthy.tcp_failures
-- TODO what do we do with the `operation` information
return incr_counter(self, "unhealthy", ip, port, hostname, limit, CTR_TCP)
end
--- Report a timeout failure.
-- If `unhealthy.timeouts` is set to zero in the configuration,
-- this function is a no-op and returns `true`.
-- @param ip IP address of the target being checked.
-- @param port the port being checked against.
-- @param hostname (optional) hostname of the target being checked.
-- @param check (optional) the type of check, either "passive" or "active", default "passive".
-- @return `true` on success, or `nil + error` on failure.
function checker:report_timeout(ip, port, hostname, check)
local limit = self.checks[check or "passive"].unhealthy.timeouts
return incr_counter(self, "unhealthy", ip, port, hostname, limit, CTR_TIMEOUT)
end
--- Sets the current status of all targets with the given hostname and port.
-- @param hostname hostname being checked.
-- @param port the port being checked against
-- @param is_healthy boolean: `true` for healthy, `false` for unhealthy
-- @return `true` on success, or `nil + error` on failure.
function checker:set_all_target_statuses_for_hostname(hostname, port, is_healthy)
assert(type(hostname) == "string", "no hostname provided")
port = assert(tonumber(port), "no port number provided")
assert(type(is_healthy) == "boolean")
local all_ok = true
local errs = {}
for _, target in ipairs(self.targets) do
if target.port == port and target.hostname == hostname then
local ok, err = self:set_target_status(target.ip, port, hostname, is_healthy)
if not ok then
all_ok = nil
table.insert(errs, err)
end
end
end
return all_ok, #errs > 0 and table.concat(errs, "; ") or nil
end
--- Sets the current status of the target.
-- This will immediately set the status and clear its counters.
-- @param ip IP address of the target being checked
-- @param port the port being checked against
-- @param hostname (optional) hostname of the target being checked.
-- @param is_healthy boolean: `true` for healthy, `false` for unhealthy
-- @return `true` on success, or `nil + error` on failure
function checker:set_target_status(ip, port, hostname, is_healthy)
ip = tostring(assert(ip, "no ip address provided"))
port = assert(tonumber(port), "no port number provided")
assert(type(is_healthy) == "boolean")
local health_report = is_healthy and "healthy" or "unhealthy"
local target = get_target(self, ip, port, hostname)
if not target then
-- sync issue: warn, but return success
self:log(WARN, "trying to set status for a target that is not in the list: ", ip, ":", port)
return true
end
local counter_key = key_for(self.TARGET_COUNTER, ip, port, hostname)
local state_key = key_for(self.TARGET_STATE, ip, port, hostname)
local ok, err = locking_target(self, ip, port, hostname, function()
local _, err = self.shm:set(counter_key, 0)
if err then
return nil, err
end