2a6d046c75
I ran across this while working with the questionably broad `rescue`
in `MonitorableResource#check_pulse`. Perhaps that rescue should be
less inclusive of things like "no such method" exceptions and crash
instead.
The test that provokes it is:
describe "#check_pulse" do
it "calls check_pulse on resource and sets pulse" do
expect(postgres_server).to receive(:check_pulse).and_return({reading: "up"})
expect { r_w_event_loop.check_pulse }.to change { r_w_event_loop.instance_variable_get(:@pulse) }.from({}).to({reading: "up"})
end
The exception is:
#<NoMethodError: undefined method '%' for nil>
67 lines
2.0 KiB
Ruby
67 lines
2.0 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
class MonitorableResource
|
|
attr_reader :deleted, :run_event_loop, :resource
|
|
attr_accessor :monitor_job_started_at, :monitor_job_finished_at
|
|
|
|
def initialize(resource)
|
|
@resource = resource
|
|
@session = nil
|
|
@pulse = {}
|
|
@pulse_check_started_at = Time.now
|
|
@pulse_thread = nil
|
|
@run_event_loop = false
|
|
@deleted = false
|
|
end
|
|
|
|
def open_resource_session
|
|
return if @session && @pulse[:reading] == "up"
|
|
|
|
@session = @resource.reload.init_health_monitor_session
|
|
rescue => ex
|
|
if ex.is_a?(Sequel::NoExistingObject)
|
|
Clog.emit("Resource is deleted.") { {resource_deleted: {ubid: @resource.ubid}} }
|
|
@session = nil
|
|
@deleted = true
|
|
end
|
|
end
|
|
|
|
def process_event_loop
|
|
return if @session.nil? || !@resource.needs_event_loop_for_pulse_check?
|
|
|
|
@pulse_thread = Thread.new do
|
|
sleep 0.01 until @run_event_loop
|
|
@session[:ssh_session].loop(0.01) { @run_event_loop }
|
|
rescue => ex
|
|
Clog.emit("SSH event loop has failed.") { {event_loop_failure: {ubid: @resource.ubid, exception: Util.exception_to_hash(ex)}} }
|
|
close_resource_session
|
|
end
|
|
end
|
|
|
|
def check_pulse
|
|
@run_event_loop = true if @resource.needs_event_loop_for_pulse_check?
|
|
|
|
@pulse_check_started_at = Time.now
|
|
begin
|
|
@pulse = @resource.check_pulse(session: @session, previous_pulse: @pulse)
|
|
Clog.emit("Got new pulse.") { {got_pulse: {ubid: @resource.ubid, pulse: @pulse}} } if (rpt = @pulse[:reading_rpt]) && (rpt < 6 || rpt % 5 == 1) || @pulse[:reading] != "up"
|
|
rescue => ex
|
|
Clog.emit("Pulse checking has failed.") { {pulse_check_failure: {ubid: @resource.ubid, exception: Util.exception_to_hash(ex)}} }
|
|
end
|
|
|
|
@run_event_loop = false if @resource.needs_event_loop_for_pulse_check?
|
|
@pulse_thread&.join
|
|
end
|
|
|
|
def close_resource_session
|
|
return if @session.nil?
|
|
|
|
@session[:ssh_session].shutdown!
|
|
begin
|
|
@session[:ssh_session].close
|
|
rescue
|
|
end
|
|
@session = nil
|
|
end
|
|
end
|