Files
ubicloud/spec/model/minio/minio_server_spec.rb
shikharbhardwaj 523d6ca958 Use public IPs for minio cluster communication
Based on some recent networking performance benchmarks, we've discovered
that the private netwroking stack seems to offer upto 10x lower
bandwidth as compared to the public one. While this needs to be
investigated further, meanwhile we switch the MinIO cluster to use
public IPs for peer communication to avoid this performance penalty.
2025-02-21 17:58:29 +05:30

149 lines
5.9 KiB
Ruby

# frozen_string_literal: true
require_relative "../spec_helper"
RSpec.describe MinioServer do
subject(:ms) {
vm = create_vm(ephemeral_net6: "fdfa:b5aa:14a3:4a3d::/64")
mc = MinioCluster.create_with_id(
location: "hetzner-fsn1",
name: "minio-cluster-name",
admin_user: "minio-admin",
admin_password: "dummy-password",
root_cert_1: "dummy-root-cert-1",
root_cert_2: "dummy-root-cert-2",
project_id: vm.project_id
)
mp = MinioPool.create_with_id(
cluster_id: mc.id,
start_index: 0,
server_count: 1,
drive_count: 1,
storage_size_gib: 100,
vm_size: "standard-2"
)
described_class.create_with_id(
minio_pool_id: mp.id,
vm_id: vm.id,
index: 0,
cert: "cert"
)
}
it "returns hostname properly" do
expect(ms.hostname).to eq("minio-cluster-name0.minio.ubicloud.com")
end
it "returns private ipv4 address properly" do
nic = instance_double(Nic, private_ipv4: NetAddr::IPv4Net.parse("192.168.0.0/32"))
expect(ms.vm).to receive(:nics).and_return([nic]).at_least(:once)
expect(ms.private_ipv4_address).to eq("192.168.0.0")
end
it "returns public ipv6 address properly" do
expect(ms.public_ipv6_address).to eq("fdfa:b5aa:14a3:4a3d::2")
end
it "returns minio cluster properly" do
expect(ms.cluster.name).to eq("minio-cluster-name")
end
describe "#minio_volumes" do
it "returns minio volumes properly for a single drive single server cluster" do
expect(ms.minio_volumes).to eq("/minio/dat1")
end
it "returns minio volumes properly for a multi drive single server cluster" do
ms.pool.update(drive_count: 4)
expect(ms.minio_volumes).to eq("/minio/dat{1...4}")
end
it "returns minio volumes properly for a multi drive multi server cluster" do
ms.pool.update(drive_count: 4, server_count: 2)
expect(ms.minio_volumes).to eq("https://minio-cluster-name{0...1}.minio.ubicloud.com:9000/minio/dat{1...2}")
end
end
it "initiates a new health monitor session" do
forward = instance_double(Net::SSH::Service::Forward)
expect(forward).to receive(:local)
session = instance_double(Net::SSH::Connection::Session)
expect(session).to receive(:forward).and_return(forward)
sshable = instance_double(Sshable)
expect(sshable).to receive(:start_fresh_session).and_return(session)
expect(ms.vm).to receive(:sshable).and_return(sshable)
expect(UNIXServer).to receive(:new)
expect(Minio::Client).to receive(:new)
expect(ms).to receive(:private_ipv4_address).and_return("192.168.1.1")
ms.init_health_monitor_session
end
it "checks pulse" do
session = {
ssh_session: instance_double(Net::SSH::Connection::Session),
minio_client: Minio::Client.new(endpoint: "https://1.2.3.4:9000", access_key: "dummy-key", secret_key: "dummy-secret", ssl_ca_file_data: "data")
}
expect(ms.vm).to receive(:ephemeral_net4).and_return("1.2.3.4").at_least(:once)
expect(ms).not_to receive(:incr_checkup)
stub_request(:get, "https://1.2.3.4:9000/minio/admin/v3/info").to_return(status: 200, body: JSON.generate({servers: [{state: "online", endpoint: "1.2.3.4:9000", drives: [{state: "ok"}]}]}))
ms.check_pulse(session: session, previous_pulse: {reading: "down", reading_rpt: 5, reading_chg: Time.now - 30})
stub_request(:get, "https://1.2.3.4:9000/minio/admin/v3/info").to_return(status: 200, body: JSON.generate({servers: [{state: "online", endpoint: "1.2.3.4:9000", drives: [{state: "faulty"}]}]}))
ms.check_pulse(session: session, previous_pulse: {})
stub_request(:get, "https://1.2.3.4:9000/minio/admin/v3/info").to_return(status: 200, body: JSON.generate({servers: [{state: "offline", endpoint: "1.2.3.4:9000"}]}))
ms.check_pulse(session: session, previous_pulse: {})
end
it "increments checkup semaphore if pulse is down for a while" do
session = {
ssh_session: instance_double(Net::SSH::Connection::Session),
minio_client: instance_double(Minio::Client)
}
expect(session[:minio_client]).to receive(:admin_info).and_raise(RuntimeError)
expect(ms).to receive(:incr_checkup)
ms.check_pulse(session: session, previous_pulse: {reading: "down", reading_rpt: 5, reading_chg: Time.now - 30})
end
it "returns endpoint properly" do
expect(ms.vm).to receive(:ephemeral_net4).and_return("1.1.1.1")
expect(ms.endpoint).to eq("1.1.1.1:9000")
expect(ms.cluster).to receive(:dns_zone).and_return("something")
expect(ms.endpoint).to eq("minio-cluster-name0.minio.ubicloud.com:9000")
end
it "needs event loop for pulse check" do
expect(ms).to be_needs_event_loop_for_pulse_check
end
it "generates /etc/hosts entries properly when there are multiple servers" do
servers = [instance_double(described_class, id: ms.id, public_ipv6_address: "fdfa:b5aa:14a3:4a3d::/64", hostname: "minio-cluster-name1.minio.ubicloud.com"),
instance_double(described_class, id: 2, public_ipv6_address: "fdfb:b5aa:14a3:4a3f::/64", hostname: "minio-cluster-name2.minio.ubicloud.com")]
mc = instance_double(MinioCluster, name: "minio-cluster-name", servers: servers)
expect(ms).to receive(:cluster).and_return(mc).at_least(:once)
expect(ms.generate_etc_hosts_entry).to eq("::1 minio-cluster-name0.minio.ubicloud.com\nfdfb:b5aa:14a3:4a3f::/64 minio-cluster-name2.minio.ubicloud.com")
end
describe "#url" do
before do
minio_project = Project.create_with_id(name: "default")
allow(Config).to receive(:minio_service_project_id).and_return(minio_project.id)
end
it "returns url properly" do
DnsZone.create_with_id(project_id: Config.minio_service_project_id, name: Config.minio_host_name)
expect(ms.server_url).to eq("https://minio-cluster-name.minio.ubicloud.com:9000")
end
it "returns ip address when dns zone is not found" do
expect(ms.vm).to receive(:ephemeral_net4).and_return("10.10.10.10")
expect(ms.server_url).to eq("https://10.10.10.10:9000")
end
end
end