ubicloud/rhizome/host/spec/boot_image_spec.rb
Enes Cakir 5e34c5a809 Use htcat to download image from presigned URL
We previously used curl to download images from presigned URLs. This
worked well for our internal MinIO cluster, but with R2 presigned URLs,
curl performs poorly since it downloads with a single connection.

First, I benchmarked aria2 [^1], which supports multiple connections.
However, Daniel recommended using a memory-safe tool instead and
suggested htcat [^2], which his team wrote about 10 years ago. After a
few small improvements, htcat now works well for our use case and
delivers performance comparable to aria2.

If the URL is from R2, we now use htcat to download it with multiple
connections.

Since htcat pipes to stdout, we calculate the sha256 hash while
downloading similar to how we did with curl.

I will download the htcat binary on existing hosts.

[^1]: https://aria2.github.io/
[^2]: https://github.com/htcat/htcat
2025-08-27 12:43:23 +03:00

129 lines
5.7 KiB
Ruby

# frozen_string_literal: true
require_relative "../lib/boot_image"
require "openssl"
require "base64"
RSpec.describe BootImage do
subject(:bi) { described_class.new("ubuntu-jammy", "20240110") }
describe "#download" do
it "can use an image that's already downloaded" do
expect(File).to receive(:exist?).with("/var/storage/images/ubuntu-jammy-20240110.raw").and_return(true)
expect(bi).not_to receive(:curl_image)
bi.download(url: "url", ca_path: "ca_path", sha256sum: "sha256sum")
end
it "can download an image" do
expect(File).to receive(:exist?).with("/var/storage/images/ubuntu-jammy-20240110.raw").and_return(false)
expect(FileUtils).to receive(:mkdir_p).with("/var/storage/images")
expect(bi).to receive(:image_ext).with("url").and_return(".img")
tmp_path = "/var/storage/images/ubuntu-jammy-20240110.img.tmp"
expect(bi).to receive(:curl_image).with("url", tmp_path, "ca_path").and_return("returned_sha256sum")
expect(bi).to receive(:verify_sha256sum).with("returned_sha256sum", "sha256sum")
expect(bi).to receive(:convert_image).with(tmp_path, "qcow2")
expect(FileUtils).to receive(:rm_r).with(tmp_path)
bi.download(url: "url", ca_path: "ca_path", sha256sum: "sha256sum")
end
it "can download an image with htcat" do
expect(File).to receive(:exist?).with("/var/storage/images/ubuntu-jammy-20240110.raw").and_return(false)
expect(FileUtils).to receive(:mkdir_p).with("/var/storage/images")
expect(bi).to receive(:image_ext).with("url").and_return(".img")
tmp_path = "/var/storage/images/ubuntu-jammy-20240110.img.tmp"
expect(bi).to receive(:htcat_image).with("url", tmp_path).and_return("returned_sha256sum")
expect(bi).to receive(:verify_sha256sum).with("returned_sha256sum", "sha256sum")
expect(bi).to receive(:convert_image).with(tmp_path, "qcow2")
expect(FileUtils).to receive(:rm_r).with(tmp_path)
bi.download(url: "url", ca_path: "ca_path", sha256sum: "sha256sum", use_htcat: true)
end
end
describe "#image_ext" do
it "can handle image without query params" do
url = "http://minio.ubicloud.com:9000/ubicloud-images/ubuntu-22.04-x64.vhd"
expect(bi.image_ext(url)).to eq(".vhd")
end
it "can handle image with query params" do
url = "http://minio.ubicloud.com:9000/ubicloud-images/ubuntu-22.04-x64.vhd?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=user%2F20240112%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240112T132931Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=aabbcc"
expect(bi.image_ext(url)).to eq(".vhd")
end
end
describe "#initial_format" do
it "fails if initial image has unsupported format" do
expect { bi.initial_format(".iso") }.to raise_error RuntimeError, "Unsupported boot_image format: .iso"
end
end
describe "#curl_image" do
it "can curl image without ca_path" do
expect(File).to receive(:open) do |path, *_args|
expect(path).to eq("/var/storage/images/ubuntu-jammy-20240110.img.tmp")
end.and_yield
expect(bi).to receive(:r).with(
"bash -c 'curl -f -L10 url | tee >(openssl dgst -sha256) > /var/storage/images/ubuntu-jammy-20240110.img.tmp'"
).and_return("SHA2-256(stdin)= 81fae9cc21e2b1e3a9a4526c7dad3131b668e346c580702235ad4d02645d9455\n")
expect(
bi.curl_image("url", "/var/storage/images/ubuntu-jammy-20240110.img.tmp", nil)
).to eq("81fae9cc21e2b1e3a9a4526c7dad3131b668e346c580702235ad4d02645d9455")
end
it "can curl image with ca_path" do
expect(File).to receive(:open) do |path, *_args|
expect(path).to eq("/var/storage/images/ubuntu-jammy-20240110.img.tmp")
end.and_yield
expect(bi).to receive(:r).with(
"bash -c 'curl -f -L10 url --cacert ca_path | tee >(openssl dgst -sha256) > /var/storage/images/ubuntu-jammy-20240110.img.tmp'"
).and_return("SHA2-256(stdin)= 81fae9cc21e2b1e3a9a4526c7dad3131b668e346c580702235ad4d02645d9455\n")
bi.curl_image("url", "/var/storage/images/ubuntu-jammy-20240110.img.tmp", "ca_path")
end
end
describe "#htcat_image" do
it "can htcat image with sha256 checksum" do
expect(File).to receive(:open) do |path, *_args|
expect(path).to eq("/var/storage/images/ubuntu-jammy-20240110.img.tmp")
end.and_yield
expect(bi).to receive(:r).with(
"bash -c 'htcat -parallelism=12 -max-fragment-size=32 URL | tee >(openssl dgst -sha256) > /var/storage/images/ubuntu-jammy-20240110.img.tmp'"
).and_return("SHA2-256(stdin)= 81fae9cc21e2b1e3a9a4526c7dad3131b668e346c580702235ad4d02645d9455\n")
bi.htcat_image("URL", "/var/storage/images/ubuntu-jammy-20240110.img.tmp")
end
end
describe "#verify_sha256sum" do
it "succeeds if sha256 sums match" do
expect { bi.verify_sha256sum("sha256sum", "sha256sum") }.not_to raise_error
end
it "fails if sha256 sums do not match" do
expect { bi.verify_sha256sum("sha256sum", "invalid") }.to raise_error(RuntimeError, "Invalid SHA256 sum.")
end
it "succeeds if expected sha256 sum is nil" do
expect { bi.verify_sha256sum("sha256sum", nil) }.not_to raise_error
end
end
describe "#convert_image" do
it "can convert image" do
expect(bi).to receive(:r).with("qemu-img convert -p -f qcow2 -O raw /var/storage/images/ubuntu-jammy-20240110.img.tmp /var/storage/images/ubuntu-jammy-20240110.raw")
bi.convert_image("/var/storage/images/ubuntu-jammy-20240110.img.tmp", "qcow2")
end
it "does not convert image if it's in raw format already" do
expect(File).to receive(:rename).with("/var/storage/images/ubuntu-jammy-20240110.img.tmp", "/var/storage/images/ubuntu-jammy-20240110.raw")
bi.convert_image("/var/storage/images/ubuntu-jammy-20240110.img.tmp", "raw")
end
end
end