refactor: deduplicate and automatically manage CPU backend

Author: wbruna

src/stable-diffusion.cpp

@@ -114,6 +114,7 @@ class StableDiffusionGGML {
     ggml_backend_t vae_backend         = nullptr;
 
     sd_ggml_backend_ptr main_backend;
+    sd_ggml_backend_ptr cpu_backend;
 
     SDVersion version;
     bool vae_decode_only         = false;
@@ -161,23 +162,20 @@ class StableDiffusionGGML {
 
     StableDiffusionGGML() = default;
 
-    ~StableDiffusionGGML() {
-        if (clip_backend != backend) {
-            ggml_backend_free(clip_backend);
-        }
-        if (control_net_backend != backend) {
-            ggml_backend_free(control_net_backend);
-        }
-        if (vae_backend != backend) {
-            ggml_backend_free(vae_backend);
-        }
-    }
+    ~StableDiffusionGGML() = default;
 
     void init_backend() {
         main_backend = sd_get_default_backend();
         backend = main_backend.get();
     }
 
+    ggml_backend_t get_cpu_backend() {
+        if (!cpu_backend) {
+            cpu_backend = sd_ggml_backend_ptr(ggml_backend_cpu_init());
+        }
+        return cpu_backend.get();
+    }
+
     std::shared_ptr<RNG> get_rng(rng_type_t rng_type) {
         if (rng_type == STD_DEFAULT_RNG) {
             return std::make_shared<STDDefaultRNG>();
@@ -387,7 +385,7 @@ class StableDiffusionGGML {
             clip_backend = backend;
             if (clip_on_cpu && !ggml_backend_is_cpu(backend)) {
                 LOG_INFO("CLIP: Using CPU backend");
-                clip_backend = ggml_backend_cpu_init();
+                clip_backend = get_cpu_backend();
             }
             if (sd_version_is_sd3(version)) {
                 cond_stage_model = std::make_shared<SD3CLIPEmbedder>(clip_backend,
@@ -569,7 +567,7 @@ class StableDiffusionGGML {
 
             if (sd_ctx_params->keep_vae_on_cpu && !ggml_backend_is_cpu(backend)) {
                 LOG_INFO("VAE Autoencoder: Using CPU backend");
-                vae_backend = ggml_backend_cpu_init();
+                vae_backend = get_cpu_backend();
             } else {
                 vae_backend = backend;
             }
@@ -665,7 +663,7 @@ class StableDiffusionGGML {
                 ggml_backend_t controlnet_backend = nullptr;
                 if (sd_ctx_params->keep_control_net_on_cpu && !ggml_backend_is_cpu(backend)) {
                     LOG_DEBUG("ControlNet: Using CPU backend");
-                    controlnet_backend = ggml_backend_cpu_init();
+                    controlnet_backend = get_cpu_backend();
                 } else {
                     controlnet_backend = backend;
                 }