Skip to content

Commit 44a5dac

Browse files
mudlermudler
authored
feat(backend): add stablediffusion-ggml (#4289)
* feat(backend): add stablediffusion-ggml Signed-off-by: Ettore Di Giacinto <[email protected]> * chore(ci): track stablediffusion-ggml Signed-off-by: Ettore Di Giacinto <[email protected]> * fixups Signed-off-by: Ettore Di Giacinto <[email protected]> * Use default scheduler and sampler if not specified Signed-off-by: Ettore Di Giacinto <[email protected]> * fixups Signed-off-by: Ettore Di Giacinto <[email protected]> * Move cfg scale out of diffusers block Signed-off-by: Ettore Di Giacinto <[email protected]> * Make it working Signed-off-by: Ettore Di Giacinto <[email protected]> * fix: set free_params_immediately to false to call the model in sequence leejet/stable-diffusion.cpp#366 Signed-off-by: Ettore Di Giacinto <[email protected]> --------- Signed-off-by: Ettore Di Giacinto <[email protected]>
1 parent 074b52b commit 44a5dac

File tree

12 files changed

+437
-21
lines changed

12 files changed

+437
-21
lines changed

‎.github/workflows/bump_deps.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ jobs:
1818
- repository: "PABannier/bark.cpp"
1919
variable: "BARKCPP_VERSION"
2020
branch: "main"
21+
- repository: "leejet/stable-diffusion.cpp"
22+
variable: "STABLEDIFFUSION_GGML_VERSION"
23+
branch: "master"
2124
- repository: "mudler/go-stable-diffusion"
2225
variable: "STABLEDIFFUSION_VERSION"
2326
branch: "master"

‎Makefile

+38-7
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ TINYDREAM_VERSION?=c04fa463ace9d9a6464313aa5f9cd0f953b6c057
3030
BARKCPP_REPO?=https://github.com/PABannier/bark.cpp.git
3131
BARKCPP_VERSION?=v1.0.0
3232

33+
# stablediffusion.cpp (ggml)
34+
STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp
35+
STABLEDIFFUSION_GGML_VERSION?=4570715727f35e5a07a76796d823824c8f42206c
36+
3337
ONNX_VERSION?=1.20.0
3438
ONNX_ARCH?=x64
3539
ONNX_OS?=linux
@@ -209,6 +213,7 @@ ALL_GRPC_BACKENDS+=backend-assets/grpc/whisper
209213
ifeq ($(ONNX_OS),linux)
210214
ifeq ($(ONNX_ARCH),x64)
211215
ALL_GRPC_BACKENDS+=backend-assets/grpc/bark-cpp
216+
ALL_GRPC_BACKENDS+=backend-assets/grpc/stablediffusion-ggml
212217
endif
213218
endif
214219

@@ -244,25 +249,26 @@ sources/go-llama.cpp:
244249
git checkout $(GOLLAMA_VERSION) && \
245250
git submodule update --init --recursive --depth 1 --single-branch
246251

252+
sources/go-llama.cpp/libbinding.a: sources/go-llama.cpp
253+
$(MAKE) -C sources/go-llama.cpp BUILD_TYPE=$(STABLE_BUILD_TYPE) libbinding.a
254+
255+
## bark.cpp
247256
sources/bark.cpp:
248-
git clone --recursive https://github.com/PABannier/bark.cpp.git sources/bark.cpp && \
257+
git clone --recursive $(BARKCPP_REPO) sources/bark.cpp && \
249258
cd sources/bark.cpp && \
250259
git checkout $(BARKCPP_VERSION) && \
251260
git submodule update --init --recursive --depth 1 --single-branch
252261

253262
sources/bark.cpp/build/libbark.a: sources/bark.cpp
254263
cd sources/bark.cpp && \
255-
mkdir build && \
264+
mkdir -p build && \
256265
cd build && \
257266
cmake $(CMAKE_ARGS) .. && \
258267
cmake --build . --config Release
259268

260269
backend/go/bark/libbark.a: sources/bark.cpp/build/libbark.a
261270
$(MAKE) -C backend/go/bark libbark.a
262271

263-
sources/go-llama.cpp/libbinding.a: sources/go-llama.cpp
264-
$(MAKE) -C sources/go-llama.cpp BUILD_TYPE=$(STABLE_BUILD_TYPE) libbinding.a
265-
266272
## go-piper
267273
sources/go-piper:
268274
mkdir -p sources/go-piper
@@ -276,7 +282,7 @@ sources/go-piper:
276282
sources/go-piper/libpiper_binding.a: sources/go-piper
277283
$(MAKE) -C sources/go-piper libpiper_binding.a example/main piper.o
278284

279-
## stable diffusion
285+
## stable diffusion (onnx)
280286
sources/go-stable-diffusion:
281287
mkdir -p sources/go-stable-diffusion
282288
cd sources/go-stable-diffusion && \
@@ -289,6 +295,30 @@ sources/go-stable-diffusion:
289295
sources/go-stable-diffusion/libstablediffusion.a: sources/go-stable-diffusion
290296
CPATH="$(CPATH):/usr/include/opencv4" $(MAKE) -C sources/go-stable-diffusion libstablediffusion.a
291297

298+
## stablediffusion (ggml)
299+
sources/stablediffusion-ggml.cpp:
300+
git clone --recursive $(STABLEDIFFUSION_GGML_REPO) sources/stablediffusion-ggml.cpp && \
301+
cd sources/stablediffusion-ggml.cpp && \
302+
git checkout $(STABLEDIFFUSION_GGML_VERSION) && \
303+
git submodule update --init --recursive --depth 1 --single-branch
304+
305+
sources/stablediffusion-ggml.cpp/build/libstable-diffusion.a: sources/stablediffusion-ggml.cpp
306+
cd sources/stablediffusion-ggml.cpp && \
307+
mkdir -p build && \
308+
cd build && \
309+
cmake $(CMAKE_ARGS) .. && \
310+
cmake --build . --config Release
311+
312+
backend/go/image/stablediffusion-ggml/libsd.a: sources/stablediffusion-ggml.cpp/build/libstable-diffusion.a
313+
$(MAKE) -C backend/go/image/stablediffusion-ggml libsd.a
314+
315+
backend-assets/grpc/stablediffusion-ggml: backend/go/image/stablediffusion-ggml/libsd.a backend-assets/grpc
316+
CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(CURDIR)/backend/go/image/stablediffusion-ggml/ LIBRARY_PATH=$(CURDIR)/backend/go/image/stablediffusion-ggml/ \
317+
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/stablediffusion-ggml ./backend/go/image/stablediffusion-ggml/
318+
ifneq ($(UPX),)
319+
$(UPX) backend-assets/grpc/stablediffusion-ggml
320+
endif
321+
292322
sources/onnxruntime:
293323
mkdir -p sources/onnxruntime
294324
curl -L https://github.com/microsoft/onnxruntime/releases/download/v$(ONNX_VERSION)/onnxruntime-$(ONNX_OS)-$(ONNX_ARCH)-$(ONNX_VERSION).tgz -o sources/onnxruntime/onnxruntime-$(ONNX_OS)-$(ONNX_ARCH)-$(ONNX_VERSION).tgz
@@ -329,7 +359,7 @@ sources/whisper.cpp:
329359
sources/whisper.cpp/libwhisper.a: sources/whisper.cpp
330360
cd sources/whisper.cpp && $(MAKE) libwhisper.a libggml.a
331361

332-
get-sources: sources/go-llama.cpp sources/go-piper sources/bark.cpp sources/whisper.cpp sources/go-stable-diffusion sources/go-tiny-dream backend/cpp/llama/llama.cpp
362+
get-sources: sources/go-llama.cpp sources/go-piper sources/stablediffusion-ggml.cpp sources/bark.cpp sources/whisper.cpp sources/go-stable-diffusion sources/go-tiny-dream backend/cpp/llama/llama.cpp
333363

334364
replace:
335365
$(GOCMD) mod edit -replace github.com/ggerganov/whisper.cpp=$(CURDIR)/sources/whisper.cpp
@@ -372,6 +402,7 @@ clean: ## Remove build related file
372402
$(MAKE) -C backend/cpp/grpc clean
373403
$(MAKE) -C backend/go/bark clean
374404
$(MAKE) -C backend/cpp/llama clean
405+
$(MAKE) -C backend/go/image/stablediffusion-ggml clean
375406
rm -rf backend/cpp/llama-* || true
376407
$(MAKE) dropreplace
377408
$(MAKE) protogen-clean

‎backend/backend.proto

+2
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,8 @@ message ModelOptions {
240240

241241
repeated string LoraAdapters = 60;
242242
repeated float LoraScales = 61;
243+
244+
repeated string Options = 62;
243245
}
244246

245247
message Result {

‎backend/go/image/stablediffusion-ggml/Makefile

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
INCLUDE_PATH := $(abspath ./)
2+
LIBRARY_PATH := $(abspath ./)
3+
4+
AR?=ar
5+
6+
BUILD_TYPE?=
7+
# keep standard at C11 and C++11
8+
CXXFLAGS = -I. -I$(INCLUDE_PATH)/../../../../sources/stablediffusion-ggml.cpp/thirdparty -I$(INCLUDE_PATH)/../../../../sources/stablediffusion-ggml.cpp/ggml/include -I$(INCLUDE_PATH)/../../../../sources/stablediffusion-ggml.cpp -O3 -DNDEBUG -std=c++17 -fPIC
9+
10+
# warnings
11+
CXXFLAGS += -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function
12+
13+
gosd.o:
14+
$(CXX) $(CXXFLAGS) gosd.cpp -o gosd.o -c
15+
16+
libsd.a: gosd.o
17+
cp $(INCLUDE_PATH)/../../../../sources/stablediffusion-ggml.cpp/build/libstable-diffusion.a ./libsd.a
18+
$(AR) rcs libsd.a gosd.o
19+
20+
clean:
21+
rm -f gosd.o libsd.a

‎backend/go/image/stablediffusion-ggml/gosd.cpp

+228
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
#include <stdio.h>
2+
#include <string.h>
3+
#include <time.h>
4+
#include <iostream>
5+
#include <random>
6+
#include <string>
7+
#include <vector>
8+
#include "gosd.h"
9+
10+
// #include "preprocessing.hpp"
11+
#include "flux.hpp"
12+
#include "stable-diffusion.h"
13+
14+
#define STB_IMAGE_IMPLEMENTATION
15+
#define STB_IMAGE_STATIC
16+
#include "stb_image.h"
17+
18+
#define STB_IMAGE_WRITE_IMPLEMENTATION
19+
#define STB_IMAGE_WRITE_STATIC
20+
#include "stb_image_write.h"
21+
22+
#define STB_IMAGE_RESIZE_IMPLEMENTATION
23+
#define STB_IMAGE_RESIZE_STATIC
24+
#include "stb_image_resize.h"
25+
26+
// Names of the sampler method, same order as enum sample_method in stable-diffusion.h
27+
const char* sample_method_str[] = {
28+
"euler_a",
29+
"euler",
30+
"heun",
31+
"dpm2",
32+
"dpm++2s_a",
33+
"dpm++2m",
34+
"dpm++2mv2",
35+
"ipndm",
36+
"ipndm_v",
37+
"lcm",
38+
};
39+
40+
// Names of the sigma schedule overrides, same order as sample_schedule in stable-diffusion.h
41+
const char* schedule_str[] = {
42+
"default",
43+
"discrete",
44+
"karras",
45+
"exponential",
46+
"ays",
47+
"gits",
48+
};
49+
50+
sd_ctx_t* sd_c;
51+
52+
sample_method_t sample_method;
53+
54+
int load_model(char *model, char* options[], int threads, int diff) {
55+
fprintf (stderr, "Loading model!\n");
56+
57+
char *stableDiffusionModel = "";
58+
if (diff == 1 ) {
59+
stableDiffusionModel = model;
60+
model = "";
61+
}
62+
63+
// decode options. Options are in form optname:optvale, or if booleans only optname.
64+
char *clip_l_path = "";
65+
char *clip_g_path = "";
66+
char *t5xxl_path = "";
67+
char *vae_path = "";
68+
char *scheduler = "";
69+
char *sampler = "";
70+
71+
// If options is not NULL, parse options
72+
for (int i = 0; options[i] != NULL; i++) {
73+
char *optname = strtok(options[i], ":");
74+
char *optval = strtok(NULL, ":");
75+
if (optval == NULL) {
76+
optval = "true";
77+
}
78+
79+
if (!strcmp(optname, "clip_l_path")) {
80+
clip_l_path = optval;
81+
}
82+
if (!strcmp(optname, "clip_g_path")) {
83+
clip_g_path = optval;
84+
}
85+
if (!strcmp(optname, "t5xxl_path")) {
86+
t5xxl_path = optval;
87+
}
88+
if (!strcmp(optname, "vae_path")) {
89+
vae_path = optval;
90+
}
91+
if (!strcmp(optname, "scheduler")) {
92+
scheduler = optval;
93+
}
94+
if (!strcmp(optname, "sampler")) {
95+
sampler = optval;
96+
}
97+
}
98+
99+
int sample_method_found = -1;
100+
for (int m = 0; m < N_SAMPLE_METHODS; m++) {
101+
if (!strcmp(sampler, sample_method_str[m])) {
102+
sample_method_found = m;
103+
}
104+
}
105+
if (sample_method_found == -1) {
106+
fprintf(stderr, "Invalid sample method, default to EULER_A!\n");
107+
sample_method_found = EULER_A;
108+
}
109+
sample_method = (sample_method_t)sample_method_found;
110+
111+
int schedule_found = -1;
112+
for (int d = 0; d < N_SCHEDULES; d++) {
113+
if (!strcmp(scheduler, schedule_str[d])) {
114+
schedule_found = d;
115+
fprintf (stderr, "Found scheduler: %s\n", scheduler);
116+
117+
}
118+
}
119+
120+
if (schedule_found == -1) {
121+
fprintf (stderr, "Invalid scheduler! using DEFAULT\n");
122+
schedule_found = DEFAULT;
123+
}
124+
125+
schedule_t schedule = (schedule_t)schedule_found;
126+
127+
fprintf (stderr, "Creating context\n");
128+
sd_ctx_t* sd_ctx = new_sd_ctx(model,
129+
clip_l_path,
130+
clip_g_path,
131+
t5xxl_path,
132+
stableDiffusionModel,
133+
vae_path,
134+
"",
135+
"",
136+
"",
137+
"",
138+
"",
139+
false,
140+
false,
141+
false,
142+
threads,
143+
SD_TYPE_COUNT,
144+
STD_DEFAULT_RNG,
145+
schedule,
146+
false,
147+
false,
148+
false,
149+
false);
150+
151+
if (sd_ctx == NULL) {
152+
fprintf (stderr, "failed loading model (generic error)\n");
153+
return 1;
154+
}
155+
fprintf (stderr, "Created context: OK\n");
156+
157+
sd_c = sd_ctx;
158+
159+
return 0;
160+
}
161+
162+
int gen_image(char *text, char *negativeText, int width, int height, int steps, int seed , char *dst, float cfg_scale) {
163+
164+
sd_image_t* results;
165+
166+
std::vector<int> skip_layers = {7, 8, 9};
167+
168+
fprintf (stderr, "Generating image\n");
169+
170+
results = txt2img(sd_c,
171+
text,
172+
negativeText,
173+
-1, //clip_skip
174+
cfg_scale, // sfg_scale
175+
3.5f,
176+
width,
177+
height,
178+
sample_method,
179+
steps,
180+
seed,
181+
1,
182+
NULL,
183+
0.9f,
184+
20.f,
185+
false,
186+
"",
187+
skip_layers.data(),
188+
skip_layers.size(),
189+
0,
190+
0.01,
191+
0.2);
192+
193+
if (results == NULL) {
194+
fprintf (stderr, "NO results\n");
195+
return 1;
196+
}
197+
198+
if (results[0].data == NULL) {
199+
fprintf (stderr, "Results with no data\n");
200+
return 1;
201+
}
202+
203+
fprintf (stderr, "Writing PNG\n");
204+
205+
fprintf (stderr, "DST: %s\n", dst);
206+
fprintf (stderr, "Width: %d\n", results[0].width);
207+
fprintf (stderr, "Height: %d\n", results[0].height);
208+
fprintf (stderr, "Channel: %d\n", results[0].channel);
209+
fprintf (stderr, "Data: %p\n", results[0].data);
210+
211+
stbi_write_png(dst, results[0].width, results[0].height, results[0].channel,
212+
results[0].data, 0, NULL);
213+
fprintf (stderr, "Saved resulting image to '%s'\n", dst);
214+
215+
// TODO: free results. Why does it crash?
216+
217+
free(results[0].data);
218+
results[0].data = NULL;
219+
free(results);
220+
fprintf (stderr, "gen_image is done", dst);
221+
222+
return 0;
223+
}
224+
225+
int unload() {
226+
free_sd_ctx(sd_c);
227+
}
228+

0 commit comments

Comments
 (0)