Skip to content

feat: Add generation parameters into video metadata #1901

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
CAHbKA-IV wants to merge 2 commits into
base: master
Choose a base branch
from
+75 −20
Open

feat: Add generation parameters into video metadata #1901

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a3fc297
Initial release
CAHbKA-IV Aug 22, 2026
33a2387
write webm metadata before frames
CAHbKA-IV Aug 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion examples/cli/main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -527,12 +527,16 @@ bool save_results(const SDCliParams& cli_params,
if (cli_params.mode == VID_GEN && num_results > 1) {
if (ext_lower != ".avi" && ext_lower != ".webp" && ext_lower != ".webm")
ext = ".avi";
std::string params = gen_params.embed_image_metadata
? get_image_params(ctx_params, gen_params, gen_params.seed, cli_params.mode)
: "";

fs::path video_path = base_path;
video_path += ext;
std::string final_ext_lower = ext.string();
std::transform(final_ext_lower.begin(), final_ext_lower.end(), final_ext_lower.begin(), ::tolower);
const bool mux_audio = generated_audio != nullptr && (final_ext_lower == ".avi" || final_ext_lower == ".webm");
if (create_video_from_sd_images(video_path.string().c_str(), results, num_results, gen_params.fps, 90, mux_audio ? generated_audio : nullptr) == 0) {
if (create_video_from_sd_images(video_path.string().c_str(), results, num_results, gen_params.fps, 90, mux_audio ? generated_audio : nullptr, params) == 0) {
LOG_INFO("save result video to '%s'", video_path.string().c_str());
if (generated_audio != nullptr && !mux_audio) {
fs::path wav_path = video_path;
Expand Down
63 changes: 52 additions & 11 deletions examples/common/media_io.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -810,7 +810,30 @@ uint8_t* load_image_from_memory(const char* image_bytes,
return load_image_common(true, image_bytes, len, width, height, expected_width, expected_height, expected_channel);
}

std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio) {

static void append_avi_metadata(std::vector<uint8_t>& data, const std::string& parameters) {
if (parameters.empty()) {
return;
}

std::vector<uint8_t> info_content;

write_fourcc(info_content, "INFO");

write_fourcc(info_content, "ICMT");
write_u32_le(info_content, static_cast<uint32_t>(parameters.size()));
info_content.insert(info_content.end(), parameters.begin(), parameters.end());
if (parameters.size() & 1u) {
info_content.push_back(0);
}

write_fourcc(data, "LIST");
write_u32_le(data, static_cast<uint32_t>(info_content.size()));
data.insert(data.end(), info_content.begin(), info_content.end());
size_t start_pos = data.size();
}

std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) {
if (num_images == 0) {
fprintf(stderr, "Error: Image array is empty.\n");
return {};
Expand Down Expand Up @@ -997,6 +1020,8 @@ std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images
const size_t movi_size = avi_data.size() - movi_size_pos - 4;
patch_u32_le(avi_data, movi_size_pos, static_cast<uint32_t>(movi_size));

append_avi_metadata(avi_data, parameters);

write_fourcc(avi_data, "idx1");
write_u32_le(avi_data, static_cast<uint32_t>(index.size() * 16));
for (const auto& entry : index) {
Expand All @@ -1012,8 +1037,8 @@ std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images
return avi_data;
}

int create_mjpg_avi_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio) {
std::vector<uint8_t> avi_data = create_mjpg_avi_from_sd_images_to_vector(images, num_images, fps, quality, audio);
int create_mjpg_avi_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) {
std::vector<uint8_t> avi_data = create_mjpg_avi_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters);
if (avi_data.empty()) {
return -1;
}
Expand Down Expand Up @@ -1143,7 +1168,7 @@ int create_animated_webp_from_sd_images(const char* filename, sd_image_t* images
#endif

#ifdef SD_USE_WEBM
std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio) {
std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) {
if (num_images == 0) {
fprintf(stderr, "Error: Image array is empty.\n");
return {};
Expand Down Expand Up @@ -1210,6 +1235,21 @@ std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images, in
segment.GetSegmentInfo()->set_writing_app("stable-diffusion.cpp");
segment.GetSegmentInfo()->set_muxing_app("stable-diffusion.cpp");

LOG_DEBUG("Embedding parameters to metadata: %s", parameters.c_str());
if (!parameters.empty()) {
mkvmuxer::Tag* tag = segment.AddTag();

if (tag) {
if (!tag->add_simple_tag("COMMENT", parameters.c_str())) {
LOG_WARN("Failed to add COMMENT simple tag.");
}
} else {
LOG_WARN("Failed to add tag to segment.");
}
} else {
LOG_INFO("Paramaters is empty, COMMENT tag not embedded.\n");
}

const uint64_t frame_duration_ns = std::max<uint64_t>(
1, static_cast<uint64_t>(std::llround(1000000000.0 / static_cast<double>(fps))));
uint64_t timestamp_ns = 0;
Expand Down Expand Up @@ -1268,8 +1308,8 @@ std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images, in
return writer.data();
}

int create_webm_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio) {
std::vector<uint8_t> webm_data = create_webm_from_sd_images_to_vector(images, num_images, fps, quality, audio);
int create_webm_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) {
std::vector<uint8_t> webm_data = create_webm_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters);
if (webm_data.empty()) {
return -1;
}
Expand All @@ -1286,7 +1326,8 @@ std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& ou
int num_images,
int fps,
int quality,
const sd_audio_t* audio) {
const sd_audio_t* audio,
const std::string& parameters) {
std::string format = output_format;
std::transform(format.begin(), format.end(), format.begin(),
[](unsigned char c) { return static_cast<char>(tolower(c)); });
Expand All @@ -1296,7 +1337,7 @@ std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& ou

#ifdef SD_USE_WEBM
if (format == "webm") {
return create_webm_from_sd_images_to_vector(images, num_images, fps, quality, audio);
return create_webm_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters);
}
#endif

Expand All @@ -1306,14 +1347,14 @@ std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& ou
}
#endif

return create_mjpg_avi_from_sd_images_to_vector(images, num_images, fps, quality, audio);
return create_mjpg_avi_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters);
}

int create_video_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio) {
int create_video_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) {
std::string path = filename ? filename : "";
auto pos = path.find_last_of('.');
std::string ext = pos == std::string::npos ? "" : path.substr(pos);
std::vector<uint8_t> video_data = create_video_from_sd_images_to_vector(ext, images, num_images, fps, quality, audio);
std::vector<uint8_t> video_data = create_video_from_sd_images_to_vector(ext, images, num_images, fps, quality, audio, parameters);
if (video_data.empty()) {
return -1;
}
Expand Down
20 changes: 13 additions & 7 deletions examples/common/media_io.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,12 +58,14 @@ int create_mjpg_avi_from_sd_images(const char* filename,
int num_images,
int fps,
int quality = 90,
const sd_audio_t* audio = nullptr);
const sd_audio_t* audio = nullptr,
const std::string& parameters = "");
std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images,
int num_images,
int fps,
int quality = 90,
const sd_audio_t* audio = nullptr);
const sd_audio_t* audio = nullptr,
const std::string& parameters = "");

#ifdef SD_USE_WEBP
int create_animated_webp_from_sd_images(const char* filename,
Expand All @@ -83,26 +85,30 @@ int create_webm_from_sd_images(const char* filename,
int num_images,
int fps,
int quality = 90,
const sd_audio_t* audio = nullptr);
const sd_audio_t* audio = nullptr,
const std::string& parameters = "");
std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images,
int num_images,
int fps,
int quality = 90,
const sd_audio_t* audio = nullptr);
const sd_audio_t* audio = nullptr,
const std::string& parameters = "");
#endif

int create_video_from_sd_images(const char* filename,
sd_image_t* images,
int num_images,
int fps,
int quality = 90,
const sd_audio_t* audio = nullptr);
const sd_audio_t* audio = nullptr,
const std::string& parameters = "");
std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& output_format,
sd_image_t* images,
int num_images,
int fps,
int quality = 90,
const sd_audio_t* audio = nullptr);
const sd_audio_t* audio = nullptr,
const std::string& parameters = "");

bool write_wav_to_file(const std::string& path,
const float* interleaved_samples,
Expand Down
6 changes: 5 additions & 1 deletion examples/server/async_jobs.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,9 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
int& output_fps,
std::string& error_message) {
sd_vid_gen_params_t params = job.vid_gen.to_sd_vid_gen_params_t();
std::string str_params = job.vid_gen.gen_params.embed_image_metadata
? get_image_params(*runtime.ctx_params, job.vid_gen.gen_params, job.vid_gen.gen_params.seed)
: "";

SDImageVec results;
int num_results = 0;
Expand All @@ -263,7 +266,8 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
num_results,
job.vid_gen.gen_params.fps,
job.vid_gen.output_compression,
generated_audio);
generated_audio,
str_params);
free_sd_audio(generated_audio);
if (video_bytes.empty()) {
error_message = "failed to encode generated video container";
Expand Down