feat: play audio and video in the browser, and open the media formats we only named

CMakeLists.txt

@@ -116,6 +116,7 @@ set(ODR_SOURCE_FILES
         "src/odr/internal/common/file.cpp"
         "src/odr/internal/common/filesystem.cpp"
         "src/odr/internal/common/image_file.cpp"
+        "src/odr/internal/common/media_file.cpp"
         "src/odr/internal/common/path.cpp"
         "src/odr/internal/common/random.cpp"
         "src/odr/internal/common/style.cpp"
@@ -138,6 +139,7 @@ set(ODR_SOURCE_FILES
         "src/odr/internal/html/html_service.cpp"
         "src/odr/internal/html/html_writer.cpp"
         "src/odr/internal/html/image_file.cpp"
+        "src/odr/internal/html/media_file.cpp"
         "src/odr/internal/html/pdf_file.cpp"
         "src/odr/internal/html/text_file.cpp"
 

apple/include/OdrCoreObjC/ODRHtml.h

@@ -16,6 +16,7 @@ typedef NS_ENUM(NSInteger, ODRHtmlResourceType) {
   ODRHtmlResourceTypeJs,
   ODRHtmlResourceTypeImage,
   ODRHtmlResourceTypeFont,
+  ODRHtmlResourceTypeMedia,
 } NS_SWIFT_NAME(HtmlResourceType);
 
 typedef NS_ENUM(NSInteger, ODRHtmlTableGridlines) {

apple/src/ODRHtml.mm

@@ -20,6 +20,7 @@
 ODR_SAME_ENUM(ODRHtmlResourceTypeJs, odr::HtmlResourceType::js);
 ODR_SAME_ENUM(ODRHtmlResourceTypeImage, odr::HtmlResourceType::image);
 ODR_SAME_ENUM(ODRHtmlResourceTypeFont, odr::HtmlResourceType::font);
+ODR_SAME_ENUM(ODRHtmlResourceTypeMedia, odr::HtmlResourceType::media);
 
 ODR_SAME_ENUM(ODRHtmlTableGridlinesNone, odr::HtmlTableGridlines::none);
 ODR_SAME_ENUM(ODRHtmlTableGridlinesSoft, odr::HtmlTableGridlines::soft);

jni/java/app/opendocument/core/HtmlResourceType.java

@@ -2,7 +2,7 @@
 
 /** Mirrors {@code odr::HtmlResourceType}; constant order must match the C++ declaration. */
 public enum HtmlResourceType {
-  HTML_FRAGMENT, CSS, JS, IMAGE, FONT;
+  HTML_FRAGMENT, CSS, JS, IMAGE, FONT, MEDIA;
 
   static HtmlResourceType fromNative(int code) {
     return code < 0 ? null : values()[code];

python/src/bind_html.cpp

@@ -21,7 +21,8 @@ void odr_python::bind_html(py::module_ &m) {
       .value("css", odr::HtmlResourceType::css)
       .value("js", odr::HtmlResourceType::js)
       .value("image", odr::HtmlResourceType::image)
-      .value("font", odr::HtmlResourceType::font);
+      .value("font", odr::HtmlResourceType::font)
+      .value("media", odr::HtmlResourceType::media);
 
   py::enum_<odr::HtmlTableGridlines>(m, "HtmlTableGridlines")
       .value("none", odr::HtmlTableGridlines::none)

src/odr/file.hpp

@@ -90,11 +90,12 @@ enum class FileType {
   // https://en.wikipedia.org/wiki/OpenType
   opentype_font,
 
-  // Detection only, like `word_perfect` above: the media formats a viewer is
-  // regularly handed alongside documents. Naming them is the point - a caller
-  // that knows a file is a video hands it to a player instead of asking us to
-  // translate it, and without a name the text fallback would call it plain
-  // text. New entries go at the end: the bindings mirror this enum by ordinal.
+  // The media formats a viewer is regularly handed alongside documents.
+  // Nothing here is decoded - opening one wraps its bytes, and translating it
+  // puts those bytes in an `<img>` or in a player and lets the browser do the
+  // work. Naming them is what makes that possible; without a name the text
+  // fallback would call a video plain text.
+  // New entries go at the end: the bindings mirror this enum by ordinal.
   // https://en.wikipedia.org/wiki/WebP
   webp,
   // https://en.wikipedia.org/wiki/TIFF

src/odr/html.cpp

@@ -14,6 +14,7 @@
 #include <odr/internal/html/font_file.hpp>
 #include <odr/internal/html/html_writer.hpp>
 #include <odr/internal/html/image_file.hpp>
+#include <odr/internal/html/media_file.hpp>
 #include <odr/internal/html/pdf_file.hpp>
 #include <odr/internal/html/text_file.hpp>
 #include <odr/internal/util/file_util.hpp>
@@ -226,6 +227,14 @@ HtmlService html::translate(const DecodedFile &file,
   if (file.is_font_file()) {
     return translate(file.as_font_file(), cache_path, config, logger);
   }
+  // No wrapper type to go through: nothing is decoded, so the plain
+  // `DecodedFile` already carries the bytes and the type that names them.
+  if (const FileCategory category = file.file_category();
+      category == FileCategory::audio || category == FileCategory::video) {
+    std::filesystem::create_directories(cache_path);
+    return internal::html::create_media_service(file, cache_path, config,
+                                                logger);
+  }
 
   throw UnsupportedFileType(file.file_type());
 }

src/odr/html.hpp

@@ -31,6 +31,9 @@ enum class HtmlResourceType {
   js,
   image,
   font,
+  // appended rather than sorted in: the bindings mirror this enum by ordinal
+  /// Audio or video, never embedded — see @ref HtmlConfig::embed_images.
+  media,
 };
 
 class HtmlResource final {

src/odr/internal/common/media_file.cpp

@@ -0,0 +1,37 @@
+#include <odr/internal/common/media_file.hpp>
+
+#include <odr/odr.hpp>
+
+namespace odr::internal {
+
+MediaFile::MediaFile(std::shared_ptr<abstract::File> file,
+                     const FileType file_type)
+    : m_file{std::move(file)}, m_file_type{file_type} {}
+
+std::shared_ptr<abstract::File> MediaFile::file() const noexcept {
+  return m_file;
+}
+
+FileType MediaFile::file_type() const noexcept { return m_file_type; }
+
+FileCategory MediaFile::file_category() const noexcept {
+  return file_category_by_file_type(m_file_type);
+}
+
+FileMeta MediaFile::file_meta() const noexcept {
+  FileMeta result;
+  result.type = file_type();
+  result.mimetype = mimetype();
+  return result;
+}
+
+std::string_view MediaFile::mimetype() const noexcept {
+  // not `mimetype_by_file_type` — that throws, and this is `noexcept`
+  const std::span<const std::string_view> mimetypes =
+      mimetypes_by_file_type(m_file_type);
+  return mimetypes.empty() ? "" : mimetypes.front();
+}
+
+bool MediaFile::is_decodable() const noexcept { return false; }
+
+} // namespace odr::internal

src/odr/internal/common/media_file.hpp

@@ -0,0 +1,28 @@
+#pragma once
+
+#include <odr/internal/abstract/file.hpp>
+
+namespace odr::internal {
+
+/// Audio and video: nothing is decoded, the bytes are handed to the browser as
+/// they are. One class covers both, the category coming from the file type
+/// table rather than from the class.
+class MediaFile final : public abstract::DecodedFile {
+public:
+  MediaFile(std::shared_ptr<abstract::File> file, FileType file_type);
+
+  [[nodiscard]] std::shared_ptr<abstract::File> file() const noexcept override;
+
+  [[nodiscard]] FileType file_type() const noexcept override;
+  [[nodiscard]] FileCategory file_category() const noexcept override;
+  [[nodiscard]] FileMeta file_meta() const noexcept override;
+  [[nodiscard]] std::string_view mimetype() const noexcept override;
+
+  [[nodiscard]] bool is_decodable() const noexcept override;
+
+private:
+  std::shared_ptr<abstract::File> m_file;
+  FileType m_file_type;
+};
+
+} // namespace odr::internal

src/odr/internal/file_type_table.cpp

@@ -474,108 +474,111 @@ constexpr std::array table{
         DocumentType::unknown,
         {.detect_by_content = true, .open = true, .translate_html = true}},
 
-    // Named but not decoded, so `detect_by_content` is the only capability
-    // they carry - see the comment on these in `FileType`.
+    // Named but not decoded: `open` wraps the bytes without looking at them
+    // and `translate_html` hands them straight to the browser, in an `<img>`
+    // for the images below and in a player for the audio and video after them.
+    // Nothing here reads a pixel or a sample - see the comment on these in
+    // `FileType`.
     Row{FileType::webp,
         "webp"sv,
         webp_extensions,
         webp_mimetypes,
         FileCategory::image,
         DocumentType::unknown,
-        {.detect_by_content = true}},
+        {.detect_by_content = true, .open = true, .translate_html = true}},
     Row{FileType::tagged_image_file_format,
         "tiff"sv,
         tiff_extensions,
         tiff_mimetypes,
         FileCategory::image,
         DocumentType::unknown,
-        {.detect_by_content = true}},
+        {.detect_by_content = true, .open = true, .translate_html = true}},
     Row{FileType::high_efficiency_image_format,
         "heif"sv,
         heif_extensions,
         heif_mimetypes,
         FileCategory::image,
         DocumentType::unknown,
-        {.detect_by_content = true}},
+        {.detect_by_content = true, .open = true, .translate_html = true}},
     Row{FileType::av1_image_file_format,
         "avif"sv,
         avif_extensions,
         avif_mimetypes,
         FileCategory::image,
         DocumentType::unknown,
-        {.detect_by_content = true}},
+        {.detect_by_content = true, .open = true, .translate_html = true}},
 
     Row{FileType::mpeg_audio,
         "mp3"sv,
         mp3_extensions,
         mp3_mimetypes,
         FileCategory::audio,
         DocumentType::unknown,
-        {.detect_by_content = true}},
+        {.detect_by_content = true, .open = true, .translate_html = true}},
     Row{FileType::mpeg4_audio,
         "m4a"sv,
         m4a_extensions,
         m4a_mimetypes,
         FileCategory::audio,
         DocumentType::unknown,
-        {.detect_by_content = true}},
+        {.detect_by_content = true, .open = true, .translate_html = true}},
     Row{FileType::ogg_audio,
         "ogg"sv,
         ogg_extensions,
         ogg_mimetypes,
         FileCategory::audio,
         DocumentType::unknown,
-        {.detect_by_content = true}},
+        {.detect_by_content = true, .open = true, .translate_html = true}},
     Row{FileType::waveform_audio,
         "wav"sv,
         wav_extensions,
         wav_mimetypes,
         FileCategory::audio,
         DocumentType::unknown,
-        {.detect_by_content = true}},
+        {.detect_by_content = true, .open = true, .translate_html = true}},
     Row{FileType::free_lossless_audio_codec,
         "flac"sv,
         flac_extensions,
         flac_mimetypes,
         FileCategory::audio,
         DocumentType::unknown,
-        {.detect_by_content = true}},
+        {.detect_by_content = true, .open = true, .translate_html = true}},
 
     Row{FileType::mpeg4_video,
         "mp4"sv,
         mp4_extensions,
         mp4_mimetypes,
         FileCategory::video,
         DocumentType::unknown,
-        {.detect_by_content = true}},
+        {.detect_by_content = true, .open = true, .translate_html = true}},
     Row{FileType::quicktime_video,
         "mov"sv,
         mov_extensions,
         mov_mimetypes,
         FileCategory::video,
         DocumentType::unknown,
-        {.detect_by_content = true}},
+        {.detect_by_content = true, .open = true, .translate_html = true}},
     Row{FileType::third_generation_partnership_video,
         "3gp"sv,
         third_gpp_extensions,
         third_gpp_mimetypes,
         FileCategory::video,
         DocumentType::unknown,
-        {.detect_by_content = true}},
+        {.detect_by_content = true, .open = true, .translate_html = true}},
     Row{FileType::matroska_video,
         "mkv"sv,
         mkv_extensions,
         mkv_mimetypes,
         FileCategory::video,
         DocumentType::unknown,
-        {.detect_by_content = true}},
+        {.detect_by_content = true, .open = true, .translate_html = true}},
     Row{FileType::audio_video_interleave,
         "avi"sv,
         avi_extensions,
         avi_mimetypes,
         FileCategory::video,
         DocumentType::unknown,
-        {.detect_by_content = true}},
+        {.detect_by_content = true, .open = true, .translate_html = true}},
 };
 
 /// Finds the row whose list, selected by @p list, contains @p needle.

src/odr/internal/html/frontend.cpp

@@ -40,6 +40,12 @@ constexpr const char *text_css = R"css(
 [contenteditable]:focus{outline:none}
 )css";
 
+constexpr const char *media_css = R"css(
+.odr-media{display:flex;align-items:center;justify-content:center;margin:0;min-height:100vh;background:#000}
+.odr-media video{max-width:100%;max-height:100vh}
+.odr-media audio{width:100%;max-width:40rem;margin:0 1rem}
+)css";
+
 constexpr const char *document_js = R"js(
 (function () {
   "use strict";
@@ -584,6 +590,8 @@ void html::write_spreadsheet_style(HtmlWriter &out) {
 
 void html::write_text_style(HtmlWriter &out) { write_style(out, text_css); }
 
+void html::write_media_style(HtmlWriter &out) { write_style(out, media_css); }
+
 void html::write_document_script(HtmlWriter &out) {
   write_script(out, document_js);
 }

src/odr/internal/html/frontend.hpp

@@ -10,6 +10,7 @@ void write_document_style(HtmlWriter &out);
 /// Written in addition to the document style.
 void write_spreadsheet_style(HtmlWriter &out);
 void write_text_style(HtmlWriter &out);
+void write_media_style(HtmlWriter &out);
 
 /// The `odr` object a document view exposes to its host: `generateDiff()`,
 /// `search()`, `searchNext()`, `searchPrevious()`, `resetSearch()`.