feat(Chat, MarkdownStream): Support Shiny UI inside of message content (#1868)

Author: cpsievert

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [UNRELEASED]
 
+## New features
+
+* Both `ui.Chat()` and `ui.MarkdownStream()` now support arbirary Shiny UI elements inside of messages. This allows for gathering input from the user (e.g., `ui.input_select()`), displaying of rich output (e.g., `render.DataGrid()`), and more. (#1868)
+
 ### Changes
 
 * Express mode's `app_opts()` requires all arguments to be keyword-only. If you are using positional arguments, you will need to update your code. (#1895)

js/chat/chat.ts

@@ -5,9 +5,12 @@ import { property } from "lit/decorators.js";
 import {
   LightElement,
   createElement,
+  renderDependencies,
   showShinyClientMessage,
 } from "../utils/_utils";
 
+import type { HtmlDep } from "../utils/_utils";
+
 type ContentType = "markdown" | "html" | "text";
 
 type Message = {
@@ -18,10 +21,13 @@ type Message = {
   icon?: string;
   operation: "append" | null;
 };
+
 type ShinyChatMessage = {
   id: string;
   handler: string;
-  obj: Message;
+  // Message keys will create custom element attributes, but html_deps are handled
+  // separately
+  obj: (Message & { html_deps?: HtmlDep[] }) | null;
 };
 
 type UpdateUserInput = {
@@ -59,7 +65,8 @@ const ICONS = {
 
 class ChatMessage extends LightElement {
   @property() content = "...";
-  @property() content_type: ContentType = "markdown";
+  @property({ attribute: "content-type" }) contentType: ContentType =
+    "markdown";
   @property({ type: Boolean, reflect: true }) streaming = false;
   @property() icon = "";
 
@@ -72,7 +79,7 @@ class ChatMessage extends LightElement {
       <div class="message-icon">${unsafeHTML(icon)}</div>
       <shiny-markdown-stream
         content=${this.content}
-        content-type=${this.content_type}
+        content-type=${this.contentType}
         ?streaming=${this.streaming}
         auto-scroll
         .onContentChange=${this.#onContentChange.bind(this)}
@@ -529,7 +536,11 @@ customElements.define(CHAT_CONTAINER_TAG, ChatContainer);
 
 window.Shiny.addCustomMessageHandler(
   "shinyChatMessage",
-  function (message: ShinyChatMessage) {
+  async function (message: ShinyChatMessage) {
+    if (message.obj?.html_deps) {
+      await renderDependencies(message.obj.html_deps);
+    }
+
     const evt = new CustomEvent(message.handler, {
       detail: message.obj,
     });

js/markdown-stream/markdown-stream.ts

@@ -3,23 +3,28 @@ import { unsafeHTML } from "lit-html/directives/unsafe-html.js";
 import { property } from "lit/decorators.js";
 
 import ClipboardJS from "clipboard";
-import { sanitize } from "dompurify";
 import hljs from "highlight.js/lib/common";
 import { Renderer, parse } from "marked";
 
 import {
   LightElement,
   createElement,
   createSVGIcon,
+  renderDependencies,
+  sanitizeHTML,
   showShinyClientMessage,
+  throttle,
 } from "../utils/_utils";
 
+import type { HtmlDep } from "../utils/_utils";
+
 type ContentType = "markdown" | "semi-markdown" | "html" | "text";
 
 type ContentMessage = {
   id: string;
   content: string;
   operation: "append" | "replace";
+  html_deps?: HtmlDep[];
 };
 
 type IsStreamingMessage = {
@@ -59,11 +64,11 @@ const markedEscapeOpts = { renderer: rendererEscapeHTML };
 
 function contentToHTML(content: string, content_type: ContentType) {
   if (content_type === "markdown") {
-    return unsafeHTML(sanitize(parse(content) as string));
+    return unsafeHTML(sanitizeHTML(parse(content) as string));
   } else if (content_type === "semi-markdown") {
-    return unsafeHTML(sanitize(parse(content, markedEscapeOpts) as string));
+    return unsafeHTML(sanitizeHTML(parse(content, markedEscapeOpts) as string));
   } else if (content_type === "html") {
-    return unsafeHTML(sanitize(content));
+    return unsafeHTML(sanitizeHTML(content));
   } else if (content_type === "text") {
     return content;
   } else {
@@ -94,6 +99,8 @@ class MarkdownElement extends LightElement {
   protected willUpdate(changedProperties: PropertyValues): void {
     if (changedProperties.has("content")) {
       this.#isContentBeingAdded = true;
+
+      MarkdownElement.#doUnBind(this);
     }
     super.willUpdate(changedProperties);
   }
@@ -106,7 +113,14 @@ class MarkdownElement extends LightElement {
       } catch (error) {
         console.warn("Failed to highlight code:", error);
       }
-      if (this.streaming) this.#appendStreamingDot();
+
+      // Render Shiny HTML dependencies and bind inputs/outputs
+      if (this.streaming) {
+        this.#appendStreamingDot();
+        MarkdownElement._throttledBind(this);
+      } else {
+        MarkdownElement.#doBind(this);
+      }
 
       // Update scrollable element after content has been added
       this.#updateScrollableElement();
@@ -148,6 +162,47 @@ class MarkdownElement extends LightElement {
     this.querySelector(`svg.${SVG_DOT_CLASS}`)?.remove();
   }
 
+  static async #doUnBind(el: HTMLElement): Promise<void> {
+    if (!window?.Shiny?.unbindAll) return;
+
+    try {
+      window.Shiny.unbindAll(el);
+    } catch (err) {
+      showShinyClientMessage({
+        status: "error",
+        message: `Failed to unbind Shiny inputs/outputs: ${err}`,
+      });
+    }
+  }
+
+  static async #doBind(el: HTMLElement): Promise<void> {
+    if (!window?.Shiny?.initializeInputs) return;
+    if (!window?.Shiny?.bindAll) return;
+
+    try {
+      window.Shiny.initializeInputs(el);
+    } catch (err) {
+      showShinyClientMessage({
+        status: "error",
+        message: `Failed to initialize Shiny inputs: ${err}`,
+      });
+    }
+
+    try {
+      await window.Shiny.bindAll(el);
+    } catch (err) {
+      showShinyClientMessage({
+        status: "error",
+        message: `Failed to bind Shiny inputs/outputs: ${err}`,
+      });
+    }
+  }
+
+  @throttle(200)
+  private static async _throttledBind(el: HTMLElement): Promise<void> {
+    await this.#doBind(el);
+  }
+
   #highlightAndCodeCopy(): void {
     const el = this.querySelector("pre code");
     if (!el) return;
@@ -244,7 +299,9 @@ if (!customElements.get("shiny-markdown-stream")) {
   customElements.define("shiny-markdown-stream", MarkdownElement);
 }
 
-function handleMessage(message: ContentMessage | IsStreamingMessage): void {
+async function handleMessage(
+  message: ContentMessage | IsStreamingMessage
+): Promise<void> {
   const el = document.getElementById(message.id) as MarkdownElement;
 
   if (!el) {
@@ -262,6 +319,10 @@ function handleMessage(message: ContentMessage | IsStreamingMessage): void {
     return;
   }
 
+  if (message.html_deps) {
+    await renderDependencies(message.html_deps);
+  }
+
   if (message.operation === "replace") {
     el.setAttribute("content", message.content);
   } else if (message.operation === "append") {

js/utils/_utils.ts

@@ -1,5 +1,8 @@
+import DOMPurify from "dompurify";
 import { LitElement } from "lit";
 
+import type { HtmlDep } from "rstudio-shiny/srcts/types/src/shiny/render";
+
 ////////////////////////////////////////////////
 // Lit helpers
 ////////////////////////////////////////////////
@@ -10,7 +13,9 @@ function createElement(
 ): HTMLElement {
   const el = document.createElement(tag_name);
   for (const [key, value] of Object.entries(attrs)) {
-    if (value !== null) el.setAttribute(key, value);
+    // Replace _ with - in attribute names
+    const attrName = key.replace(/_/g, "-");
+    if (value !== null) el.setAttribute(attrName, value);
   }
   return el;
 }
@@ -49,4 +54,90 @@ function showShinyClientMessage({
   );
 }
 
-export { LightElement, createElement, createSVGIcon, showShinyClientMessage, };
+async function renderDependencies(deps: HtmlDep[]): Promise<void> {
+  if (!window.Shiny) return;
+  if (!deps) return;
+
+  try {
+    await window.Shiny.renderDependenciesAsync(deps);
+  } catch (renderError) {
+    showShinyClientMessage({
+      status: "error",
+      message: `Failed to render HTML dependencies: ${renderError}`,
+    });
+  }
+}
+
+////////////////////////////////////////////////
+// General helpers
+////////////////////////////////////////////////
+
+function sanitizeHTML(html: string): string {
+  return sanitizer.sanitize(html, {
+    // Sanitize scripts manually (see below)
+    ADD_TAGS: ["script"],
+    // Allow any (defined) custom element
+    CUSTOM_ELEMENT_HANDLING: {
+      tagNameCheck: (tagName) => {
+        return window.customElements.get(tagName) !== undefined;
+      },
+      attributeNameCheck: (attr) => true,
+      allowCustomizedBuiltInElements: true,
+    },
+  });
+}
+
+// Allow htmlwidgets' script tags through the sanitizer
+// by allowing `<script type="application/json" data-for="*"`,
+// which every widget should follow, and seems generally safe.
+const sanitizer = DOMPurify();
+sanitizer.addHook("uponSanitizeElement", (node, data) => {
+  if (node.nodeName && node.nodeName === "SCRIPT") {
+    const isOK =
+      node.getAttribute("type") === "application/json" &&
+      node.getAttribute("data-for") !== null;
+
+    data.allowedTags["script"] = isOK;
+  }
+});
+
+/**
+ * Creates a throttle decorator that ensures the decorated method isn't called more
+ * frequently than the specified delay
+ * @param delay The minimum time (in ms) that must pass between calls
+ */
+export function throttle(delay: number) {
+  /* eslint-disable @typescript-eslint/no-explicit-any */
+  return function (
+    _target: any,
+    _propertyKey: string,
+    descriptor: PropertyDescriptor
+  ) {
+    const originalMethod = descriptor.value;
+    let timeout: number | undefined;
+
+    descriptor.value = function (...args: any[]) {
+      if (timeout) {
+        window.clearTimeout(timeout);
+      }
+
+      timeout = window.setTimeout(() => {
+        originalMethod.apply(this, args);
+        timeout = undefined;
+      }, delay);
+    };
+
+    return descriptor;
+  };
+}
+
+export {
+  LightElement,
+  createElement,
+  createSVGIcon,
+  renderDependencies,
+  sanitizeHTML,
+  showShinyClientMessage,
+};
+
+export type { HtmlDep };

shiny/express/ui/_hold.py

@@ -4,7 +4,7 @@
 from types import TracebackType
 from typing import Callable, Optional, Type, TypeVar
 
-from htmltools import wrap_displayhook_handler
+from htmltools import TagList, wrap_displayhook_handler
 
 from ..._docstring import no_example
 from ..._typing_extensions import ParamSpec
@@ -46,9 +46,9 @@ def hold() -> HoldContextManager:
 
 class HoldContextManager:
     def __init__(self):
-        self.content: list[object] = list()
+        self.content = TagList()
 
-    def __enter__(self) -> list[object]:
+    def __enter__(self) -> TagList:
         self.prev_displayhook = sys.displayhook
         sys.displayhook = wrap_displayhook_handler(
             self.content.append  # pyright: ignore[reportArgumentType]