diff --git a/docs/backends.rst b/docs/backends.rst index 151d61fd..632c5ae3 100644 --- a/docs/backends.rst +++ b/docs/backends.rst @@ -20,16 +20,6 @@ The table below gives an overview of the names in the different ``rendercanvas`` | ``RenderCanvas`` (alias) | ``loop`` (an ``AsyncioLoop``) - | A lightweight backend. - * - ``jupyter`` - - | ``JupyterRenderCanvas`` - | ``RenderCanvas`` (alias) - | ``loop`` (an ``AsyncioLoop``) - - | Integrate in Jupyter notebook / lab. - * - ``offscreen`` - - | ``OffscreenRenderCanvas`` - | ``RenderCanvas`` (alias) - | ``loop`` (a ``StubLoop``) - - | For offscreen rendering. * - ``qt`` - | ``QRenderCanvas`` (toplevel) | ``RenderCanvas`` (alias) @@ -46,6 +36,21 @@ The table below gives an overview of the names in the different ``rendercanvas`` | ``loop`` - | Create a standalone canvas using wx, or | integrate a render canvas in a wx application. + * - ``offscreen`` + - | ``OffscreenRenderCanvas`` + | ``RenderCanvas`` (alias) + | ``loop`` (a ``StubLoop``) + - | For offscreen rendering. + * - ``anywidget`` + - | ``AnywidgetRenderCanvas`` + | ``RenderCanvas`` (alias) + | ``loop`` (an ``AsyncioLoop``) + - | Integrate in notebooks using anywidget. + * - ``jupyter`` + - | ``JupyterRenderCanvas`` + | ``RenderCanvas`` (alias) + | ``loop`` (an ``AsyncioLoop``) + - | Integrate in notebooks via ``jupyter_rfb`` (deprecated). * - ``pyodide`` - | ``PyodideRenderCanvas`` (toplevel) | ``RenderCanvas`` (alias) @@ -53,7 +58,6 @@ The table below gives an overview of the names in the different ``rendercanvas`` - | Backend when Python is running in the browser, | via Pyodide or PyScript. - There are also three loop-backends. These are mainly intended for use with the glfw backend: .. list-table:: @@ -262,17 +266,17 @@ object, but in some cases it's convenient to do so with a canvas-like API. array = canvas.draw() # numpy array with shape (400, 500, 4) -Support for Jupyter lab and notebook ------------------------------------- +Support for notebooks +--------------------- + +With the ``anywidget`` backend, RenderCanvas can be used in Jupyter lab, Jupyter notebook, VSCode, Google Colab, Marimo notebooks, and anywhere else where ``anywidget`` is supported. +When the ``auto`` backend is used in a notebook, the ``anywidget`` is selected automatically. -RenderCanvas can be used in Jupyter lab and the Jupyter notebook. This canvas -is based on `jupyter_rfb `_, an ipywidget -subclass implementing a remote frame-buffer. There are also some `wgpu examples `_. +The ``jupyter`` backend is the previous backend to provide notebook support, which is based on ``jupyter_rfb``. It's kept for backwards compatibility. .. code-block:: py - # from rendercanvas.jupyter import RenderCanvas # Direct approach - from rendercanvas.auto import RenderCanvas # also works, because rendercanvas detects Jupyter + from rendercanvas.auto import RenderCanvas # uses anywidget when in a notebook canvas = RenderCanvas() @@ -281,6 +285,10 @@ subclass implementing a remote frame-buffer. There are also some `wgpu examples canvas # Use as cell output +.. autoclass:: rendercanvas.anywidget.AnywidgetRenderCanvas + :members: + + Support for Pyodide ------------------- @@ -417,9 +425,8 @@ Many interactive environments have some sort of GUI support, allowing the repl to stay active (i.e. you can run new code), while the GUI windows is also alive. In rendercanvas we try to select the GUI that matches the current environment. -On ``jupyter notebook`` and ``jupyter lab`` the jupyter backend (i.e. -``jupyter_rfb``) is normally selected. When you are using ``%gui qt``, rendercanvas will -honor that and use Qt instead. +In a notebook (e.g. jupyter) one of the notebook capable backends (``anywidget`` or ``jupyter``) is selected. +When you are using ``%gui qt``, rendercanvas will honor that and use Qt instead. On ``jupyter console`` and ``qtconsole``, the kernel is the same as in ``jupyter notebook``, making it (about) impossible to tell that we cannot actually use diff --git a/docs/conf.py b/docs/conf.py index 00bca06c..27d3a553 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,6 +22,32 @@ os.environ["RENDERCANVAS_FORCE_OFFSCREEN"] = "true" +class FakeTrait: + def __init__(self, *args, **kwargs): + pass + + def tag(self, *args, **kwargs): + pass + + +class FakeModule: + # from anywidget import AnyWidget + AnyWidget = object + # from traitlets import ... + Dict = FakeTrait + Unicode = FakeTrait + Int = FakeTrait + Bool = FakeTrait + # from IPython.display import ... + HTML = lambda *a, **kw: None + display = lambda *a, **kw: None + + +sys.modules["anywidget"] = FakeModule +sys.modules["traitlets"] = FakeModule +sys.modules["IPython.display"] = FakeModule + + # Load wgpu so autodoc can query docstrings import rendercanvas # noqa: E402 import rendercanvas.stub # noqa: E402 - we use the stub backend to generate docs diff --git a/examples/rendercanvas.ipynb b/examples/rendercanvas.ipynb index 7ef3679b..92bc470f 100644 --- a/examples/rendercanvas.ipynb +++ b/examples/rendercanvas.ipynb @@ -23,8 +23,10 @@ "metadata": {}, "outputs": [], "source": [ + "# from rendercanvas.anywidget import RenderCanvas\n", + "# from rendercanvas.jupyter import RenderCanvas\n", + "from rendercanvas.auto import RenderCanvas # Defaults to anywidget when in a notebook\n", "from rendercanvas.utils.cube import setup_drawing_sync\n", - "from rendercanvas.jupyter import RenderCanvas\n", "\n", "canvas = RenderCanvas(update_mode=\"continuous\")\n", "draw_frame = setup_drawing_sync(canvas)\n", @@ -41,7 +43,29 @@ "outputs": [], "source": [ "# Set title to non-empty string to show the title bar\n", - "canvas.set_title(\"Rotating cube\")" + "canvas.set_title(\"Rotating cube\")\n", + "canvas.set_minimizable(True)\n", + "canvas.set_closable(True)" + ] + }, + { + "cell_type": "markdown", + "id": "da1ea8e0-e7bd-41a1-b2f5-0084d0d48888", + "metadata": {}, + "source": [ + "## Snapshots\n", + "\n", + "It is also possible to create snapshots. This is also possible without displaying the interactive canvas itself. The benefit of snapshots is that the output remains visible when the notebook is shown in a static viewer. This makes them useful for e.g. tutorials." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16a4a4fe-4c64-4c07-8da9-577a1a010fbe", + "metadata": {}, + "outputs": [], + "source": [ + "canvas.snapshot()" ] }, { @@ -72,14 +96,6 @@ "\n", "out" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "65f86895-9e9a-4c22-8a54-919bd70fd80b", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/pyproject.toml b/pyproject.toml index 9fce626a..166070db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,23 +18,24 @@ keywords = [ "jupyter", ] requires-python = ">= 3.10" -dependencies = ['numpy'] # Numpy is the only hard dependency +dependencies = ["numpy"] # Numpy is the only hard dependency + [project.optional-dependencies] # For users -jupyter = ["jupyter_rfb>=0.4.2"] +jupyter = ["jupyter_rfb>=0.4.2", "simplejpeg; implementation_name != 'pypy'"] +notebook = ["anywidget", "simplejpeg; implementation_name != 'pypy'"] glfw = ["glfw>=1.9"] # For devs / ci lint = ["ruff", "pre-commit"] examples = ["flit", "numpy", "wgpu", "glfw", "pyside6", "imageio", "pytest"] -docs = [ - "flit", - "sphinx>7.2", - "sphinx_rtd_theme", - "sphinx-gallery", - "numpy", +docs = ["flit", "sphinx>7.2", "sphinx_rtd_theme", "sphinx-gallery", "wgpu"] +tests = [ + "pytest", "wgpu", + "glfw", + "trio", + "simplejpeg; implementation_name != 'pypy'", ] -tests = ["pytest", "numpy", "wgpu", "glfw", "trio"] dev = ["rendercanvas[lint,tests,examples,docs]"] [project.entry-points."pyinstaller40"] diff --git a/rendercanvas/anywidget.py b/rendercanvas/anywidget.py new file mode 100644 index 00000000..bee0bb08 --- /dev/null +++ b/rendercanvas/anywidget.py @@ -0,0 +1,409 @@ +""" +A backend based on anywidget, supporting canvases inside a large variety of notebooks and similar browser-like environments. +""" + +__all__ = ["AnywidgetRenderCanvas", "RenderCanvas", "loop"] + +import time +import asyncio +from base64 import encodebytes +from importlib.resources import files as resource_files + +from .base import BaseCanvasGroup, BaseRenderCanvas, logger +from .asyncio import loop +from .core.encoders import encode_array, CAN_JPEG + +import numpy as np +import anywidget +from IPython.display import display, HTML +from traitlets import Bool, Dict, Int, Unicode + + +def _load_js_and_css(): + js = "" + for fname in ["renderview.js", "renderview-afm.js"]: + js_path = resource_files("rendercanvas.core").joinpath(fname) + js += js_path.read_text() + "\n\n" + + css_path = resource_files("rendercanvas.core").joinpath("renderview.css") + + return js, css_path.read_text() + + +JS, CSS = _load_js_and_css() + + +class AnywidgetCanvasGroup(BaseCanvasGroup): + pass + + +class AnywidgetRenderCanvas(BaseRenderCanvas, anywidget.AnyWidget): + """An anywidget canvas to use in notebooks (e.g. jupyter, marimo, VSCode, etc.). + + This is an AnyWidget subclass, so you can easily combine it with other widgets. + """ + + # This class uses some '_rfb_' prefixes to avoid name clashes with super and sub classes. + # This specific prefix was inherited from jupyter_rfb, and we decided to keep it as is. + + _rc_canvas_group = AnywidgetCanvasGroup(loop) + + _esm = JS + _css = CSS + + # Client -> server + _frame_feedback = Dict({}).tag(sync=True) + _has_visible_views = Bool(False).tag(sync=True) + # Server -> client + _css_width = Unicode("500px").tag(sync=True) + _css_height = Unicode("300px").tag(sync=True) + _resizable = Bool(True).tag(sync=True) + _is_minimizable = Bool(False).tag(sync=True) + _is_closable = Bool(False).tag(sync=True) + _has_titlebar = Bool(False).tag(sync=True) + _title = Unicode("").tag(sync=True) + _cursor = Unicode("default").tag(sync=True) + # Server only + _max_buffered_frames = Int(2, min=1) + _quality = Int(80, min=1, max=100) + + def __init__(self, *args, **kwargs): + # This backend's default title is empty + kwargs["title"] = kwargs.get("title", "") + + super().__init__(*args, **kwargs) + + self._is_closed = False + + self._rfb_draw_requested = False + self._rfb_frame_index = 0 + self._rfb_last_confirmed_index = 0 + self._rfb_warned_png = False + self._rfb_lossless_draw_info = None + self._rfb_last_resize_event = None + self._rfb_pending_snapshot_display = None + + self._last_set_logical_size = 100, 100 + self._use_websocket = True # Could be a prop, private for now + + self.reset_stats() + self.on_msg(self._rfb_handle_msg) + self.observe( + self._rfb_schedule_maybe_draw, + names=["_frame_feedback", "_has_visible_views"], + ) + + # Set size, title, etc. + self._final_canvas_init() + + def snapshot(self, *, pixel_ratio=None): + """Render a frame and include the resulting image in the output. + + An initial placeholder output is produced, which is replaced by an html + ```` as soon as the next frame is rendered. + + If the widget is not displayed yet, a resize event is emitted to mimic a widget + size. The ``pixel_ratio`` argument is then used to calculate the physical size. + """ + if self._rfb_last_resize_event is None: + w, h = self._last_set_logical_size + r = float(pixel_ratio) if pixel_ratio is not None else 1.0 + pw, ph = int(w * r), (h * r) + event = { + "type": "resize", + "width": pw / r, + "height": ph / r, + "pwidth": pw, + "pheight": ph, + "ratio": r, + "timestamp": 0, + } + self._rfb_handle_msg(self, event, []) + + self._rfb_pending_snapshot_display = display( + HTML( + "
pending screenshot ..." + ), + display_id=True, + ) + self.request_draw() + + def _replace_snapshot(self, array): + pending_display = self._rfb_pending_snapshot_display + self._rfb_pending_snapshot_display = None + + event = self._rfb_last_resize_event or {} + w = event.get("width", array.shape[1]) + h = event.get("height", array.shape[0]) + + mimetype, data = encode_array(array, 70) + src = f"data:image/{mimetype};base64," + encodebytes(data).decode() + html = f"" + + pending_display.update(HTML(html)) + + def _rfb_handle_msg(self, widget, content, buffers): + """Receive custom messages and filter our events.""" + event_type = content.get("type") + + if event_type is not None: + event = content + + if event_type == "resize": + self._last_event = event + self._size_info.set_physical_size( + event["pwidth"], event["pheight"], event["ratio"] + ) + elif event_type == "close": + self.close() + else: + # Compatibility between new renderview event spec and current rendercanvas/pygfx events + event["event_type"] = event.pop("type") + event["time_stamp"] = event.pop("timestamp") + # Turn lists into tuples (js/json does not have tuples) + if "buttons" in event: + event["buttons"] = tuple(event["buttons"]) + if "modifiers" in event: + event["modifiers"] = tuple(event["modifiers"]) + self.submit_event(event) + + def _rfb_schedule_maybe_draw(self, *args): + """Schedule _maybe_draw() to be called in a fresh event loop iteration.""" + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return + loop.call_soon(self._rfb_maybe_draw) + + def _rfb_maybe_draw(self): + """Perform a draw, if we can and should.""" + feedback = self._frame_feedback + # Update stats + self._rfb_update_stats(feedback) + # Determine whether we should perform a draw: a draw was requested, and + # the client is ready for a new frame, and the client widget is visible. + frames_in_flight = self._rfb_frame_index - feedback.get("index", 0) + should_draw = ( + self._rfb_draw_requested + and frames_in_flight < self._max_buffered_frames + and (self._has_visible_views or self._rfb_pending_snapshot_display) + ) + # Do the draw if we should. + if should_draw: + self._rfb_draw_requested = False + self._time_to_draw() # -> _rc_present_bitmap -> _rfb_send_frame + + def _rfb_schedule_lossless_draw(self, array, delay=0.3): + self._rfb_cancel_lossless_draw() + loop = asyncio.get_running_loop() + handle = loop.call_later(delay, self._rfb_lossless_draw) + self._rfb_lossless_draw_info = array, handle + + def _rfb_cancel_lossless_draw(self): + if self._rfb_lossless_draw_info: + _, handle = self._rfb_lossless_draw_info + self._rfb_lossless_draw_info = None + handle.cancel() + + def _rfb_lossless_draw(self): + array, _ = self._rfb_lossless_draw_info + self._rfb_send_frame(array, True) + + def _rfb_send_frame(self, array, is_lossless_redraw=False): + """Actually send a frame over to the client.""" + # For considerations about performance, + # see https://github.com/vispy/jupyter_rfb/issues/3 + + quality = 100 if is_lossless_redraw else self._quality + + self._rfb_frame_index += 1 + timestamp = time.time() + + # Turn array into a based64-encoded JPEG or PNG + t1 = time.perf_counter() + mimetype, data = encode_array(array, quality) + if self._use_websocket: + datas = [data] + data_b64 = None + else: + datas = [] + data_b64 = f"data:{mimetype};base64," + encodebytes(data).decode() + t2 = time.perf_counter() + + if "jpeg" in mimetype: + self._rfb_schedule_lossless_draw(array) + else: + self._rfb_cancel_lossless_draw() + # Issue png warning? + if quality < 100 and not CAN_JPEG and not self._rfb_warned_png: + self._rfb_warned_png = True + logger.warning( + "No JPEG encoder found, using PNG instead. Install simplejpeg for better performance." + ) + + if is_lossless_redraw: + # No stats, also not on the confirmation of this frame + self._rfb_last_confirmed_index = self._rfb_frame_index + else: + # Stats + self._rfb_stats["img_encoding_sum"] += t2 - t1 + self._rfb_stats["sent_frames"] += 1 + if self._rfb_stats["start_time"] <= 0: # Start measuring + self._rfb_stats["start_time"] = timestamp + self._rfb_last_confirmed_index = self._rfb_frame_index - 1 + + # Reload the output if we did not have a frame when the widget was first loaded + if self._rfb_pending_snapshot_display is not None: + self._replace_snapshot(array) + + # Compose message and send + msg = dict( + type="framebufferdata", + mimetype=mimetype, + data_b64=data_b64, + index=self._rfb_frame_index, + timestamp=timestamp, + ) + self.send(msg, datas) + + # ----- related to stats + + def reset_stats(self): + """Restart measuring statistics from the next sent frame.""" + self._rfb_stats = { + "start_time": 0, + "last_time": 1, + "sent_frames": 0, + "confirmed_frames": 0, + "roundtrip_count": 0, + "roundtrip_sum": 0, + "delivery_sum": 0, + "img_encoding_sum": 0, + } + + def get_stats(self): + """Get the current stats since the last time ``.reset_stats()`` was called. + + Stats is a dict with the following fields: + + * *sent_frames*: the number of frames sent. + * *confirmed_frames*: number of frames confirmed by the client. + * *roundtrip*: average time for processing a frame, including receiver confirmation. + * *delivery*: average time for processing a frame until it's received by the client. + This measure assumes that the clock of the server and client are precisely synced. + * *img_encoding*: the average time spent on encoding the array into an image. + * *b64_encoding*: the average time spent on base64 encoding the data. + * *fps*: the average FPS, measured from the first frame sent since ``.reset_stats()`` + was called, until the last confirmed frame. + """ + d = self._rfb_stats + roundtrip_count_div = d["roundtrip_count"] or 1 + sent_frames_div = d["sent_frames"] or 1 + fps_div = (d["last_time"] - d["start_time"]) or 0.001 + return { + "sent_frames": d["sent_frames"], + "confirmed_frames": d["confirmed_frames"], + "roundtrip": d["roundtrip_sum"] / roundtrip_count_div, + "delivery": d["delivery_sum"] / roundtrip_count_div, + "img_encoding": d["img_encoding_sum"] / sent_frames_div, + "fps": d["confirmed_frames"] / fps_div, + } + + def _rfb_update_stats(self, feedback): + """Update the stats when a new frame feedback has arrived.""" + last_index = feedback.get("index", 0) + if last_index > self._rfb_last_confirmed_index: + timestamp = feedback["timestamp"] + nframes = last_index - self._rfb_last_confirmed_index + self._rfb_last_confirmed_index = last_index + self._rfb_stats["confirmed_frames"] += nframes + self._rfb_stats["roundtrip_count"] += 1 + self._rfb_stats["roundtrip_sum"] += time.time() - timestamp + self._rfb_stats["delivery_sum"] += feedback["localtime"] - timestamp + self._rfb_stats["last_time"] = time.time() + + # --- the API to be a rendercanvas backend + + def _rc_gui_poll(self): + pass + + def _rc_get_present_info(self, present_methods): + # Only allow simple format for now. srgb is assumed. + if "bitmap" in present_methods: + return { + "method": "bitmap", + "formats": ["rgba-u8"], + } + else: + return None # raises error + + def _rc_request_draw(self): + # Technically, _maybe_draw() may not perform a draw if there are too + # many frames in-flight. But in this case, we'll eventually get + # new frame_feedback, which will then trigger a draw. + if not self._rfb_draw_requested: + self._rfb_draw_requested = True + self._rfb_cancel_lossless_draw() + self._rfb_schedule_maybe_draw() + + def _rc_request_paint(self): + # We technically don't need to call _time_to_paint, because this backend only does bitmap mode. + # But in case the base backend will do something in _time_to_paint later, we behave nice. + loop = self._rc_canvas_group.get_loop() + loop.call_soon(self._time_to_paint) + + def _rc_force_paint(self): + pass # works as-is via push_frame + + def _rc_present_bitmap(self, *, data, format, **kwargs): + assert format == "rgba-u8" + self._rfb_send_frame(np.asarray(data)) + + def _rc_set_logical_size(self, width, height): + self._last_set_logical_size = width, height + self._css_width = f"{width}px" + self._css_height = f"{height}px" + + def _rc_close(self): + anywidget.AnyWidget.close(self) + self._rfb_handle_msg(self, {"type": "close"}, []) + self._is_closed = True + + def _rc_get_closed(self): + return self._is_closed + + def _rc_set_title(self, title): + self._title = str(title) + self._has_titlebar = bool(title) + + def _rc_set_cursor(self, cursor): + self._cursor = cursor + + def set_css_width(self, css_width: str): + """Set the width of the canvas as a CSS string.""" + self._css_width = css_width + + def set_css_height(self, css_height: str): + """Set the height of the canvas as a CSS string.""" + self._css_height = css_height + + def set_resizable(self, resizable: bool): + """Set whether the canvas is manually resizable.""" + self._resizable = resizable + + def set_minimizable(self, minimizable: bool): + """Set whether the canvas is manually minimizable via a button in the titlebar. + + If all views of the canvas are hidden (out of view or minimized), its rendering is paused, + saving CPU cycles and battery. + """ + self._is_minimizable = minimizable + + def set_closable(self, closable: bool): + """Set whether the canvas is manually closable via a button in the titlebar.""" + self._is_closable = closable + + +# Make available under a common name +RenderCanvas = AnywidgetRenderCanvas +loop = loop diff --git a/rendercanvas/auto.py b/rendercanvas/auto.py index c9b386dd..04411efe 100644 --- a/rendercanvas/auto.py +++ b/rendercanvas/auto.py @@ -19,7 +19,7 @@ # Note that wx is not in here, because it does not (yet) fully implement base.BaseRenderCanvas -BACKEND_NAMES = ["glfw", "qt", "jupyter", "offscreen"] +BACKEND_NAMES = ["glfw", "qt", "anywidget", "jupyter", "offscreen"] def _load_backend(backend_name): @@ -28,6 +28,8 @@ def _load_backend(backend_name): from . import glfw as module elif backend_name == "qt": from . import qt as module + elif backend_name == "anywidget": + from . import anywidget as module elif backend_name == "jupyter": from . import jupyter as module elif backend_name == "wx": @@ -130,11 +132,11 @@ def get_env_var(*varnames): def backends_by_notebook(): - """Generate backend names that are appropriate for the current Jupyter session (if any).""" + """Generate backend names that are appropriate for the current notebook session (if any).""" # Detect Marimo: https://github.com/marimo-team/marimo/discussions/8865 if "marimo" in sys.modules and sys.modules["marimo"].running_in_notebook(): - yield "jupyter", "running on Marimo" + yield "anywidget", "running on Marimo" try: ip = get_ipython() # type: ignore @@ -161,7 +163,7 @@ def backends_by_notebook(): # elif "wx" in app.__class__.__name__.lower() == "wx": # yield "wx", "running on Jupyter with wx gui" - yield "jupyter", "running on Jupyter" + yield "anywidget", "running on Jupyter" def backends_by_imported_modules(): diff --git a/rendercanvas/core/encoders.py b/rendercanvas/core/encoders.py new file mode 100644 index 00000000..648ed8b4 --- /dev/null +++ b/rendercanvas/core/encoders.py @@ -0,0 +1,138 @@ +import io +import struct +import zlib + +import numpy as np + +try: + import simplejpeg +except ImportError: + simplejpeg = None + + +CAN_JPEG = simplejpeg is not None + + +def encode_array(array, quality: int = 75): + """Encode an image array to a compressed format. + + If the quality is 100, a PNG is returned. Otherwise, JPEG is + preferred and PNG is used as a fallback. Returns (mimetype, bytes). + """ + + if quality >= 100 or not CAN_JPEG: + # Drop alpha channel if it has one + if array.ndim == 3 and array.shape[2] == 4: + array = array[:, :, :3] + mimetype = "image/png" + data = encode_png(array) + else: + mimetype = "image/jpeg" + data = encode_jpeg(array, quality) + + return mimetype, data + + +def encode_jpeg(array, quality: int = 75): + """Encode an image array to bytes to the jpeg format. + + The image shape must be NxM, NxMx3, or NxMx4. + """ + + if simplejpeg is None: + raise RuntimeError("encode_jpeg() needs simplejpeg but it is not installed.") + if not (isinstance(array, np.ndarray) and array.dtype == "uint8"): + raise TypeError("encode_jpeg() requires an uint8 numpy array") + + # Fix and check shape and contiguity + if array.ndim == 2: + array = array.reshape(*array.shape, 1) + if array.ndim == 3 and array.shape[2] == 1: + colorspace = "GRAY" + colorsubsampling = "Gray" + elif array.ndim == 3 and array.shape[2] in (3, 4): + colorspace = "RGBA"[: array.shape[2]] + colorsubsampling = "444" # 420 does not seem to help the compression much + else: + raise ValueError( + f"encode_jpeg() expects an NxM, NxMx3, or NxMx4 array, but got {array.shape}" + ) + + # Make sure it is contiguous + array = np.ascontiguousarray(array) + + # Encode! + return simplejpeg.encode_jpeg( + array, + quality=quality, + colorspace=colorspace, + colorsubsampling=colorsubsampling, + fastdct=True, + ) + + +def encode_png(array, level: int = 6): + """Encode an image array to bytes to the png format. + + The image shape must be NxM, NxMx3, or NxMx4. + The written image is in RGB or RGBA format, with 8 bit precision, + zlib-compressed, without interlacing. + """ + + if not (isinstance(array, np.ndarray) and array.dtype == "uint8"): + raise TypeError("encode_png() requires an uint8 numpy array") + + # Fix and check shape and contiguity + if array.ndim == 3 and array.shape[2] == 1: + array = array.reshape(*array.shape[:2]) + if array.ndim == 2: + array3 = np.empty((*array.shape[:2], 3), np.uint8) + array3[..., 0] = array + array3[..., 1] = array + array3[..., 2] = array + array = array3 + elif array.ndim == 3 and array.shape[2] in (3, 4): + pass + else: + raise ValueError( + f"encode_png() expects an NxM, NxMx3, or NxMx4 array, but got {array.shape}" + ) + + # Get file object + f = io.BytesIO() + + def add_chunk(data, name): + name = name.encode("ASCII") + crc = zlib.crc32(data, zlib.crc32(name)) + f.write(struct.pack(">I", len(data))) + f.write(name) + f.write(data) + f.write(struct.pack(">I", crc & 0xFFFFFFFF)) + + # Write ... + + # Header + f.write(b"\x89PNG\x0d\x0a\x1a\x0a") + + # First chunk + h, w, c = array.shape + depth = 8 + ctyp = 0b0110 if c == 4 else 0b0010 + ihdr = struct.pack(">IIBBBBB", w, h, depth, ctyp, 0, 0, 0) + add_chunk(ihdr, "IHDR") + + # Chunk with pixels. Just one chunk, no fancy filters. + compressor = zlib.compressobj(level=level) + compressed_data = [] + for row_index in range(array.shape[0]): + row = np.ascontiguousarray(array[row_index]) + compressed_data.append(compressor.compress(b"\x00")) # prepend filter bytes + compressed_data.append(compressor.compress(row)) + compressed_data.append(compressor.flush()) + add_chunk(b"".join(compressed_data), "IDAT") + + # Closing chunk + add_chunk(b"", "IEND") + + f.flush() + return f.getvalue() diff --git a/rendercanvas/core/renderview-afm.js b/rendercanvas/core/renderview-afm.js new file mode 100644 index 00000000..79bc4646 --- /dev/null +++ b/rendercanvas/core/renderview-afm.js @@ -0,0 +1,268 @@ +/************************************************************************************************* + renderview-afm.js + + The Anywidget Frontend Module for renderview. + + *************************************************************************************************/ + +/* global BaseRenderView getTimestamp */ + +/** + * An object that represents the model(wrapping the anywidget model object), that can have multiple views. + */ +class RenderviewAnywidgetModel { + constructor (anymodel) { + this.anymodel = anymodel + this.views = [] + this._hasVisibleViews = false + + // Variables to store frames and the last frame + this._frames = [] + this._lastSrc = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR42mOor68HAAL+AX6E2KOJAAAAAElFTkSuQmCC' + this._lastFrame = { + src: this._lastSrc, + index: 0, + timestamp: 0 + } + + // Register callbacks + anymodel.on('msg:custom', (msg, buffers) => { + if (msg.type === 'framebufferdata') { + this._frames.push({ ...msg, buffers }) + this._request_animation_frame() + } + }) + // For traits we allow the public and private version, this means we can use the + // same JS code for multiple anywidget implementations (specifically rendercanvas.anywidget vs jupyter_rfb) + for (const prefix of ['', '_']) { + anymodel.on(`change:${prefix}css_width`, () => { + const cssWidth = anymodel.get(`${prefix}css_width`) + for (const view of this.views) { + view.setCssWidth(cssWidth) + } + }) + anymodel.on(`change:${prefix}css_height`, () => { + const cssHeight = anymodel.get(`${prefix}css_height`) + for (const view of this.views) { + view.setCssHeight(cssHeight) + } + }) + anymodel.on(`change:${prefix}resizable`, () => { + const resizable = anymodel.get(`${prefix}resizable`) + for (const view of this.views) { + view.setResizable(resizable) + } + }) + anymodel.on(`change:${prefix}is_minimizable`, () => { + const minimizable = anymodel.get(`${prefix}is_minimizable`) + for (const view of this.views) { + view.setMinimizable(minimizable) + } + }) + anymodel.on(`change:${prefix}is_closable`, () => { + const closable = anymodel.get(`${prefix}is_closable`) + for (const view of this.views) { + view.setClosable(closable) + } + }) + anymodel.on(`change:${prefix}has_titlebar`, () => { + const titlebar = anymodel.get(`${prefix}has_titlebar`) + for (const view of this.views) { + view.showTitlebar(titlebar) + } + }) + anymodel.on(`change:${prefix}title`, () => { + const title = anymodel.get(`${prefix}title`) + for (const view of this.views) { + view.setTitle(title) + } + }) + anymodel.on(`change:${prefix}cursor`, () => { + const cursor = anymodel.get(`${prefix}cursor`) + for (const view of this.views) { + view.setCursor(cursor) + } + }) + } + + // Start the animation loop + this._img_update_pending = false + this._request_animation_frame() + } + + close () { + URL.revokeObjectURL(this._lastSrc) + this._lastSrc = null + this._lastFrame = null + this._frames = [] + for (const view of this.views) { + view.close() + } + // This gets called when the model is closed and the comm is removed. Notify Py just in time! + const t = getTimestamp() + const event = { + type: 'close', + timestamp: t + } + this.onEvent(event) + } + + addView (view) { + this.views.push(view) + this.updateVisibility() + // Init attrs + const anymodel = this.anymodel + view.setCssWidth(anymodel.get('_css_width') ?? anymodel.get('css_width')) + view.setCssHeight(anymodel.get('_css_height') ?? anymodel.get('css_height')) + view.setResizable(anymodel.get('_resizable') ?? anymodel.get('resizable')) + view.showTitlebar(anymodel.get('_has_titlebar') ?? anymodel.get('has_titlebar')) + view.setTitle(anymodel.get('_title') ?? anymodel.get('title')) + view.setCursor(anymodel.get('_cursor') ?? anymodel.get('cursor')) + // Init view + if (this._lastSrc) { + view.viewElement.src = this._lastSrc + } + } + + removeView (view) { + this.views = this.views.filter(v => v !== view) + this.updateVisibility() + } + + updateVisibility () { + let visibleViewsCount = 0 + for (const view of this.views) { + if (view.isVisible) { visibleViewsCount += 1 } + } + const hasVisibleViews = visibleViewsCount > 0 + this.anymodel.set('_has_visible_views', hasVisibleViews) + this.anymodel.save_changes() + if (hasVisibleViews) { + this._request_animation_frame() + } + } + + _send_response () { + // Let Python know what we have at the frame. This prop is a dict, making it "atomic". + const frame = this._lastFrame + const frameFeedback = { index: frame.index, timestamp: frame.timestamp, localtime: Date.now() / 1000 } + this.anymodel.set('_frame_feedback', frameFeedback) + this.anymodel.save_changes() + } + + _request_animation_frame () { + // Request an animation frame. + // Before the anywidget refactor, we did this via a tiny delay, which supposedly made things more smooth, + // but it also increases the delay for a frame to hit the screen, and limits the max fps, so let's not do that. + if (!this._img_update_pending) { + this._img_update_pending = true + window.requestAnimationFrame(this._animate.bind(this)) + // window.setTimeout(window.requestAnimationFrame, 5, this._animate.bind(this)) // via a delay + } + } + + _animate () { + this._img_update_pending = false + if (this._frames.length === 0) { return }; + + // Pick the oldest frame from the stack, and get its source + const frame = this._frames.shift() + let newSrc + if (frame.buffers.length > 0) { + const blob = new Blob([frame.buffers[0].buffer], { type: frame.mimetype }) + newSrc = URL.createObjectURL(blob) + } else { + newSrc = frame.data_b64 + } + + // Revoke last objectURL + URL.revokeObjectURL(this._lastSrc) + this._lastSrc = newSrc + + // Update the image sources + for (const view of this.views) { + view.viewElement.src = newSrc + view.viewElement.onload = this._request_animation_frame.bind(this) + } + + // Let the server know we processed the image (even if it's not shown yet) + this._lastFrame = frame + this._send_response() + } + + onEvent (event) { + try { + this.anymodel.send(event) + } catch { } // probably attempt to send when widget is closed + } +} + +/** + * View to show the anywidget output and observe events, based on renderview.js. + */ +class AnywidgetRenderView extends BaseRenderView { + constructor (model, containerElement) { + // Create the wrapper element + const wrapperElement = document.createElement('div') + wrapperElement.classList.add('renderview-wrapper') + wrapperElement.classList.add('is-resizable') + // wrapperElement.classList.add('has-titlebar') -> not by default + containerElement.appendChild(wrapperElement) + + // Create img element + const viewElement = document.createElement('img') + viewElement.decoding = 'sync' + viewElement.loading = 'eager' + viewElement.style.touchAction = 'none' // prevent default pan/zoom behavior + viewElement.ondragstart = () => false // prevent browser's built-in image drag + + // Call super + super(viewElement, wrapperElement) + this.setThrottle(20) // 20ms -> max 50 move/wheel events per second + + // Connect to the model, it will initialize it with the current size and frame-data + this.model = model + this.model.addView(this) + } + + close () { + super.close() + this.model.removeView(this) + } + + onEvent (event) { + if (event.type === 'resize') { + // Note that there can be multiple views that can possibly be individually resized. + // TODO: keep logical size in check between different views? + if (event.width * event.height > 0) { + this.model.onEvent(event) + } + } else if (event.type === 'close') { + // we don't close when one view closes, only when the widget closes + } else if (event.type === 'show') { + this.isVisible = true + this.model.updateVisibility() + } else if (event.type === 'hide') { + this.isVisible = false + this.model.updateVisibility() + } else { + this.model.onEvent(event) + } + } +} + +// anywidget lifecycle export +export default () => { + let model + return { + initialize (ctx) { + model = new RenderviewAnywidgetModel(ctx.model) + // window.model = model // debug + return () => { model.close() } + }, + render (ctx) { + const view = new AnywidgetRenderView(model, ctx.el) + return () => { view.close() } + } + } +} diff --git a/rendercanvas/jupyter.py b/rendercanvas/jupyter.py index deea7084..72e01821 100644 --- a/rendercanvas/jupyter.py +++ b/rendercanvas/jupyter.py @@ -30,7 +30,7 @@ class JupyterRenderCanvas(BaseRenderCanvas, RemoteFrameBuffer): _event_compatibility = 1 def __init__(self, *args, **kwargs): - # The jupyter backend's default title is empty + # This backend's default title is empty kwargs["title"] = kwargs.get("title", "") super().__init__(*args, **kwargs) diff --git a/tests/test_backends.py b/tests/test_backends.py index b8552f60..4d0bb280 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -256,6 +256,14 @@ def test_pyodide_module(): assert canvas_class.name == "PyodideRenderCanvas" +def test_anywidget_module(): + m = Module("anywidget") + + canvas_class = m.get_canvas_class() + m.check_canvas(canvas_class) + assert canvas_class.name == "AnywidgetRenderCanvas" + + def test_jupyter_module(): m = Module("jupyter") diff --git a/tests/test_encoders.py b/tests/test_encoders.py new file mode 100644 index 00000000..7e80a431 --- /dev/null +++ b/tests/test_encoders.py @@ -0,0 +1,142 @@ +"""Test the jpeg and png encoders for the remote backends.""" + +from rendercanvas.core import encoders +from rendercanvas.core.encoders import encode_array, encode_jpeg, encode_png +from testutils import run_tests +import pytest +import numpy as np + + +def get_random_im(*shape): + """Get a random image.""" + return np.random.randint(0, 100, shape).astype(np.uint8) + + +def test_encode_array(): + """Test the encode_array function.""" + + # This test assumes that simplejpeg is installed + + im = np.random.randint(0, 255, (100, 100, 3)).astype(np.uint8) + + # Basic check + preamble, bb = encode_array(im) + assert isinstance(preamble, str) + assert isinstance(bb, bytes) + assert "jpeg" in preamble and "png" not in preamble + + # Check compression + preamble1, bb1 = encode_array(im, 90) + preamble2, bb2 = encode_array(im, 30) + assert len(bb2) < len(bb1) + + # Check quality 100 + preamble3, bb3 = encode_array(im, 100) + assert len(bb3) > len(bb1) + + assert "jpeg" in preamble1 and "png" not in preamble1 + assert "jpeg" in preamble2 and "png" not in preamble1 + assert "png" in preamble3 and "jpeg" not in preamble3 + + # Check that RGBA is made RGB + im4 = np.random.randint(0, 255, (100, 100, 4)).astype(np.uint8) + im3 = im4[:, :, :3] + _, bb1 = encode_array(im4, 90) + _, bb2 = encode_array(im3, 90) + assert bb1 == bb2 + + # Also for PNG mode + _, bb1 = encode_array(im4, 100) + _, bb2 = encode_array(im3, 100) + assert bb1 == bb2 + + # Check fallback - disable JPEG encoding, we get PNG + encoders.CAN_JPEG = False + try: + preamble, bb = encode_array(im) + assert isinstance(preamble, str) + assert isinstance(bb, bytes) + assert "png" in preamble and "jpeg" not in preamble + + finally: + encoders.CAN_JPEG = True + + # Should be back to normal now + preamble, bb = encode_array(im) + assert "jpeg" in preamble and "png" not in preamble + + +def test_encode_jpeg(): + """Tests for encode_jpeg function.""" + + _perform_checks(encode_jpeg, 90, 20) + _perform_error_checks(encode_jpeg) + + +def test_encode_png(): + """Tests for encode_jpeg function.""" + + _perform_checks(encode_png, 3, 9) + _perform_error_checks(encode_png) + + +def _perform_checks(encode, c1, c2): + + # Works without compression/level param + im = get_random_im(100, 100, 3) + _bb0 = encode(im) + + # RGB + bb1 = encode(im, c1) + bb2 = encode(im, c2) + assert isinstance(bb1, bytes) + assert len(bb2) < len(bb1) + + # RGB non-contiguous + im = get_random_im(100, 100, 3) + bb1 = encode(im[20:-20, 20:-20, :], c1) + bb2 = encode(im[20:-20, 20:-20, :], c2) + assert isinstance(bb1, bytes) + assert len(bb2) < len(bb1) + + # Gray1 + im = get_random_im(100, 100) + bb1 = encode(im, c1) + bb2 = encode(im, c2) + assert isinstance(bb1, bytes) + assert len(bb2) < len(bb1) + + # Gray2 + im = get_random_im(100, 100, 1) + bb1 = encode(im, c1) + bb2 = encode(im, c2) + assert isinstance(bb1, bytes) + assert len(bb2) < len(bb1) + + # Gray non-contiguous + im = get_random_im(200, 200) + bb1 = encode(im[20:-20, 20:-20], c1) + bb2 = encode(im[20:-20, 20:-20], c2) + assert isinstance(bb1, bytes) + assert len(bb2) < len(bb1) + + +def _perform_error_checks(encode): + # Just to verify that this is ok + encode(get_random_im(10, 10, 3)) + + with pytest.raises(TypeError): # not a numpy array + encode([1, 2, 3, 4]) + + with pytest.raises(TypeError): # not a numpy array + encode(b"1234") + + with pytest.raises(ValueError): # NxMx2? + encode(get_random_im(10, 10, 2)) + + with pytest.raises(TypeError): + encode(get_random_im(10, 10, 3).astype(np.float32)) + + +if __name__ == "__main__": + run_tests(globals())