Merge pull request #448 from Renumics/feature/allow-to-start-on-non-l…

…ocalhost-without-tls

Allow to start on non-localhost without TLS

pyproject.toml

@@ -127,7 +127,7 @@ pandas-stubs = "^2.0.2.230605"
 ruff = "^0.2.1"
 check-wheel-contents = "^0.6.0"
 torch = { version = "^2.1.1+cpu", source = "torch-cpu" }
-ray = {extras = ["data"], version = "^2.9.3"}
+ray = {extras = ["data"], version = "==2.9.3"}
 
 [tool.poetry.group.playbook.dependencies]
 towhee = "^0.9.0"
@@ -162,7 +162,6 @@ plugins = "pydantic.mypy"
 [[tool.mypy.overrides]]
 module = [
   "pygltflib",
-  "av",
   "validators",
   "h5py",
   "pandas",

renumics/spotlight/cli.py

@@ -124,6 +124,12 @@ def cli_dtype_callback(
 @click.option(
     "--ssl-keyfile-password", type=str, default=None, help="SSL keyfile password"
 )
+@click.option(
+    "--no-ssl",
+    is_flag=True,
+    default=False,
+    help="Do not require SSL sertificate and keyfile when starting on non-localhost.",
+)
 @click.option("-v", "--verbose", is_flag=True)
 @click.version_option(spotlight.__version__)
 def main(
@@ -142,6 +148,7 @@ def main(
     ssl_keyfile: Optional[str],
     ssl_certfile: Optional[str],
     ssl_keyfile_password: Optional[str],
+    no_ssl: bool,
     verbose: bool,
 ) -> None:
     """
@@ -171,4 +178,5 @@ def main(
         ssl_keyfile=ssl_keyfile,
         ssl_certfile=ssl_certfile,
         ssl_keyfile_password=ssl_keyfile_password,
+        no_ssl=no_ssl,
     )

renumics/spotlight/embeddings/preprocessors.py

@@ -4,7 +4,7 @@
 """
 
 import io
-from typing import List
+from typing import List, cast
 
 import av
 import numpy as np
@@ -36,10 +36,11 @@ def preprocess_audio_batch(
     for raw_data in raw_values:
         with av.open(io.BytesIO(raw_data), "r") as container:
             resampler = av.AudioResampler(
-                format="dbl", layout="mono", rate=sampling_rate
+                format="dbl", layout="mono", rate=sampling_rate  # type: ignore
             )
             data = []
             for frame in container.decode(audio=0):
+                frame = cast(av.audio.AudioFrame, frame)
                 resampled_frames = resampler.resample(frame)
                 for resampled_frame in resampled_frames:
                     frame_array = resampled_frame.to_ndarray()[0]

renumics/spotlight/io/audio.py

@@ -4,9 +4,10 @@
 
 import io
 import os
-from typing import IO, Dict, Tuple, Union
+from typing import IO, Dict, Tuple, Union, cast
 
 import av
+import av.audio
 import numpy as np
 import requests
 import validators
@@ -82,6 +83,7 @@ def read_audio(file: FileType) -> Tuple[np.ndarray, int]:
 
         data = []
         for frame in container.decode(audio=0):
+            frame = cast(av.audio.AudioFrame, frame)
             frame_array = frame.to_ndarray()
             if len(frame_array) == 1:
                 frame_array = frame_array.reshape((-1, num_channels))
@@ -123,13 +125,13 @@ def write_audio(
     # `AudioFrame.from_ndarray` expects an C-contiguous array as input.
     data = np.ascontiguousarray(data)
     num_channels = len(data)
-    frame = av.audio.AudioFrame.from_ndarray(data, data_format, num_channels)
+    frame = av.audio.AudioFrame.from_ndarray(data, data_format, num_channels)  # type: ignore
     frame.rate = sampling_rate
     with av.open(file, "w", format_) as container:
         stream = container.add_stream(codec, sampling_rate)
-        stream.channels = num_channels
-        container.mux(stream.encode(frame))
-        container.mux(stream.encode(None))
+        stream.channels = num_channels  # type: ignore
+        container.mux(stream.encode(frame))  # type: ignore
+        container.mux(stream.encode(None))  # type: ignore
 
 
 def transcode_audio(
@@ -155,10 +157,10 @@ def transcode_audio(
 
             for frame in input_container.decode(input_stream):
                 frame.pts = None
-                for packet in output_stream.encode(frame):
+                for packet in output_stream.encode(frame):  # type: ignore
                     output_container.mux(packet)
 
-            for packet in output_stream.encode(None):
+            for packet in output_stream.encode(None):  # type: ignore
                 output_container.mux(packet)
 
 
@@ -169,7 +171,8 @@ def get_format_codec(file: FileType) -> Tuple[str, str]:
     file = prepare_input_file(file)
     with av.open(file, "r") as input_container:
         stream = input_container.streams.audio[0]
-        return input_container.format.name, stream.name
+        return input_container.format.name, stream.name  # type: ignore
+    assert False  # unnecessary, but mypy fails otherwise
 
 
 def get_waveform(file: FileType) -> np.ndarray:

renumics/spotlight/media/mesh.py

@@ -1,7 +1,7 @@
 import io
 import math
 import os
-from typing import IO, Dict, List, Optional, Tuple, Union
+from typing import IO, Dict, List, Optional, Tuple, Union, cast
 from urllib.parse import urlparse
 
 import numpy as np
@@ -122,7 +122,7 @@ def from_trimesh(cls, mesh: trimesh.Trimesh) -> "Mesh":
         Import a `trimesh.Trimesh` mesh.
         """
         return cls(
-            mesh.vertices, mesh.faces, mesh.vertex_attributes, mesh.face_attributes
+            mesh.vertices, mesh.faces, mesh.vertex_attributes, mesh.face_attributes  # type: ignore
         )
 
     @classmethod
@@ -155,6 +155,7 @@ def from_file(cls, filepath: PathType) -> "Mesh":
             raise exceptions.InvalidFile(
                 f"Mesh {filepath} does not exist or could not be read."
             ) from e
+        mesh = cast(trimesh.Trimesh, mesh)
         return cls.from_trimesh(mesh)
 
     @classmethod

renumics/spotlight/server.py

@@ -72,6 +72,7 @@ def __init__(
         ssl_keyfile: Optional[str] = None,
         ssl_certfile: Optional[str] = None,
         ssl_keyfile_password: Optional[str] = None,
+        no_ssl: bool = False,
     ) -> None:
         self.process = None
 
@@ -80,7 +81,7 @@ def __init__(
         self._app_config = AppConfig()
 
         self._host = host
-        if self._host not in ("127.0.0.1", "localhost"):
+        if self._host not in ("127.0.0.1", "localhost") and not no_ssl:
             if ssl_certfile is None:
                 raise MissingTLSCertificate(
                     "Starting Spotlight on non-localhost without TLS certificate is insecure. Please provide TLS certificate and key."

renumics/spotlight/viewer.py

@@ -132,6 +132,7 @@ class Viewer:
     _ssl_keyfile: Optional[str]
     _ssl_certfile: Optional[str]
     _ssl_keyfile_password: Optional[str]
+    _no_ssl: bool
     _server: Optional[Server]
     _df: Optional[pd.DataFrame]
 
@@ -142,12 +143,14 @@ def __init__(
         ssl_keyfile: Optional[str] = None,
         ssl_certfile: Optional[str] = None,
         ssl_keyfile_password: Optional[str] = None,
+        no_ssl: bool = False,
     ) -> None:
         self._host = host
         self._requested_port = port
         self._ssl_keyfile = ssl_keyfile
         self._ssl_certfile = ssl_certfile
         self._ssl_keyfile_password = ssl_keyfile_password
+        self._no_ssl = no_ssl
         self._server = None
         self._df = None
 
@@ -233,6 +236,7 @@ def show(
                 self._ssl_keyfile,
                 self._ssl_certfile,
                 self._ssl_keyfile_password,
+                self._no_ssl,
             )
             self._server.start(config)
 
@@ -406,6 +410,7 @@ def show(
     ssl_keyfile: Optional[str] = None,
     ssl_certfile: Optional[str] = None,
     ssl_keyfile_password: Optional[str] = None,
+    no_ssl: bool = False,
 ) -> Viewer:
     """
     Start a new Spotlight viewer.
@@ -435,6 +440,7 @@ def show(
         ssl_keyfile: Optional SSL key file.
         ssl_certfile: Optional SSL certificate file.
         ssl_certfile: Optional SSL keyfile password.
+        no_ssl: Do not require SSL sertificate and keyfile when starting on non-localhost.
     """
 
     viewer = None
@@ -445,7 +451,9 @@ def show(
                 viewer = _VIEWERS[index]
                 break
     if not viewer:
-        viewer = Viewer(host, port, ssl_keyfile, ssl_certfile, ssl_keyfile_password)
+        viewer = Viewer(
+            host, port, ssl_keyfile, ssl_certfile, ssl_keyfile_password, no_ssl
+        )
 
     viewer.show(
         dataset,

renumics/spotlight_plugins/core/huggingface_datasource.py

@@ -41,7 +41,6 @@ class HuggingfaceDataSource(DataSource):
     _guessed_dtypes: spotlight_dtypes.DTypeMap
 
     def __init__(self, source: datasets.Dataset):
-        super().__init__(source)
         self._dataset = source
         self._intermediate_dtypes = {
             col: _get_intermediate_dtype(feat)