From f5744380308d1941d5803b538f67fb41cd626bae Mon Sep 17 00:00:00 2001 From: Andrew Truong Date: Fri, 23 Aug 2024 11:50:09 -0400 Subject: [PATCH 1/3] move to legacy: util --- weave/filesystem.py | 4 +--- weave/legacy/api.py | 2 +- weave/legacy/artifact_local.py | 3 ++- weave/legacy/artifact_wandb.py | 2 +- weave/legacy/context.py | 2 +- weave/legacy/dispatch.py | 2 +- weave/legacy/environment.py | 2 +- .../ops_domain/run_history/history_op_common.py | 4 +--- weave/legacy/ops_domain/table.py | 3 ++- weave/legacy/panels_py/panel_autoboard.py | 3 ++- weave/legacy/panels_py/panel_llm_monitor.py | 4 ++-- weave/legacy/panels_py/panel_trace_monitor.py | 4 ++-- weave/legacy/scripts/syndata.py | 2 +- weave/legacy/server.py | 3 ++- weave/legacy/server_error_handling.py | 2 +- weave/legacy/show.py | 2 +- weave/{ => legacy}/util.py | 16 ++++++++-------- weave/tests/fixture_fakewandb.py | 3 +-- weave/tests/legacy/test_relpath_no_syscalls.py | 2 +- weave/trace/api.py | 3 +-- weave/trace/feedback.py | 3 +-- weave/trace/rich_pydantic_util.py | 2 +- weave/weave_server.py | 2 +- 23 files changed, 36 insertions(+), 39 deletions(-) rename weave/{ => legacy}/util.py (93%) diff --git a/weave/filesystem.py b/weave/filesystem.py index d46b06c68436..559ecaa1bd80 100644 --- a/weave/filesystem.py +++ b/weave/filesystem.py @@ -20,9 +20,7 @@ from aiofiles.threadpool import binary as aiofiles_binary from aiofiles.threadpool import text as aiofiles_text -from weave.legacy import cache, engine_trace, environment, errors - -from . import util +from weave.legacy import cache, engine_trace, environment, errors, util tracer = engine_trace.tracer() # type: ignore async_utime = aiofiles_os.wrap(os.utime) # type: ignore diff --git a/weave/legacy/api.py b/weave/legacy/api.py index 177efdf3d208..54ad4e2fcb5b 100644 --- a/weave/legacy/api.py +++ b/weave/legacy/api.py @@ -14,7 +14,7 @@ from .. import weave_internal as _weave_internal -from .. import util as _util +from weave.legacy import util as _util from weave.legacy import context as _context from .. import weave_init as _weave_init diff --git a/weave/legacy/artifact_local.py b/weave/legacy/artifact_local.py index 498c3e5d2d57..ff271ae4c485 100644 --- a/weave/legacy/artifact_local.py +++ b/weave/legacy/artifact_local.py @@ -9,7 +9,7 @@ import typing from datetime import datetime -from weave import filesystem, util +from weave import filesystem from weave import weave_types as types from weave.legacy import ( artifact_fs, @@ -19,6 +19,7 @@ file_base, file_util, uris, + util, ) WORKING_DIR_PREFIX = "__working__" diff --git a/weave/legacy/artifact_wandb.py b/weave/legacy/artifact_wandb.py index 1b44de0bfe45..2c3173a3b8c7 100644 --- a/weave/legacy/artifact_wandb.py +++ b/weave/legacy/artifact_wandb.py @@ -15,7 +15,6 @@ from weave import ( filesystem, - util, ) from weave import weave_types as types from weave.legacy import ( @@ -29,6 +28,7 @@ memo, uris, wandb_client_api, + util, ) from weave.legacy.wandb_interface import wandb_artifact_pusher diff --git a/weave/legacy/context.py b/weave/legacy/context.py index 8333bcd96923..ab01ec337279 100644 --- a/weave/legacy/context.py +++ b/weave/legacy/context.py @@ -4,7 +4,7 @@ import typing from urllib.parse import urlparse -from weave import util +from weave.legacy import util from weave.legacy import urls, client, context_state from weave.legacy.client_interface import ClientInterface diff --git a/weave/legacy/dispatch.py b/weave/legacy/dispatch.py index 872ad267e688..9ab44a173120 100644 --- a/weave/legacy/dispatch.py +++ b/weave/legacy/dispatch.py @@ -5,7 +5,7 @@ import typing from dataclasses import dataclass -from weave import util +from weave.legacy import util from weave import weave_types as types from weave.legacy import graph, memo, op_args, op_def, pyfunc_type_util, registry_mem, errors from weave.legacy.language_features.tagging.is_tag_getter import is_tag_getter diff --git a/weave/legacy/environment.py b/weave/legacy/environment.py index ed4a0ecd6d60..a2bc2d818526 100644 --- a/weave/legacy/environment.py +++ b/weave/legacy/environment.py @@ -11,7 +11,7 @@ from distutils.util import strtobool from urllib.parse import urlparse from weave.legacy import errors -from .. import util +from weave.legacy import util if typing.TYPE_CHECKING: from weave.legacy import logs diff --git a/weave/legacy/ops_domain/run_history/history_op_common.py b/weave/legacy/ops_domain/run_history/history_op_common.py index b95c1bf14b46..c1e6f7a1f0fb 100644 --- a/weave/legacy/ops_domain/run_history/history_op_common.py +++ b/weave/legacy/ops_domain/run_history/history_op_common.py @@ -4,12 +4,10 @@ import pyarrow as pa from pyarrow import parquet as pq -from weave import ( - util, -) from weave import weave_types as types from weave.legacy.api import use from weave.legacy import ( + util, errors, engine_trace, registry_mem, diff --git a/weave/legacy/ops_domain/table.py b/weave/legacy/ops_domain/table.py index 1253293fe95a..1f295cf62b52 100644 --- a/weave/legacy/ops_domain/table.py +++ b/weave/legacy/ops_domain/table.py @@ -5,7 +5,7 @@ import logging import typing -from weave import util, weave_internal +from weave import weave_internal from weave import weave_types as types from weave.legacy.api import op, weave_class from weave.legacy import ( @@ -16,6 +16,7 @@ wandb_util, engine_trace, errors, + util, ) from weave.legacy import timestamp as weave_timestamp from weave.legacy.ops_domain import trace_tree, wbmedia diff --git a/weave/legacy/panels_py/panel_autoboard.py b/weave/legacy/panels_py/panel_autoboard.py index b25c48c276ce..ab37b9813ac4 100644 --- a/weave/legacy/panels_py/panel_autoboard.py +++ b/weave/legacy/panels_py/panel_autoboard.py @@ -26,9 +26,10 @@ import typing import weave -from weave import util, weave_internal +from weave import weave_internal from weave.legacy.panels import panel_plot from weave.legacy.panels_py.generator_templates import template_registry +from weave.legacy import util @weave.type() diff --git a/weave/legacy/panels_py/panel_llm_monitor.py b/weave/legacy/panels_py/panel_llm_monitor.py index cdaa9cffd829..73cc9278e2bd 100644 --- a/weave/legacy/panels_py/panel_llm_monitor.py +++ b/weave/legacy/panels_py/panel_llm_monitor.py @@ -2,10 +2,10 @@ import typing import weave -from weave import util, weave_internal +from weave import weave_internal from weave import weave_internal as internal from weave import weave_types as types -from weave.legacy import dispatch +from weave.legacy import util, dispatch from weave.legacy.panels import panel_board, panel_group from weave.legacy.panels_py import panel_autoboard from weave.legacy.panels_py.generator_templates import template_registry diff --git a/weave/legacy/panels_py/panel_trace_monitor.py b/weave/legacy/panels_py/panel_trace_monitor.py index d4fab19ccee5..cb0a9a9e3340 100644 --- a/weave/legacy/panels_py/panel_trace_monitor.py +++ b/weave/legacy/panels_py/panel_trace_monitor.py @@ -1,10 +1,10 @@ import os import weave -from weave import util, weave_internal +from weave import weave_internal from weave import weave_internal as internal from weave import weave_types as types -from weave.legacy import dispatch, graph +from weave.legacy import util, dispatch, graph from weave.legacy.panels import panel_board, panel_group, panel_trace from weave.legacy.panels.panel_trace_span import TraceSpanModelPanel, TraceSpanPanel from weave.legacy.panels_py import panel_autoboard diff --git a/weave/legacy/scripts/syndata.py b/weave/legacy/scripts/syndata.py index 9f7c697b556e..c2436a7efd90 100644 --- a/weave/legacy/scripts/syndata.py +++ b/weave/legacy/scripts/syndata.py @@ -5,7 +5,7 @@ import pyarrow as pa from scipy.signal import butter, filtfilt -from ... import util +from weave.legacy import util from ..ops_arrow import ArrowWeaveList value_fns: list[typing.Any] = [ diff --git a/weave/legacy/server.py b/weave/legacy/server.py index d008f30e1227..de94e6b17bb6 100644 --- a/weave/legacy/server.py +++ b/weave/legacy/server.py @@ -23,11 +23,12 @@ value_or_error, wandb_api, logs, + util, ) from weave.legacy.language_features.tagging import tag_store from weave.legacy.language_features.tagging.tag_store import isolated_tagging_context -from .. import storage, util, weave_types +from .. import storage, weave_types # A function to monkeypatch the request post method # def patch_request_post(): diff --git a/weave/legacy/server_error_handling.py b/weave/legacy/server_error_handling.py index 66c89590accd..f7c262a24c03 100644 --- a/weave/legacy/server_error_handling.py +++ b/weave/legacy/server_error_handling.py @@ -28,7 +28,7 @@ from werkzeug import http as werkzeug_http from weave.legacy import errors -from .. import util +from weave.legacy import util class WeaveInternalHttpException(werkzeug_exceptions.HTTPException): diff --git a/weave/legacy/show.py b/weave/legacy/show.py index 7091b2f8444a..0f03b3da9007 100644 --- a/weave/legacy/show.py +++ b/weave/legacy/show.py @@ -6,7 +6,7 @@ from IPython.display import IFrame, display from weave.legacy import storage, artifact_fs, context, errors, graph, ops, node_ref, panel, ref_base -from weave import util +from weave.legacy import util from . import usage_analytics from .. import weave_types as types from . import weavejs_fixes diff --git a/weave/util.py b/weave/legacy/util.py similarity index 93% rename from weave/util.py rename to weave/legacy/util.py index 1053f197c2bc..8027cf35a9d8 100644 --- a/weave/util.py +++ b/weave/legacy/util.py @@ -8,12 +8,12 @@ # import ipynbname import typing -from .legacy.errors import WeaveFingerprintErrorMixin +from weave.legacy.errors import WeaveFingerprintErrorMixin sentry_inited = False -def init_sentry(): +def init_sentry(): # type: ignore global sentry_inited if sentry_inited: return @@ -59,11 +59,11 @@ def capture_exception_with_sentry_if_available( # return ipynbname.name() -def get_hostname(): +def get_hostname(): # type: ignore return socket.gethostname() -def get_pid(): +def get_pid(): # type: ignore return os.getpid() @@ -95,7 +95,7 @@ def parse_number_env_var(name: str) -> typing.Optional[typing.Union[int, float]] return float(raw_val) -def find_names(obj): +def find_names(obj): # type: ignore if hasattr(obj, "name"): return [obj.name] frame = inspect.currentframe() @@ -110,7 +110,7 @@ def find_names(obj): return obj_names -def is_colab(): +def is_colab(): # type: ignore import importlib spec = importlib.util.find_spec("google.colab") @@ -118,7 +118,7 @@ def is_colab(): def is_notebook() -> bool: - if is_colab(): + if is_colab(): # type: ignore[no-untyped-call] return True try: from IPython import get_ipython @@ -135,7 +135,7 @@ def is_notebook() -> bool: return True -def is_pandas_dataframe(obj): +def is_pandas_dataframe(obj): # type: ignore try: import pandas as pd except ImportError: diff --git a/weave/tests/fixture_fakewandb.py b/weave/tests/fixture_fakewandb.py index 764abac362ed..bbce4d66b918 100644 --- a/weave/tests/fixture_fakewandb.py +++ b/weave/tests/fixture_fakewandb.py @@ -11,12 +11,11 @@ import wandb import weave -from weave import util # Note: We're mocking out the whole io_service right now. This is too # high level and doesn't test the actual io implementation. We should # mock wandb_api instead probably. -from weave.legacy import io_service, wandb_api, wandb_client_api +from weave.legacy import io_service, util, wandb_api, wandb_client_api from weave.legacy.artifact_wandb import ( WandbArtifact, WandbArtifactManifest, diff --git a/weave/tests/legacy/test_relpath_no_syscalls.py b/weave/tests/legacy/test_relpath_no_syscalls.py index 3138cabaabda..412aed2b06e4 100644 --- a/weave/tests/legacy/test_relpath_no_syscalls.py +++ b/weave/tests/legacy/test_relpath_no_syscalls.py @@ -1,6 +1,6 @@ import pytest -from ...util import relpath_no_syscalls +from weave.legacy.util import relpath_no_syscalls @pytest.mark.parametrize( diff --git a/weave/trace/api.py b/weave/trace/api.py index 449c1b347582..e6615a7221a6 100644 --- a/weave/trace/api.py +++ b/weave/trace/api.py @@ -7,14 +7,13 @@ from typing import Any, Callable, Iterator, Optional, Union from weave.client_context import weave_client as weave_client_context -from weave.legacy import urls +from weave.legacy import urls, util from weave.trace.call_context import get_current_call # TODO: type_serializers is imported here to trigger registration of the image serializer. # There is probably a better place for this, but including here for now to get the fix in. from .. import ( type_serializers, # noqa: F401 - util, weave_init, ) from . import context, weave_client diff --git a/weave/trace/feedback.py b/weave/trace/feedback.py index 01103f82d024..8f2eafd378f3 100644 --- a/weave/trace/feedback.py +++ b/weave/trace/feedback.py @@ -6,6 +6,7 @@ from rich.table import Table from weave.client_context import weave_client as weave_client_context +from weave.legacy import util from weave.trace import rich_pydantic_util from weave.trace.refs import parse_uri from weave.trace.rich_container import AbstractRichContainer @@ -13,8 +14,6 @@ from weave.trace_server import trace_server_interface as tsi from weave.trace_server.interface.query import Query -from .. import util - class Feedbacks(AbstractRichContainer[tsi.Feedback]): """A collection of Feedback objects with utilities.""" diff --git a/weave/trace/rich_pydantic_util.py b/weave/trace/rich_pydantic_util.py index 1c2c1474ef71..2c5e21fa2ed1 100644 --- a/weave/trace/rich_pydantic_util.py +++ b/weave/trace/rich_pydantic_util.py @@ -6,7 +6,7 @@ from rich.console import Console from rich.table import Table -from .. import util +from weave.legacy import util def dict_to_table(d: dict[str, Any]) -> Table: diff --git a/weave/weave_server.py b/weave/weave_server.py index 509fea89cced..edb84bef3931 100644 --- a/weave/weave_server.py +++ b/weave/weave_server.py @@ -27,7 +27,6 @@ from weave import ( filesystem, - util, ) from weave.legacy import ( context_state, @@ -39,6 +38,7 @@ registry_mem, server, storage, + util, value_or_error, wandb_api, weavejs_fixes, From 28fbfbc8d721b23632c60f7e0bc45e52f3296422 Mon Sep 17 00:00:00 2001 From: Andrew Truong Date: Fri, 23 Aug 2024 12:02:06 -0400 Subject: [PATCH 2/3] move to legacy: filesystem --- weave/legacy/artifact_local.py | 2 +- weave/legacy/artifact_wandb.py | 4 +--- weave/legacy/file_util.py | 2 +- weave/{ => legacy}/filesystem.py | 0 weave/legacy/io_service.py | 2 +- weave/legacy/runfiles_wandb.py | 2 +- weave/legacy/wandb_file_manager.py | 4 ++-- weave/test_scripts/wandb_artifact_perf.py | 3 ++- weave/tests/legacy/test_filesystem.py | 4 +--- weave/tests/legacy/test_io_service.py | 4 +--- weave/weave_http.py | 4 +--- weave/weave_server.py | 4 +--- 12 files changed, 13 insertions(+), 22 deletions(-) rename weave/{ => legacy}/filesystem.py (100%) diff --git a/weave/legacy/artifact_local.py b/weave/legacy/artifact_local.py index ff271ae4c485..0a4e9f6ad003 100644 --- a/weave/legacy/artifact_local.py +++ b/weave/legacy/artifact_local.py @@ -9,7 +9,7 @@ import typing from datetime import datetime -from weave import filesystem +from weave.legacy import filesystem from weave import weave_types as types from weave.legacy import ( artifact_fs, diff --git a/weave/legacy/artifact_wandb.py b/weave/legacy/artifact_wandb.py index 2c3173a3b8c7..575f2a3a14c4 100644 --- a/weave/legacy/artifact_wandb.py +++ b/weave/legacy/artifact_wandb.py @@ -13,11 +13,9 @@ from wandb.apis.public import api as wb_public from wandb.sdk.lib.hashutil import b64_to_hex_id, hex_to_b64_id -from weave import ( - filesystem, -) from weave import weave_types as types from weave.legacy import ( + filesystem, urls, errors, engine_trace, diff --git a/weave/legacy/file_util.py b/weave/legacy/file_util.py index b43c6dad627a..31da6fc6647b 100644 --- a/weave/legacy/file_util.py +++ b/weave/legacy/file_util.py @@ -2,7 +2,7 @@ import pathlib import typing -from weave import filesystem +from weave.legacy import filesystem from weave.legacy import cache, environment, path_util diff --git a/weave/filesystem.py b/weave/legacy/filesystem.py similarity index 100% rename from weave/filesystem.py rename to weave/legacy/filesystem.py diff --git a/weave/legacy/io_service.py b/weave/legacy/io_service.py index 701363c0a366..448660c64f10 100644 --- a/weave/legacy/io_service.py +++ b/weave/legacy/io_service.py @@ -23,10 +23,10 @@ import aioprocessing from weave import ( - filesystem, weave_http, ) from weave.legacy import ( + filesystem, errors, engine_trace, server_error_handling, diff --git a/weave/legacy/runfiles_wandb.py b/weave/legacy/runfiles_wandb.py index 5aef47279619..819dc7696ea2 100644 --- a/weave/legacy/runfiles_wandb.py +++ b/weave/legacy/runfiles_wandb.py @@ -7,7 +7,7 @@ import typing from weave.legacy import environment as weave_env -from weave import filesystem +from weave.legacy import filesystem from weave import weave_types as types from weave.legacy import ( artifact_fs, diff --git a/weave/legacy/wandb_file_manager.py b/weave/legacy/wandb_file_manager.py index 2e437c0021ad..cd1662409235 100644 --- a/weave/legacy/wandb_file_manager.py +++ b/weave/legacy/wandb_file_manager.py @@ -12,9 +12,9 @@ from requests.auth import HTTPBasicAuth from wandb.sdk.lib import hashutil -from weave import filesystem, weave_http +from weave import weave_http from weave.legacy import environment as weave_env -from weave.legacy import artifact_wandb, cache, errors, wandb_api, engine_trace +from weave.legacy import filesystem, artifact_wandb, cache, errors, wandb_api, engine_trace tracer = engine_trace.tracer() # type: ignore diff --git a/weave/test_scripts/wandb_artifact_perf.py b/weave/test_scripts/wandb_artifact_perf.py index 0ba0b420b0a5..bfe7b3cc2ae2 100644 --- a/weave/test_scripts/wandb_artifact_perf.py +++ b/weave/test_scripts/wandb_artifact_perf.py @@ -17,11 +17,12 @@ artifact_wandb, async_map, engine_trace, + filesystem, wandb_api, wandb_file_manager, ) -from .. import filesystem, weave_http +from .. import weave_http tracer = engine_trace.tracer() # type: ignore diff --git a/weave/tests/legacy/test_filesystem.py b/weave/tests/legacy/test_filesystem.py index aaae05b21e97..3fd8cde70cbf 100644 --- a/weave/tests/legacy/test_filesystem.py +++ b/weave/tests/legacy/test_filesystem.py @@ -2,9 +2,7 @@ import pytest -from weave.legacy import environment, errors - -from ... import filesystem +from weave.legacy import environment, errors, filesystem @pytest.fixture() diff --git a/weave/tests/legacy/test_io_service.py b/weave/tests/legacy/test_io_service.py index 5f3f0103d2ca..053ee5d19cd9 100644 --- a/weave/tests/legacy/test_io_service.py +++ b/weave/tests/legacy/test_io_service.py @@ -2,9 +2,7 @@ import pytest -from weave.legacy import io_service - -from ... import filesystem +from weave.legacy import filesystem, io_service @pytest.mark.timeout(10) diff --git a/weave/weave_http.py b/weave/weave_http.py index 075693b8edf3..773625fe9ec8 100644 --- a/weave/weave_http.py +++ b/weave/weave_http.py @@ -13,9 +13,7 @@ import requests.auth import yarl -from weave.legacy import engine_trace, server_error_handling - -from . import filesystem +from weave.legacy import engine_trace, filesystem, server_error_handling logging.getLogger("aiohttp.access").setLevel(logging.WARNING) logging.getLogger("aiohttp.client").setLevel(logging.WARNING) diff --git a/weave/weave_server.py b/weave/weave_server.py index edb84bef3931..b07886df75d2 100644 --- a/weave/weave_server.py +++ b/weave/weave_server.py @@ -25,14 +25,12 @@ from flask_cors import CORS from werkzeug.exceptions import HTTPException -from weave import ( - filesystem, -) from weave.legacy import ( context_state, engine_trace, environment, errors, + filesystem, graph, logs, registry_mem, From 633776673ffd932f5ee67f2c4c1e6baa3ac7efa4 Mon Sep 17 00:00:00 2001 From: Andrew Truong Date: Fri, 23 Aug 2024 12:16:26 -0400 Subject: [PATCH 3/3] move to legacy: weave_http --- weave/legacy/io_service.py | 4 +--- weave/legacy/wandb_file_manager.py | 3 +-- weave/{ => legacy}/weave_http.py | 0 weave/test_scripts/wandb_artifact_perf.py | 3 +-- 4 files changed, 3 insertions(+), 7 deletions(-) rename weave/{ => legacy}/weave_http.py (100%) diff --git a/weave/legacy/io_service.py b/weave/legacy/io_service.py index 448660c64f10..9091573a914a 100644 --- a/weave/legacy/io_service.py +++ b/weave/legacy/io_service.py @@ -22,10 +22,8 @@ import aioprocessing -from weave import ( - weave_http, -) from weave.legacy import ( + weave_http, filesystem, errors, engine_trace, diff --git a/weave/legacy/wandb_file_manager.py b/weave/legacy/wandb_file_manager.py index cd1662409235..ecce61b90d06 100644 --- a/weave/legacy/wandb_file_manager.py +++ b/weave/legacy/wandb_file_manager.py @@ -12,9 +12,8 @@ from requests.auth import HTTPBasicAuth from wandb.sdk.lib import hashutil -from weave import weave_http from weave.legacy import environment as weave_env -from weave.legacy import filesystem, artifact_wandb, cache, errors, wandb_api, engine_trace +from weave.legacy import filesystem, artifact_wandb, cache, errors, wandb_api, engine_trace, weave_http tracer = engine_trace.tracer() # type: ignore diff --git a/weave/weave_http.py b/weave/legacy/weave_http.py similarity index 100% rename from weave/weave_http.py rename to weave/legacy/weave_http.py diff --git a/weave/test_scripts/wandb_artifact_perf.py b/weave/test_scripts/wandb_artifact_perf.py index bfe7b3cc2ae2..bc38aeb1d647 100644 --- a/weave/test_scripts/wandb_artifact_perf.py +++ b/weave/test_scripts/wandb_artifact_perf.py @@ -20,10 +20,9 @@ filesystem, wandb_api, wandb_file_manager, + weave_http, ) -from .. import weave_http - tracer = engine_trace.tracer() # type: ignore