diff --git a/.chronus/changes/exclue-venv-2024-6-22-15-39-49.md b/.chronus/changes/exclue-venv-2024-6-22-15-39-49.md
new file mode 100644
index 0000000000..d4436766fc
--- /dev/null
+++ b/.chronus/changes/exclue-venv-2024-6-22-15-39-49.md
@@ -0,0 +1,7 @@
+---
+changeKind: internal
+packages:
+  - "@azure-tools/typespec-python"
+---
+
+Skip to format files under `venv`/`env` and any folder start with `.`
diff --git a/packages/typespec-python/generator/pygen/__init__.py b/packages/typespec-python/generator/pygen/__init__.py
index 6050cbec00..8def09e632 100644
--- a/packages/typespec-python/generator/pygen/__init__.py
+++ b/packages/typespec-python/generator/pygen/__init__.py
@@ -54,7 +54,7 @@ def write_file(self, filename: Union[str, Path], file_content: str) -> None:
             fd.write(file_content)
 
     def list_file(self) -> List[str]:
-        return [str(f) for f in self.output_folder.glob("**/*") if f.is_file()]
+        return [str(f.relative_to(self.output_folder)) for f in self.output_folder.glob("**/*") if f.is_file()]
 
 
 class Plugin(ReaderAndWriter, ABC):
diff --git a/packages/typespec-python/generator/pygen/black.py b/packages/typespec-python/generator/pygen/black.py
index b1d48aff82..78c55dfc83 100644
--- a/packages/typespec-python/generator/pygen/black.py
+++ b/packages/typespec-python/generator/pygen/black.py
@@ -37,15 +37,14 @@ def process(self) -> bool:
                 [
                     Path(f)
                     for f in self.list_file()
-                    if all(
-                        item not in f
-                        for item in (
-                            "__pycache__",
-                            "node_modules",
-                            ".tox",
-                            ".mypy_cache",
-                        )
+                    if Path(f).parts[0]
+                    not in (
+                        "__pycache__",
+                        "node_modules",
+                        "venv",
+                        "env",
                     )
+                    and not Path(f).parts[0].startswith(".")
                     and Path(f).suffix == ".py"
                 ],
             )