diff --git a/.github/workflows/e2e_preview.yml b/.github/workflows/e2e_preview.yml
index aa54f8963e..905acc7e90 100644
--- a/.github/workflows/e2e_preview.yml
+++ b/.github/workflows/e2e_preview.yml
@@ -36,7 +36,9 @@ jobs:
         sudo apt install ffmpeg  # for local Whisper tests
 
     - name: Install Haystack
-      run: pip install .[dev,preview] langdetect transformers[torch,sentencepiece]==4.32.1 'sentence-transformers>=2.2.0' pypdf openai-whisper tika 'azure-ai-formrecognizer>=3.2.0b2'
+      run: |
+          pip install .[dev,preview] langdetect transformers[torch,sentencepiece]==4.34.1 'sentence-transformers>=2.2.0' pypdf tika 'azure-ai-formrecognizer>=3.2.0b2'
+          pip install --no-deps llvmlite numba 'openai-whisper>=20230918'  # prevent outdated version of tiktoken pinned by openai-whisper
 
     - name: Run tests
       run: pytest e2e/preview
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
index 012535bd7c..bd8c782db9 100644
--- a/.github/workflows/linting.yml
+++ b/.github/workflows/linting.yml
@@ -39,7 +39,9 @@ jobs:
           python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Haystack
-        run: pip install ".[all,dev]"
+        run: |
+          pip install ".[all,dev]"
+          pip install --no-deps llvmlite numba "openai-whisper>=20230918"
 
       - name: Mypy
         if: steps.files.outputs.any_changed == 'true'
@@ -74,6 +76,7 @@ jobs:
       - name: Install Haystack
         run: |
           pip install ".[all,dev]"
+          pip install --no-deps llvmlite numba "openai-whisper>=20230918"
           pip install ./haystack-linter
 
       - name: Pylint
diff --git a/.github/workflows/linting_preview.yml b/.github/workflows/linting_preview.yml
index ffd7c2b442..c291788f6d 100644
--- a/.github/workflows/linting_preview.yml
+++ b/.github/workflows/linting_preview.yml
@@ -38,7 +38,9 @@ jobs:
           python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Haystack
-        run: pip install .[dev,preview] langdetect transformers[torch,sentencepiece]==4.32.1 'sentence-transformers>=2.2.0' pypdf openai-whisper tika 'azure-ai-formrecognizer>=3.2.0b2'
+        run: |
+          pip install .[dev,preview] langdetect transformers[torch,sentencepiece]==4.34.1 'sentence-transformers>=2.2.0' pypdf tika 'azure-ai-formrecognizer>=3.2.0b2'
+          pip install --no-deps llvmlite numba 'openai-whisper>=20230918'  # prevent outdated version of tiktoken pinned by openai-whisper
 
       - name: Mypy
         if: steps.files.outputs.any_changed == 'true'
@@ -72,7 +74,8 @@ jobs:
 
       - name: Install Haystack
         run: |
-          pip install .[dev,preview] langdetect transformers[torch,sentencepiece]==4.32.1 'sentence-transformers>=2.2.0' pypdf openai-whisper tika 'azure-ai-formrecognizer>=3.2.0b2'
+          pip install .[dev,preview] langdetect transformers[torch,sentencepiece]==4.34.1 'sentence-transformers>=2.2.0' pypdf tika 'azure-ai-formrecognizer>=3.2.0b2'
+          pip install --no-deps llvmlite numba 'openai-whisper>=20230918'  # prevent outdated version of tiktoken pinned by openai-whisper
           pip install ./haystack-linter
 
       - name: Pylint
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 5406b14ab9..d4e50eed2d 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -202,7 +202,9 @@ jobs:
           python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Haystack
-        run: pip install .[preview,dev] langdetect transformers[torch,sentencepiece]==4.32.1 sentence-transformers>=2.2.0 pypdf openai-whisper tika 'azure-ai-formrecognizer>=3.2.0b2'
+        run: |
+          pip install .[dev,preview] langdetect transformers[torch,sentencepiece]==4.34.1 'sentence-transformers>=2.2.0' pypdf tika 'azure-ai-formrecognizer>=3.2.0b2'
+          pip install --no-deps llvmlite numba 'openai-whisper>=20230918'  # prevent outdated version of tiktoken pinned by openai-whisper
 
       - name: Run
         run: pytest --cov-report xml:coverage.xml --cov="haystack" -m "unit" test/preview
@@ -946,7 +948,9 @@ jobs:
           sudo apt install ffmpeg  # for local Whisper tests
 
       - name: Install Haystack
-        run: pip install .[dev,preview] langdetect transformers[torch,sentencepiece]==4.32.1 'sentence-transformers>=2.2.0' pypdf openai-whisper tika 'azure-ai-formrecognizer>=3.2.0b2'
+        run: |
+          pip install .[dev,preview] langdetect transformers[torch,sentencepiece]==4.34.1 'sentence-transformers>=2.2.0' pypdf tika 'azure-ai-formrecognizer>=3.2.0b2'
+          pip install --no-deps llvmlite numba 'openai-whisper>=20230918'  # prevent outdated version of tiktoken pinned by openai-whisper
 
       - name: Run tests
         run: |
diff --git a/.github/workflows/tests_preview.yml b/.github/workflows/tests_preview.yml
index 9f570491b7..b8feed0b4e 100644
--- a/.github/workflows/tests_preview.yml
+++ b/.github/workflows/tests_preview.yml
@@ -116,7 +116,9 @@ jobs:
           python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Haystack
-        run: pip install .[dev,preview] langdetect transformers[torch,sentencepiece]==4.32.1 'sentence-transformers>=2.2.0' pypdf openai-whisper tika 'azure-ai-formrecognizer>=3.2.0b2'
+        run: |
+          pip install .[dev,preview] langdetect transformers[torch,sentencepiece]==4.34.1 'sentence-transformers>=2.2.0' pypdf tika 'azure-ai-formrecognizer>=3.2.0b2'
+          pip install --no-deps llvmlite numba 'openai-whisper>=20230918'  # prevent outdated version of tiktoken pinned by openai-whisper
 
       - name: Run
         run: pytest -m "unit" test/preview
@@ -175,7 +177,9 @@ jobs:
           sudo apt install ffmpeg  # for local Whisper tests
 
       - name: Install Haystack
-        run: pip install .[dev,preview] langdetect transformers[torch,sentencepiece]==4.32.1 'sentence-transformers>=2.2.0' pypdf openai-whisper tika 'azure-ai-formrecognizer>=3.2.0b2'
+        run: |
+          pip install .[dev,preview] langdetect transformers[torch,sentencepiece]==4.34.1 'sentence-transformers>=2.2.0' pypdf tika 'azure-ai-formrecognizer>=3.2.0b2'
+          pip install --no-deps llvmlite numba 'openai-whisper>=20230918'  # prevent outdated version of tiktoken pinned by openai-whisper
 
       - name: Run
         run: pytest --maxfail=5 -m "integration" test/preview
@@ -230,7 +234,9 @@ jobs:
           colima start
 
       - name: Install Haystack
-        run: pip install .[dev,preview] langdetect transformers[torch,sentencepiece]==4.32.1 'sentence-transformers>=2.2.0' pypdf openai-whisper tika 'azure-ai-formrecognizer>=3.2.0b2'
+        run: |
+          pip install .[dev,preview] langdetect transformers[torch,sentencepiece]==4.34.1 'sentence-transformers>=2.2.0' pypdf tika 'azure-ai-formrecognizer>=3.2.0b2'
+          pip install --no-deps llvmlite numba 'openai-whisper>=20230918'  # prevent outdated version of tiktoken pinned by openai-whisper
 
       - name: Run Tika
         run: docker run -d -p 9998:9998 apache/tika:2.9.0.0
@@ -282,7 +288,9 @@ jobs:
           python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Haystack
-        run: pip install .[dev,preview] langdetect transformers[torch,sentencepiece]==4.32.1 'sentence-transformers>=2.2.0' pypdf openai-whisper tika 'azure-ai-formrecognizer>=3.2.0b2'
+        run: |
+          pip install .[dev,preview] langdetect transformers[torch,sentencepiece]==4.34.1 'sentence-transformers>=2.2.0' pypdf tika 'azure-ai-formrecognizer>=3.2.0b2'
+          pip install --no-deps llvmlite numba 'openai-whisper>=20230918'  # prevent outdated version of tiktoken pinned by openai-whisper
 
       - name: Run
         run: pytest --maxfail=5 -m "integration" test/preview -k 'not tika'
diff --git a/haystack/nodes/audio/whisper_transcriber.py b/haystack/nodes/audio/whisper_transcriber.py
index 1e95669cd3..d43d3f3e22 100644
--- a/haystack/nodes/audio/whisper_transcriber.py
+++ b/haystack/nodes/audio/whisper_transcriber.py
@@ -28,6 +28,8 @@ class WhisperTranscriber(BaseComponent):
 
     To use Whisper locally, install it following the instructions on
     the Whisper [GitHub repo](https://github.com/openai/whisper) and omit the `api_key` parameter.
+    You can work around a dependency conflict caused by openai-whisper pinning an older tiktoken version than required
+    by Haystack if you install via `pip install --no-deps numba llvmlite 'openai-whisper>=20230918'`.
 
     To use the API implementation, provide an api_key. You can get one by signing up
     for an [OpenAI account](https://beta.openai.com/).
diff --git a/haystack/preview/components/audio/whisper_local.py b/haystack/preview/components/audio/whisper_local.py
index e2f8526277..7b4106cb08 100644
--- a/haystack/preview/components/audio/whisper_local.py
+++ b/haystack/preview/components/audio/whisper_local.py
@@ -6,7 +6,10 @@
 from haystack.preview import component, Document, default_to_dict, ComponentError
 from haystack.preview.lazy_imports import LazyImport
 
-with LazyImport("Run 'pip install openai-whisper'") as whisper_import:
+with LazyImport(
+    "Run 'pip install transformers[torch]==4.34.1' to install torch and "
+    "'pip install --no-deps numba llvmlite 'openai-whisper>=20230918'' to install whisper."
+) as whisper_import:
     import torch
     import whisper
 
diff --git a/haystack/preview/components/rankers/similarity.py b/haystack/preview/components/rankers/similarity.py
index 108b91d256..6bcc8ba00f 100644
--- a/haystack/preview/components/rankers/similarity.py
+++ b/haystack/preview/components/rankers/similarity.py
@@ -8,7 +8,7 @@
 logger = logging.getLogger(__name__)
 
 
-with LazyImport(message="Run 'pip install transformers[torch,sentencepiece]==4.32.1'") as torch_and_transformers_import:
+with LazyImport(message="Run 'pip install transformers[torch,sentencepiece]==4.34.1'") as torch_and_transformers_import:
     import torch
     from transformers import AutoModelForSequenceClassification, AutoTokenizer
 
diff --git a/haystack/preview/components/readers/extractive.py b/haystack/preview/components/readers/extractive.py
index 3d331e81c5..4377edb644 100644
--- a/haystack/preview/components/readers/extractive.py
+++ b/haystack/preview/components/readers/extractive.py
@@ -7,7 +7,7 @@
 from haystack.preview.lazy_imports import LazyImport
 
 with LazyImport(
-    "Run 'pip install transformers[torch,sentencepiece]==4.32.1 sentence-transformers>=2.2.0'"
+    "Run 'pip install transformers[torch,sentencepiece]==4.34.1 sentence-transformers>=2.2.0'"
 ) as torch_and_transformers_import:
     from transformers import AutoModelForQuestionAnswering, AutoTokenizer
     from tokenizers import Encoding
@@ -192,17 +192,17 @@ def _postprocess(
         start_candidates = start_candidates.cpu()
         end_candidates = end_candidates.cpu()
 
-        start_candidates = [
+        start_candidates_char_indices = [
             [encoding.token_to_chars(start)[0] for start in candidates]
             for candidates, encoding in zip(start_candidates, encodings)
         ]
-        end_candidates = [
+        end_candidates_char_indices = [
             [encoding.token_to_chars(end)[1] for end in candidates]
             for candidates, encoding in zip(end_candidates, encodings)
         ]
         probabilities = candidates.values.cpu()
 
-        return start_candidates, end_candidates, probabilities
+        return start_candidates_char_indices, end_candidates_char_indices, probabilities
 
     def _nest_answers(
         self,
diff --git a/pyproject.toml b/pyproject.toml
index d4e9452cbc..6ab456c790 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -49,7 +49,7 @@ dependencies = [
   "requests",
   "httpx",
   "pydantic<2",
-  "transformers==4.32.1",
+  "transformers==4.34.1",
   "pandas",
   "rank_bm25",
   "scikit-learn>=1.3.0", # TF-IDF and metrics
@@ -62,7 +62,6 @@ dependencies = [
   "networkx",  # graphs library
   "quantulum3",  # quantities extraction from text
   "posthog",  # telemetry
-  # audio's espnet-model-zoo requires huggingface-hub version <0.8 while we need >=0.5 to be able to use create_repo in FARMReader
   "tenacity",  # retry decorator
   "sseclient-py",  # server side events for OpenAI streaming
   "more_itertools",  # utilities
@@ -102,7 +101,7 @@ preview = [
   "more-itertools",  # TextDocumentSplitter
 ]
 inference = [
-  "transformers[torch,sentencepiece]==4.32.1",
+  "transformers[torch,sentencepiece]==4.34.1",
   "sentence-transformers>=2.2.0",  # See haystack/nodes/retriever/_embedding_encoder.py, _SentenceTransformersEmbeddingEncoder
   "huggingface-hub>=0.5.0",
 ]
@@ -152,9 +151,6 @@ docstores = [
 docstores-gpu = [
   "farm-haystack[elasticsearch,faiss-gpu,weaviate,pinecone,opensearch]",
 ]
-audio = [
-  "openai-whisper"
-]
 aws = [
   "boto3",
   # Costraint botocore to avoid taking to much time to resolve the dependency tree.
diff --git a/releasenotes/notes/bump-transformers-to-4-34-38d045d8e42ea0a2.yaml b/releasenotes/notes/bump-transformers-to-4-34-38d045d8e42ea0a2.yaml
new file mode 100644
index 0000000000..a147f7ef95
--- /dev/null
+++ b/releasenotes/notes/bump-transformers-to-4-34-38d045d8e42ea0a2.yaml
@@ -0,0 +1,5 @@
+---
+enhancements:
+  - |
+    Upgrade Transformers to the latest version 4.34.1.
+    This version adds support for the new Mistral, Persimmon, BROS, ViTMatte, and Nougat models.
diff --git a/test/modeling/test_model_loading.py b/test/modeling/test_model_loading.py
index c74072f6cc..859ad23d71 100644
--- a/test/modeling/test_model_loading.py
+++ b/test/modeling/test_model_loading.py
@@ -27,7 +27,7 @@ def test_basic_loading(pretrained_model_name_or_path, lm_class, monkeypatch):
 
 @pytest.mark.unit
 def test_basic_loading_unknown_model():
-    with pytest.raises(OSError):
+    with pytest.raises(RuntimeError):
         get_language_model("model_that_doesnt_exist")
 
 
diff --git a/test/preview/components/audio/test_whisper_local.py b/test/preview/components/audio/test_whisper_local.py
index e1b322bcff..9dd0db92db 100644
--- a/test/preview/components/audio/test_whisper_local.py
+++ b/test/preview/components/audio/test_whisper_local.py
@@ -145,7 +145,7 @@ def test_transcribe_stream(self):
     @pytest.mark.integration
     @pytest.mark.skipif(sys.platform in ["win32", "cygwin"], reason="ffmpeg not installed on Windows CI")
     def test_whisper_local_transcriber(self, preview_samples_path):
-        comp = LocalWhisperTranscriber(model_name_or_path="medium")
+        comp = LocalWhisperTranscriber(model_name_or_path="medium", whisper_params={"language": "english"})
         comp.warm_up()
         output = comp.run(
             audio_files=[