diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx
index d29ae8472fed..2e4556f8c4c7 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx
@@ -58,9 +58,13 @@ export const CallPage: FC<{
   return <CallPageInnerVertical {...props} call={call.result} />;
 };
 
-const useCallTabs = (call: CallSchema) => {
+export const useShowRunnableUI = () => {
   const viewerInfo = useViewerInfo();
-  const showScores = viewerInfo.loading ? false : viewerInfo.userInfo?.admin;
+  return viewerInfo.loading ? false : viewerInfo.userInfo?.admin;
+};
+
+const useCallTabs = (call: CallSchema) => {
+  const showScores = useShowRunnableUI();
   const codeURI = call.opVersionRef;
   const {entity, project, callId} = call;
   const weaveRef = makeRefCall(entity, project, callId);
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallScoresViewer.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallScoresViewer.tsx
index daa60c951b63..f477332e3f50 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallScoresViewer.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallScoresViewer.tsx
@@ -26,8 +26,6 @@ import {CallSchema} from '../wfReactInterface/wfDataModelHooksInterface';
 
 const RUNNABLE_REF_PREFIX = 'wandb.runnable';
 
-
-
 const useLatestActionDefinitionsForCall = (call: CallSchema) => {
   const actionSpecs = (
     useBaseObjectInstances('ActionSpec', {
@@ -266,7 +264,7 @@ export const CallScoresViewer: React.FC<{
     {
       field: 'run',
       headerName: '',
-      width:75,
+      width: 75,
       rowSpanValueGetter: (value, row) => row.displayName,
       renderCell: params => {
         const actionRef = params.row.runnableActionRef;
@@ -323,7 +321,6 @@ export const CallScoresViewer: React.FC<{
   );
 };
 
-
 const RunButton: React.FC<{
   actionRef: string;
   callId: string;
@@ -353,10 +350,13 @@ const RunButton: React.FC<{
     }
   };
 
-
   if (error) {
     return (
-      <Button variant="destructive" onClick={handleRunClick} disabled style={{width: '55px'}}>
+      <Button
+        variant="destructive"
+        onClick={handleRunClick}
+        disabled
+        style={{width: '55px'}}>
         Error
       </Button>
     );
@@ -364,9 +364,13 @@ const RunButton: React.FC<{
 
   return (
     <div>
-      <Button variant="secondary" onClick={handleRunClick} disabled={isRunning} style={{width: '55px'}}>
+      <Button
+        variant="secondary"
+        onClick={handleRunClick}
+        disabled={isRunning}
+        style={{width: '55px'}}>
         {isRunning ? '...' : 'Run'}
       </Button>
     </div>
   );
-};
\ No newline at end of file
+};
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/NewScorerDrawer.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/NewScorerDrawer.tsx
index 33fbc1f8c172..79887a72bcd3 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/NewScorerDrawer.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/NewScorerDrawer.tsx
@@ -1,8 +1,16 @@
 import {Box, Drawer} from '@material-ui/core';
 import {Button} from '@wandb/weave/components/Button';
 import {Icon, IconName, IconNames} from '@wandb/weave/components/Icon';
-import React, {FC, ReactNode, useCallback, useEffect, useState} from 'react';
-
+import React, {
+  FC,
+  ReactNode,
+  useCallback,
+  useEffect,
+  useMemo,
+  useState,
+} from 'react';
+
+import {useShowRunnableUI} from '../CallPage/CallPage';
 import {TraceServerClient} from '../wfReactInterface/traceServerClient';
 import {useGetTraceServerClientContext} from '../wfReactInterface/traceServerClientContext';
 import {AutocompleteWithLabel} from './FormComponents';
@@ -16,7 +24,7 @@ import {
 const HUMAN_ANNOTATION_LABEL = 'Human annotation';
 export const HUMAN_ANNOTATION_VALUE = 'ANNOTATION';
 const LLM_JUDGE_LABEL = 'LLM judge';
-const LLM_JUDGE_VALUE = 'LLM_JUDGE';
+export const LLM_JUDGE_VALUE = 'LLM_JUDGE';
 const PROGRAMMATIC_LABEL = 'Programmatic scorer';
 const PROGRAMMATIC_VALUE = 'PROGRAMMATIC';
 
@@ -48,7 +56,7 @@ export const scorerTypeRecord: Record<ScorerType, ScorerTypeConfig<any>> = {
     },
   },
   LLM_JUDGE: {
-    label: LLM_JUDGE_LABEL,
+    label: LLM_JUDGE_LABEL + ' (W&B Admin Preview)',
     value: LLM_JUDGE_VALUE,
     icon: IconNames.RobotServiceMember,
     Component: LLMJudgeScorerForm.LLMJudgeScorerForm,
@@ -124,6 +132,12 @@ export const NewScorerDrawer: FC<NewScorerDrawerProps> = ({
   }, [selectedScorerType, entity, project, formData, getClient, onClose]);
 
   const ScorerFormComponent = scorerTypeRecord[selectedScorerType].Component;
+  const showRunnableUI = useShowRunnableUI();
+  const options = useMemo(() => {
+    return scorerTypeOptions.filter(
+      opt => showRunnableUI || opt.value !== LLM_JUDGE_VALUE
+    );
+  }, [showRunnableUI]);
 
   return (
     <SaveableDrawer
@@ -134,8 +148,8 @@ export const NewScorerDrawer: FC<NewScorerDrawerProps> = ({
       saveDisabled={!isFormValid}>
       <AutocompleteWithLabel
         label="Scorer type"
-        options={scorerTypeOptions}
-        value={scorerTypeOptions.find(opt => opt.value === selectedScorerType)}
+        options={options}
+        value={options.find(opt => opt.value === selectedScorerType)}
         formatOptionLabel={option => (
           <Box display="flex" alignItems="center" style={{gap: '4px'}}>
             <Icon name={option.icon} />
diff --git a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/ScorersPage.tsx b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/ScorersPage.tsx
index f7690066e612..adf6ff76816c 100644
--- a/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/ScorersPage.tsx
+++ b/weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ScorersPage/ScorersPage.tsx
@@ -2,6 +2,7 @@ import {Button} from '@wandb/weave/components/Button';
 import {IconNames} from '@wandb/weave/components/Icon';
 import React, {useState} from 'react';
 
+import {useShowRunnableUI} from '../CallPage/CallPage';
 import {SimplePageLayoutWithHeader} from '../common/SimplePageLayout';
 import {AnnotationsTab} from './AnnotationsTab';
 import {ProgrammaticScorersTab} from './CoreScorersTab';
@@ -22,6 +23,8 @@ export const ScorersPage: React.FC<{
     scorerTypeRecord.ANNOTATION.value
   );
 
+  const showRunnableUI = useShowRunnableUI();
+
   return (
     <>
       <SimplePageLayoutWithHeader
@@ -32,11 +35,15 @@ export const ScorersPage: React.FC<{
             icon: scorerTypeRecord.ANNOTATION.icon,
             content: <AnnotationsTab entity={entity} project={project} />,
           },
-          {
-            label: scorerTypeRecord.LLM_JUDGE.label + 's',
-            icon: scorerTypeRecord.LLM_JUDGE.icon,
-            content: <LLMJudgesTab entity={entity} project={project} />,
-          },
+          ...(showRunnableUI
+            ? [
+                {
+                  label: scorerTypeRecord.LLM_JUDGE.label + 's',
+                  icon: scorerTypeRecord.LLM_JUDGE.icon,
+                  content: <LLMJudgesTab entity={entity} project={project} />,
+                },
+              ]
+            : []),
           {
             label: scorerTypeRecord.PROGRAMMATIC.label + 's',
             icon: scorerTypeRecord.PROGRAMMATIC.icon,