diff --git a/lib/utils.js b/lib/utils.js
index 025485888..48f79749b 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -340,14 +340,18 @@ class Utils {
         return serverId;
     }
     /**
-     * Return custom server ID if provided, or default server ID otherwise.
+     * Returns the custom server ID if provided, otherwise returns the default server ID.
      */
     static getCustomOrDefaultServerId() {
+        const customServerId = this.getInputtedCustomId();
+        return customServerId || this.getRunDefaultServerId();
+    }
+    static getInputtedCustomId() {
         let customServerId = core.getInput(Utils.CUSTOM_SERVER_ID);
         if (customServerId) {
             return customServerId;
         }
-        return Utils.getRunDefaultServerId();
+        return undefined;
     }
     /**
      * Return the default server ID for JFrog CLI server configuration.
@@ -407,15 +411,26 @@ class Utils {
         });
     }
     /**
-     * Removed configured JFrog CLI servers that are saved in the servers env var, and unset the env var.
+     * Removes configured JFrog CLI servers saved in the environment variable.
+     * If a custom server ID is defined, only remove the custom server ID.
      */
     static removeJFrogServers() {
         return __awaiter(this, void 0, void 0, function* () {
-            for (const serverId of Utils.getConfiguredJFrogServers()) {
-                core.debug(`Removing server ID: '${serverId}'...`);
-                yield Utils.runCli(['c', 'rm', serverId, '--quiet']);
+            const customServerId = this.getInputtedCustomId();
+            core.info(`The value of custom is: '${customServerId}'`);
+            if (customServerId) {
+                // Remove only the custom server ID
+                core.debug(`Removing custom server ID: '${customServerId}'...`);
+                yield Utils.runCli(['c', 'rm', customServerId, '--quiet']);
+            }
+            else {
+                // Remove all configured server IDs
+                for (const serverId of Utils.getConfiguredJFrogServers()) {
+                    core.debug(`Removing server ID: '${serverId}'...`);
+                    yield Utils.runCli(['c', 'rm', serverId, '--quiet']);
+                }
+                core.exportVariable(Utils.JFROG_CLI_SERVER_IDS_ENV_VAR, '');
             }
-            core.exportVariable(Utils.JFROG_CLI_SERVER_IDS_ENV_VAR, '');
         });
     }
     /**
diff --git a/src/utils.ts b/src/utils.ts
index c5907a8cc..487afdd70 100644
--- a/src/utils.ts
+++ b/src/utils.ts
@@ -387,14 +387,19 @@ export class Utils {
     }
 
     /**
-     * Return custom server ID if provided, or default server ID otherwise.
+     * Returns the custom server ID if provided, otherwise returns the default server ID.
      */
     private static getCustomOrDefaultServerId(): string {
+        const customServerId: string | undefined = this.getInputtedCustomId();
+        return customServerId || this.getRunDefaultServerId();
+    }
+
+    private static getInputtedCustomId(): string | undefined {
         let customServerId: string = core.getInput(Utils.CUSTOM_SERVER_ID);
         if (customServerId) {
             return customServerId;
         }
-        return Utils.getRunDefaultServerId();
+        return undefined;
     }
 
     /**
@@ -467,14 +472,25 @@ export class Utils {
     }
 
     /**
-     * Removed configured JFrog CLI servers that are saved in the servers env var, and unset the env var.
+     * Removes configured JFrog CLI servers saved in the environment variable.
+     * If a custom server ID is defined, only remove the custom server ID.
      */
     public static async removeJFrogServers() {
-        for (const serverId of Utils.getConfiguredJFrogServers()) {
-            core.debug(`Removing server ID: '${serverId}'...`);
-            await Utils.runCli(['c', 'rm', serverId, '--quiet']);
+        const customServerId: string | undefined = this.getInputtedCustomId();
+        core.info(`The value of custom is: '${customServerId}'`);
+
+        if (customServerId) {
+            // Remove only the custom server ID
+            core.debug(`Removing custom server ID: '${customServerId}'...`);
+            await Utils.runCli(['c', 'rm', customServerId, '--quiet']);
+        } else {
+            // Remove all configured server IDs
+            for (const serverId of Utils.getConfiguredJFrogServers()) {
+                core.debug(`Removing server ID: '${serverId}'...`);
+                await Utils.runCli(['c', 'rm', serverId, '--quiet']);
+            }
+            core.exportVariable(Utils.JFROG_CLI_SERVER_IDS_ENV_VAR, '');
         }
-        core.exportVariable(Utils.JFROG_CLI_SERVER_IDS_ENV_VAR, '');
     }
 
     /**
diff --git a/test/main.spec.ts b/test/main.spec.ts
index ce4a64446..7fdbc78a2 100644
--- a/test/main.spec.ts
+++ b/test/main.spec.ts
@@ -6,7 +6,6 @@ import semver = require('semver/preload');
 jest.mock('os');
 jest.mock('@actions/core');
 jest.mock('semver');
-
 const DEFAULT_CLI_URL: string = 'https://releases.jfrog.io/artifactory/jfrog-cli/';
 const CUSTOM_CLI_URL: string = 'http://127.0.0.1:8081/artifactory/jfrog-cli-remote/';
 // Config in JFrog CLI 1.46.3 and below
@@ -425,3 +424,37 @@ describe('isJobSummarySupported', () => {
         expect(semver.gte).toHaveBeenCalledWith(version, MIN_CLI_VERSION_JOB_SUMMARY);
     });
 });
+
+describe('Utils.removeJFrogServers', () => {
+    beforeEach(() => {
+        jest.clearAllMocks();
+    });
+
+    it('should remove only the custom server ID if defined', async () => {
+        const customServerId: string = 'custom-server-id';
+        jest.spyOn(Utils as any, 'getInputtedCustomId').mockReturnValue(customServerId);
+        jest.spyOn(Utils as any, 'runCli').mockResolvedValue(undefined);
+
+        await Utils.removeJFrogServers();
+
+        expect(core.info).toHaveBeenCalledWith(`The value of custom is: '${customServerId}'`);
+        expect(core.debug).toHaveBeenCalledWith(`Removing custom server ID: '${customServerId}'...`);
+        expect(Utils.runCli).toHaveBeenCalledWith(['c', 'rm', customServerId, '--quiet']);
+    });
+
+    it('should remove all configured server IDs if no custom server ID is defined', async () => {
+        jest.spyOn(Utils as any, 'getInputtedCustomId').mockReturnValue(undefined);
+        const serverIds: string[] = ['server1', 'server2'];
+        jest.spyOn(Utils as any, 'getConfiguredJFrogServers').mockReturnValue(serverIds);
+        jest.spyOn(Utils as any, 'runCli').mockResolvedValue(undefined);
+
+        await Utils.removeJFrogServers();
+
+        expect(core.info).toHaveBeenCalledWith(`The value of custom is: 'undefined'`);
+        for (const serverId of serverIds) {
+            expect(core.debug).toHaveBeenCalledWith(`Removing server ID: '${serverId}'...`);
+            expect(Utils.runCli).toHaveBeenCalledWith(['c', 'rm', serverId, '--quiet']);
+        }
+        expect(core.exportVariable).toHaveBeenCalledWith(Utils.JFROG_CLI_SERVER_IDS_ENV_VAR, '');
+    });
+});