diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 00000000..f09f0f71 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,299 @@ +# Benchmark results in environments + +We tested our environments on some safe reinforcement learning algorithms using [OmniSafe](https://github.com/OmniSafeAI/omnisafe) as an out-of-the-box RL framework. The results are shown below. + +## Important notes +- All the algorithms are tested on the same hardware [AMD EPYC 7H12 64-Core Processor]. +- The algorithms are tested with the same hyperparameters which are defaulted in [OmniSafe](https://github.com/OmniSafeAI/omnisafe/tree/main/omnisafe/configs/on-policy). +- Each algorithm is tested on at least 5 seeds and the average performance and standard deviation are shown in the figure. And the five seeds are [0, 5, 10, 15, 20]. We believe that the results are stable. +- The `Goal`, `Push`, and `Button` in `v0` series tasks of Safe Navigation are implemented as the original settings of [Safety-Gym](https://openai.com/research/safety-gym), We just made them more intelligible and accessible. The same situation happens on agents which appear in Safety-Gym. Further, we will fix some defects of design in the original Safety-Gym and release the new version of tasks as `v1` to facilitate research in the SafeRL community. +- The `Racecar` and `Ant` agents are newly added to `Safety-Gymnasium`, We are currently fine-tuning the physical parameters of these agents to make them more realistic and more compatible with various tasks. If you find any issues when using them, please feel free to open an issue, and we are welcome to PRs. + +## Benchmark results +### SafetyAntVelocity-v1(1e6) +
+ +
+
+ +### SafetyAntVelocity-v1(1e7) +
+ +
+
+ +### SafetyHalfCheetahVelocity-v1(1e6) +
+ +
+
+ +### SafetyHalfCheetahVelocity-v1(1e7) +
+ +
+
+ +### SafetyHopperVelocity-v1(1e6) +
+ +
+
+ +### SafetyHopperVelocity-v1(1e7) +
+ +
+
+ +### SafetyHumanoidVelocity-v1(1e6) +
+ +
+
+ +### SafetyHumanoidVelocity-v1(1e7) +
+ +
+
+ +### SafetyWalker2dVelocity-v1(1e6) +
+ +
+
+ +### SafetyWalker2dVelocity-v1(1e7) +
+ +
+
+ +### SafetySwimmerVelocity-v1(1e6) +
+ +
+
+ +### SafetySwimmerVelocity-v1(1e7) +
+ +
+
+ +### SafetyPointGoal0-v0(1e7) +
+ +
+
+ +### SafetyPointGoal1-v0(1e7) +
+ +
+
+ +### SafetyPointGoal2-v0(1e7) +
+ +
+
+ +### SafetyCarGoal0-v0(1e7) +
+ +
+
+ +### SafetyCarGoal1-v0(1e7) +
+ +
+
+ +### SafetyCarGoal2-v0(1e7) +
+ +
+
+ +### SafetyPointButton0-v0(1e7) +
+ +
+
+ +### SafetyPointButton1-v0(1e7) +
+ +
+
+ +### SafetyPointButton2-v0(1e7) +
+ +
+
+ +### SafetyCarButton0-v0(1e7) +
+ +
+
+ +### SafetyCarButton1-v0(1e7) +
+ +
+
+ +### SafetyCarButton2-v0(1e7) +
+ +
+
+ +### SafetyPointPush0-v0(1e7) +
+ +
+
+ +### SafetyPointPush1-v0(1e7) +
+ +
+
+ +### SafetyPointPush2-v0(1e7) +
+ +
+
+ +### SafetyCarPush0-v0(1e7) +
+ +
+
+ +### SafetyCarPush1-v0(1e7) +
+ +
+
+ +### SafetyCarPush2-v0(1e7) +
+ +
+
+ +### SafetyPointCircle0-v0(1e7) +
+ +
+
+ +### SafetyPointCircle1-v0(1e7) +
+ +
+
+ +### SafetyPointCircle2-v0(1e7) +
+ +
+
+ +### SafetyCarCircle0-v0(1e7) +
+ +
+
+ +### SafetyCarCircle1-v0(1e7) +
+ +
+
+ +### SafetyCarCircle2-v0(1e7) +
+ +
+
diff --git a/benchmarks/benchmarks/first_order_ant_1e6.png b/benchmarks/benchmarks/first_order_ant_1e6.png new file mode 100644 index 00000000..b52ce4b0 Binary files /dev/null and b/benchmarks/benchmarks/first_order_ant_1e6.png differ diff --git a/benchmarks/benchmarks/first_order_ant_1e7.png b/benchmarks/benchmarks/first_order_ant_1e7.png new file mode 100644 index 00000000..cdc5233a Binary files /dev/null and b/benchmarks/benchmarks/first_order_ant_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_carbutton0_1e7.png b/benchmarks/benchmarks/first_order_carbutton0_1e7.png new file mode 100644 index 00000000..c1332bda Binary files /dev/null and b/benchmarks/benchmarks/first_order_carbutton0_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_carbutton1_1e7.png b/benchmarks/benchmarks/first_order_carbutton1_1e7.png new file mode 100644 index 00000000..0220dfda Binary files /dev/null and b/benchmarks/benchmarks/first_order_carbutton1_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_carbutton2_1e7.png b/benchmarks/benchmarks/first_order_carbutton2_1e7.png new file mode 100644 index 00000000..80569c0a Binary files /dev/null and b/benchmarks/benchmarks/first_order_carbutton2_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_carcircle0_1e7.png b/benchmarks/benchmarks/first_order_carcircle0_1e7.png new file mode 100644 index 00000000..915c49f1 Binary files /dev/null and b/benchmarks/benchmarks/first_order_carcircle0_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_carcircle1_1e7.png b/benchmarks/benchmarks/first_order_carcircle1_1e7.png new file mode 100644 index 00000000..f4db3e56 Binary files /dev/null and b/benchmarks/benchmarks/first_order_carcircle1_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_carcircle2_1e7.png b/benchmarks/benchmarks/first_order_carcircle2_1e7.png new file mode 100644 index 00000000..fe7a8d5b Binary files /dev/null and b/benchmarks/benchmarks/first_order_carcircle2_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_cargoal0_1e7.png b/benchmarks/benchmarks/first_order_cargoal0_1e7.png new file mode 100644 index 00000000..823a36e7 Binary files /dev/null and b/benchmarks/benchmarks/first_order_cargoal0_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_cargoal1_1e7.png b/benchmarks/benchmarks/first_order_cargoal1_1e7.png new file mode 100644 index 00000000..b9fdc18e Binary files /dev/null and b/benchmarks/benchmarks/first_order_cargoal1_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_cargoal2_1e7.png b/benchmarks/benchmarks/first_order_cargoal2_1e7.png new file mode 100644 index 00000000..e0e67908 Binary files /dev/null and b/benchmarks/benchmarks/first_order_cargoal2_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_carpush0_1e7.png b/benchmarks/benchmarks/first_order_carpush0_1e7.png new file mode 100644 index 00000000..25b0d0e6 Binary files /dev/null and b/benchmarks/benchmarks/first_order_carpush0_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_carpush1_1e7.png b/benchmarks/benchmarks/first_order_carpush1_1e7.png new file mode 100644 index 00000000..1d65575a Binary files /dev/null and b/benchmarks/benchmarks/first_order_carpush1_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_carpush2_1e7.png b/benchmarks/benchmarks/first_order_carpush2_1e7.png new file mode 100644 index 00000000..5b36470d Binary files /dev/null and b/benchmarks/benchmarks/first_order_carpush2_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_halfcheetah_1e6.png b/benchmarks/benchmarks/first_order_halfcheetah_1e6.png new file mode 100644 index 00000000..2a8bcbd8 Binary files /dev/null and b/benchmarks/benchmarks/first_order_halfcheetah_1e6.png differ diff --git a/benchmarks/benchmarks/first_order_halfcheetah_1e7.png b/benchmarks/benchmarks/first_order_halfcheetah_1e7.png new file mode 100644 index 00000000..54a1bf97 Binary files /dev/null and b/benchmarks/benchmarks/first_order_halfcheetah_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_hopper_1e6.png b/benchmarks/benchmarks/first_order_hopper_1e6.png new file mode 100644 index 00000000..17b939ee Binary files /dev/null and b/benchmarks/benchmarks/first_order_hopper_1e6.png differ diff --git a/benchmarks/benchmarks/first_order_hopper_1e7.png b/benchmarks/benchmarks/first_order_hopper_1e7.png new file mode 100644 index 00000000..21609361 Binary files /dev/null and b/benchmarks/benchmarks/first_order_hopper_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_humanoid_1e6.png b/benchmarks/benchmarks/first_order_humanoid_1e6.png new file mode 100644 index 00000000..97b56577 Binary files /dev/null and b/benchmarks/benchmarks/first_order_humanoid_1e6.png differ diff --git a/benchmarks/benchmarks/first_order_humanoid_1e7.png b/benchmarks/benchmarks/first_order_humanoid_1e7.png new file mode 100644 index 00000000..f69faddd Binary files /dev/null and b/benchmarks/benchmarks/first_order_humanoid_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_pointbutton0_1e7.png b/benchmarks/benchmarks/first_order_pointbutton0_1e7.png new file mode 100644 index 00000000..4884b8df Binary files /dev/null and b/benchmarks/benchmarks/first_order_pointbutton0_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_pointbutton1_1e7.png b/benchmarks/benchmarks/first_order_pointbutton1_1e7.png new file mode 100644 index 00000000..823d799e Binary files /dev/null and b/benchmarks/benchmarks/first_order_pointbutton1_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_pointbutton2_1e7.png b/benchmarks/benchmarks/first_order_pointbutton2_1e7.png new file mode 100644 index 00000000..8642f405 Binary files /dev/null and b/benchmarks/benchmarks/first_order_pointbutton2_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_pointcircle0_1e7.png b/benchmarks/benchmarks/first_order_pointcircle0_1e7.png new file mode 100644 index 00000000..e52d89b9 Binary files /dev/null and b/benchmarks/benchmarks/first_order_pointcircle0_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_pointcircle1_1e7.png b/benchmarks/benchmarks/first_order_pointcircle1_1e7.png new file mode 100644 index 00000000..6caae88d Binary files /dev/null and b/benchmarks/benchmarks/first_order_pointcircle1_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_pointcircle2_1e7.png b/benchmarks/benchmarks/first_order_pointcircle2_1e7.png new file mode 100644 index 00000000..9850298c Binary files /dev/null and b/benchmarks/benchmarks/first_order_pointcircle2_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_pointgoal0_1e7.png b/benchmarks/benchmarks/first_order_pointgoal0_1e7.png new file mode 100644 index 00000000..2c59282c Binary files /dev/null and b/benchmarks/benchmarks/first_order_pointgoal0_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_pointgoal1_1e7.png b/benchmarks/benchmarks/first_order_pointgoal1_1e7.png new file mode 100644 index 00000000..0dd191a5 Binary files /dev/null and b/benchmarks/benchmarks/first_order_pointgoal1_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_pointgoal2_1e7.png b/benchmarks/benchmarks/first_order_pointgoal2_1e7.png new file mode 100644 index 00000000..e29f5cd4 Binary files /dev/null and b/benchmarks/benchmarks/first_order_pointgoal2_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_pointpush0_1e7.png b/benchmarks/benchmarks/first_order_pointpush0_1e7.png new file mode 100644 index 00000000..21fd2396 Binary files /dev/null and b/benchmarks/benchmarks/first_order_pointpush0_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_pointpush1_1e7.png b/benchmarks/benchmarks/first_order_pointpush1_1e7.png new file mode 100644 index 00000000..bce801a2 Binary files /dev/null and b/benchmarks/benchmarks/first_order_pointpush1_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_pointpush2_1e7.png b/benchmarks/benchmarks/first_order_pointpush2_1e7.png new file mode 100644 index 00000000..11705ea1 Binary files /dev/null and b/benchmarks/benchmarks/first_order_pointpush2_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_swimmer_1e6.png b/benchmarks/benchmarks/first_order_swimmer_1e6.png new file mode 100644 index 00000000..7948c054 Binary files /dev/null and b/benchmarks/benchmarks/first_order_swimmer_1e6.png differ diff --git a/benchmarks/benchmarks/first_order_swimmer_1e7.png b/benchmarks/benchmarks/first_order_swimmer_1e7.png new file mode 100644 index 00000000..2bc1fa3e Binary files /dev/null and b/benchmarks/benchmarks/first_order_swimmer_1e7.png differ diff --git a/benchmarks/benchmarks/first_order_walker2d_1e6.png b/benchmarks/benchmarks/first_order_walker2d_1e6.png new file mode 100644 index 00000000..71e58cbd Binary files /dev/null and b/benchmarks/benchmarks/first_order_walker2d_1e6.png differ diff --git a/benchmarks/benchmarks/first_order_walker2d_1e7.png b/benchmarks/benchmarks/first_order_walker2d_1e7.png new file mode 100644 index 00000000..112e25e7 Binary files /dev/null and b/benchmarks/benchmarks/first_order_walker2d_1e7.png differ