Merge branch 'main' into origin/remove-extra-catch2

.github/workflows/cicd-release-validation.yml

@@ -118,7 +118,7 @@ jobs:
     uses: ./.github/workflows/reusable-test.yml
     with:
       pre_test: Invoke-WebRequest https://github.com/Alan-Jowett/bpf_conformance/releases/download/v0.0.6/bpf_conformance_runner.exe -OutFile bpf_conformance_runner.exe
-      test_command: .\bpf_conformance_runner.exe --test_file_directory %SOURCE_ROOT%\external\ebpf-verifier\external\bpf_conformance\tests --cpu_version v4 --exclude_regex local --plugin_path bpf2c_plugin.exe --debug true --plugin_options "--include %SOURCE_ROOT%\include"
+      test_command: .\bpf_conformance_runner.exe --test_file_directory %SOURCE_ROOT%\external\ebpf-verifier\external\bpf_conformance\tests --cpu_version v4 --plugin_path bpf2c_plugin.exe --debug true --plugin_options "--include %SOURCE_ROOT%\include"
       name: bpf2c_conformance
       build_artifact: Build-x64
       environment: windows-2022

.github/workflows/cicd.yml

@@ -186,7 +186,7 @@ jobs:
     uses: ./.github/workflows/reusable-test.yml
     with:
       pre_test: Invoke-WebRequest https://github.com/Alan-Jowett/bpf_conformance/releases/download/v0.0.6/bpf_conformance_runner.exe -OutFile bpf_conformance_runner.exe
-      test_command: .\bpf_conformance_runner.exe --test_file_directory %SOURCE_ROOT%\external\ebpf-verifier\external\bpf_conformance\tests --cpu_version v4 --exclude_regex local --plugin_path bpf2c_plugin.exe --debug true --plugin_options "--include %SOURCE_ROOT%\include"
+      test_command: .\bpf_conformance_runner.exe --test_file_directory %SOURCE_ROOT%\external\ebpf-verifier\external\bpf_conformance\tests --cpu_version v4 --plugin_path bpf2c_plugin.exe --debug true --plugin_options "--include %SOURCE_ROOT%\include"
       name: bpf2c_conformance
       build_artifact: Build-x64
       environment: windows-2022

.github/workflows/dependency-review.yml

@@ -25,6 +25,6 @@ jobs:
           egress-policy: audit
 
       - name: 'Checkout Repository'
-        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@5a2ce3f5b92ee19cbb1541a4984c76d921601d7c # v4.3.4
+        uses: actions/dependency-review-action@4081bf99e2866ebe428fc0477b69eb4fcda7220a # v4.4.0

.github/workflows/nuget_update.yaml

@@ -37,14 +37,14 @@ jobs:
       with:
         egress-policy: audit
 
-    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
       with:
         # Only check out main repo, not submodules.
         ref: ${{ github.event.workflow_run.head_branch }}
 
 
     - name: Cache nuget packages
-      uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8
+      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a
       env:
         cache-name: cache-nuget-modules
       with:

.github/workflows/ossar-scan.yml

@@ -47,7 +47,7 @@ jobs:
         paths_ignore: '["**.md", "**/docs/**"]'
 
     # Checking out the branch is needed to correctly log security alerts.
-    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
       if: steps.skip_check.outputs.should_skip != 'true'
       with:
         # Only check out main repo, not submodules.
@@ -81,6 +81,6 @@ jobs:
 
     - name: Upload results to Security tab
       if: steps.skip_check.outputs.should_skip != 'true'
-      uses: github/codeql-action/upload-sarif@f779452ac5af1c261dce0346a8f964149f49322b
+      uses: github/codeql-action/upload-sarif@662472033e021d55d94146f66f6058822b0b39fd
       with:
         sarif_file: ${{ steps.ossar.outputs.sarifFile }}

.github/workflows/reusable-build.yml

@@ -117,7 +117,7 @@ jobs:
         powershell.exe "echo 'ASAN_WIN_CONTINUE_ON_INTERCEPTION_FAILURE=true' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append"
         powershell.exe "echo 'VCINSTALLDIR=%VCINSTALLDIR%' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append"
 
-    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
       if: steps.skip_check.outputs.should_skip != 'true'
       with:
         repository: ${{inputs.repository}}
@@ -135,7 +135,7 @@ jobs:
 
     - name: Initialize CodeQL
       if: inputs.build_codeql == true && steps.skip_check.outputs.should_skip != 'true'
-      uses: github/codeql-action/init@f779452ac5af1c261dce0346a8f964149f49322b
+      uses: github/codeql-action/init@662472033e021d55d94146f66f6058822b0b39fd
       with:
         languages: 'cpp'
 
@@ -154,7 +154,7 @@ jobs:
 
     - name: Cache nuget packages
       if: steps.skip_check.outputs.should_skip != 'true'
-      uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8
+      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a
       env:
         cache-name: cache-nuget-modules
       with:
@@ -164,7 +164,7 @@ jobs:
     - name: Cache verifier project
       # The hash is based on the HEAD of the ebpf-verifier submodule, the Directory.Build.props file, and the build variant.
       if: steps.skip_check.outputs.should_skip != 'true'
-      uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8
+      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a
       env:
         cache-name: cache-verifier-project
       with:
@@ -294,4 +294,4 @@ jobs:
 
     - name: Perform CodeQL Analysis
       if: inputs.build_codeql == true && steps.skip_check.outputs.should_skip != 'true'
-      uses: github/codeql-action/analyze@f779452ac5af1c261dce0346a8f964149f49322b
+      uses: github/codeql-action/analyze@662472033e021d55d94146f66f6058822b0b39fd

.github/workflows/reusable-test.yml

@@ -100,15 +100,15 @@ jobs:
         paths_ignore: '["**.md", "**/docs/**"]'
 
     # Checking out the branch is needed to gather correct code coverage data.
-    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
       # Only check out source code if code coverage is being gathered.
       if: (inputs.code_coverage == true) && (steps.skip_check.outputs.should_skip != 'true')
       with:
         submodules: 'recursive'
         ref: ${{ github.event.workflow_run.head_branch }}
 
     # Perform shallow checkout for self-hosted runner.
-    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
       if: (inputs.environment == 'ebpf_cicd_tests_ws2019' || inputs.environment == 'ebpf_cicd_tests_ws2022' || inputs.environment == 'ebpf_cicd_perf_ws2022') && (steps.skip_check.outputs.should_skip != 'true')
       with:
         ref: ${{ github.event.workflow_run.head_branch }}
@@ -123,7 +123,7 @@ jobs:
         files: .github/workflows/reusable-test.yml
 
     # Check out just this file if code hasn't been checked out yet.
-    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
       if: (steps.check_reusable_test_locally.outputs.files_exists != 'true') && (steps.skip_check.outputs.should_skip != 'true')
       with:
         sparse-checkout: |
@@ -142,7 +142,7 @@ jobs:
       # Add cache entry for any choco packages that are installed.
       # The cache key is based on the hash of this file so if any choco packages are added or removed, the cache will be invalidated.
       if: (inputs.gather_dumps == true) && (steps.skip_check.outputs.should_skip != 'true')
-      uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8
+      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a
       env:
         cache-name: cache-choco-packages
       with:

.github/workflows/scorecards-analysis.yml

@@ -45,7 +45,7 @@ jobs:
 
       - name: "Checkout code"
         if: github.ref_name == 'main'
-        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
         with:
           persist-credentials: false
 
@@ -76,6 +76,6 @@ jobs:
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
         if: github.ref_name == 'main'
-        uses: github/codeql-action/upload-sarif@f779452ac5af1c261dce0346a8f964149f49322b
+        uses: github/codeql-action/upload-sarif@662472033e021d55d94146f66f6058822b0b39fd
         with:
           sarif_file: results.sarif

.github/workflows/update-docs.yml

@@ -41,7 +41,7 @@ jobs:
       run: |
         sudo apt install doxygen
 
-    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
 
     - name: Clone docs
       run: |

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "external/ebpf-verifier"]
 	path = external/ebpf-verifier
-	url = https://github.com/vbpf/ebpf-verifier.git
+	url = https://github.com/dthaler/ebpf-verifier.git
 [submodule "external/ubpf"]
 	path = external/ubpf
 	url = https://github.com/iovisor/ubpf.git

docs/BpfSyscallCompatibility.md

@@ -0,0 +1,126 @@
+# `bpf()` syscall compatibility with Linux ABI
+
+There is a desire to have first-class eBPF support for Windows for the Go ecosystem.
+The Go community prefers for libraries that can be easily cross-compiled and distributed.
+However, calling into libbpf via Go's foreign function interface (CGo) goes against this spirit, as it requires distributing C headers, a C toolchain, and other dependencies.
+
+To address this, one of the most popular ways to interact with eBPF on Linux is through the [ebpf-go library], which intentionally avoids CGo.
+Given these considerations, this proposal explores what is necessary to port the ebpf-go library to the eBPF for Windows runtime.
+Such a port would provide a solution that aligns with Go's cross-platform philosophy while offering robust eBPF support for Windows.
+
+## ebpf-go architecture
+
+ebpf-go uses BTF from a Linux kernel image to generate syscall bindings.
+This happens at development time, and the bindings are checked in.
+These are used from the rest of the library to provide an "idiomatic" public API.
+
+```mermaid
+flowchart LR
+    public[public API] --> sys[syscall bindings]
+    sys --> bpf["syscall(SYS_BPF)"]
+    subgraph auto[auto-generated from BTF]
+        sys
+    end
+```
+
+The library follows the same ELF conventions as libbpf so you are able to load the same ELF files.
+The API however is [completely distinct][architecture] and doesn't follow libbpf semantics.
+
+## ebpf-go on Windows
+
+The goal of porting ebpf-go is to end up with a library that facilitates porting programs written for Linux to Windows.
+In the best case the public API stays the same regardless of operating system.
+
+A natural point to start a port is to replace the auto-generated syscall bindings with a Windows-specific wrapper.
+[An attempt][stable ioctl] was made to directly issue `DeviceIoControl` from ebpf-go.
+This approach was discarded because the exact format of the request and response buffers is currently an implementation detail, which is a good thing considering that the current approach [has performance implications][ioctl allocs].
+It also forces us to reimplement the scheme used to load signed native images, which might also still change.
+
+Instead of replacing the auto-generated syscall bindings we can replace the `bpf()` syscall on Windows with an existing [wrapper][bpf wrapper].
+This has a number of benefits:
+
+- The necessary changes are much smaller since we don't need to write new code to generate bindings for Windows.
+- We don't need to add a new stable API to the Windows runtime and instead can reuse the existing `bpf()` emulation.
+
+For this to work we need the `bpf()` emulation to be binary compatible with the Linux syscall.
+Otherwise the auto generated bindings in ebpf-go won't work, negating one of the key benefits of this approach.
+This is problematic since so far eBPF for Windows has only targeted source-level compatibility, not binary compatibility.
+
+### Differing `bpf_cmd_id` values
+
+The values of `bpf_cmd_id` do not match Linux, which leads to ebpf-go performing the wrong syscalls.
+
+### Differing field size
+
+On Windows the length of an object's name is restricted to 64 characters, while Linux only allows 16.
+Unfortunately this constant is used to size an array embedded in key structures like `bpf_map_info` and `bpf_prog_info`:
+
+```C
+struct bpf_map_info
+{
+    // ...
+    char name[BPF_OBJ_NAME_LEN]; ///< Null-terminated map name.
+    // ...
+```
+
+The result is that fields after `name` have a different offset.
+
+### Differing field order / presence
+
+Sometimes the field order doesn't match. For example, `id` and `type` are swapped in `bpf_map_info`.
+
+```C
+struct bpf_map_info
+{
+    // Cross-platform fields.
+    ebpf_id_t id;                ///< Map ID.
+    ebpf_map_type_t type;        ///< Type of map.
+    // ...
+```
+
+Some unsupported fields are missing completely:
+
+```C
+struct bpf_prog_info
+{
+    ebpf_id_t id;                ///< Program ID.
+    enum bpf_prog_type type;     ///< Program type, if a cross-platform type.
+    // 8 missing fields
+    uint32_t nr_map_ids;         ///< Number of maps associated with this program.
+```
+
+### Additional fields
+
+Some structures contain Windows-specific fields:
+
+```C
+struct bpf_map_info
+{
+    // Cross-platform fields.
+    ebpf_id_t id;                ///< Map ID.
+    // ...
+
+    // Windows-specific fields.
+    ebpf_id_t inner_map_id;     ///< ID of inner map template.
+    uint32_t pinned_path_count; ///< Number of pinned paths.
+};
+```
+
+This is problematic because Linux exposes many more fields in `bpf_map_info`, aliasing with the Windows-specific fields.
+Adding a new cross-platform field before the Windows-specific fields will break compatibility in several ways.
+
+## Resolve ABI incompatibilities in the `bpf()` syscall emulation
+
+Changing the problematic types globally is possible but undesirable, since it will cause problems for existing users of the C API.
+Instead we will restrict the necessary changes to the `bpf()` syscall emulation.
+A separate header will contain type definitions compatible with the Linux ABI.
+The syscall emulation layer is then responsible for translating between the Linux
+and native types.
+This localises breakage to users of `bpf()` and allows the C API to evolve on its own.
+
+[ebpf-go library]: https://ebpf-go.dev
+[architecture]: https://ebpf-go.dev/contributing/architecture/
+[stable ioctl]: https://github.com/microsoft/ebpf-for-windows/issues/3700
+[ioctl allocs]: https://github.com/microsoft/ebpf-for-windows/issues/3726
+[bpf wrapper]: https://github.com/microsoft/ebpf-for-windows/blob/main/libs/api/bpf_syscall.cpp
+[api tests]: https://github.com/microsoft/ebpf-for-windows/issues/3729#issuecomment-2259330472

docs/GettingStarted.md

@@ -53,7 +53,7 @@ Alternative install steps (for *basic* Visual Studio Community edition):
    Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
    ```
 
-1. Run the following command to automatically setup the dev environment:
+1. Run the following command to automatically set up the dev environment:
 
    ```ps
    Invoke-WebRequest 'https://raw.githubusercontent.com/microsoft/ebpf-for-windows/main/scripts/Setup-DevEnv.ps1' -OutFile $env:TEMP\Setup-DeveEnv.ps1
@@ -596,4 +596,4 @@ development cycle.  See the [Windows Hardware Developer documentation](https://l
 Extensions that integrate with eBPF for Windows:
 - XDP extension: [xdp-for-windows](https://github.com/microsoft/xdp-for-windows)
 - ntosebpfext extension: [ntosebpfext](https://github.com/microsoft/ntosebpfext)
-- Network event extension: [neteventebpfext](https://github.com/microsoft/ntosebpfext/blob/main/docs/neteventebpfext.md)
+- Network event extension: [neteventebpfext](https://github.com/microsoft/ntosebpfext/blob/main/docs/neteventebpfext.md)

docs/isa-support.rst

@@ -103,7 +103,7 @@ opcode  src_reg  offset  imm   description
 0x7f    any      0       0x00  dst >>= src                                             Y        Y     Y      rsh-reg
 0x84    0x0      0       0x00  dst = (u32)-dst                                         Y        Y     Y      neg
 0x85    0x0      0       any   call helper function by static ID                       Y        Y     Y      call_unwind_fail
-0x85    0x1      0       any   call PC += imm                                          no       no    no     call_local
+0x85    0x1      0       any   call PC += imm                                          Y        part  Y      call_local
 0x85    0x2      0       any   call helper function by BTF ID                          no       no    no     ???
 0x87    0x0      0       0x00  dst = -dst                                              Y        Y     Y      neg64
 0x8d    0x0      0       0x00  call helper function by static ID in register           Y        Y     no     callx

ebpfcore/usersim/EbpfCore_Usersim.vcxproj

@@ -207,8 +207,8 @@
     <FilesToPackage Include="$(TargetPath)" />
   </ItemGroup>
   <ItemGroup>
-    <ProjectReference Include="..\..\external\usersim\cxplat\src\cxplat_winkernel\cxplat_winkernel.vcxproj">
-      <Project>{1ebe3966-7dc4-49b4-b840-3d33d63415ec}</Project>
+    <ProjectReference Include="..\..\external\usersim\cxplat\src\cxplat_winuser\cxplat_winuser.vcxproj">
+      <Project>{f2ca70ab-af9a-47d1-9da9-94d5ab573ac2}</Project>
     </ProjectReference>
     <ProjectReference Include="..\..\external\usersim\src\usersim.vcxproj">
       <Project>{030a7ac6-14dc-45cf-af34-891057ab1402}</Project>
@@ -236,4 +236,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>

ebpfcore/usersim/EbpfCore_Usersim.vcxproj.filters

@@ -35,4 +35,9 @@
       <Filter>Header Files</Filter>
     </ClInclude>
   </ItemGroup>
-</Project>
+  <ItemGroup>
+    <ResourceCompile Include="$(SolutionDir)resource\ebpf_resource.rc">
+      <Filter>Resource Files</Filter>
+    </ResourceCompile>
+  </ItemGroup>
+</Project>