feat: add windows deps (#187)

Issue #, if available: *Description of changes:* - Adds all dependencies needed to run lima/finch standalone - This includes a lot of DLLs, need a better way to automate this - TODO: after merge, need to modify Finch to set path when calling Lima so that these executables are actually used *Testing done:* - All exes work with an empty `%PATH%` (e.g. `$env:Path = ""; ssh.exe --help` works) - [x] I've reviewed the guidance in CONTRIBUTING.md #### License Acceptance By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. --------- Signed-off-by: Justin Alvarez <[email protected]> Signed-off-by: Vishwas Siravara <[email protected]>
runfinch · Oct 14, 2023 · f8061d8 · f8061d8
1 parent ef838ce
commit f8061d8
Show file tree

Hide file tree

Showing 3 changed files with 110 additions and 22 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,6 @@
 _output/
 downloads/
+wingit-temp/
 *.idea
 *.DS_Store
 *.bak

diff --git a/Makefile b/Makefile
@@ -6,6 +6,7 @@ HASH_DIR ?= $(CURDIR)/hashes
 DOWNLOAD_DIR := $(CURDIR)/downloads
 OS_DOWNLOAD_DIR := $(DOWNLOAD_DIR)/os
 LIMA_DOWNLOAD_DIR := $(DOWNLOAD_DIR)/dependencies
+LIMA_OUTDIR ?= $(OUTDIR)/lima
 ROOTFS_DOWNLOAD_DIR := $(DOWNLOAD_DIR)/os
 DEPENDENCIES_DOWNLOAD_DIR :=  $(DOWNLOAD_DIR)/dependencies
 SOCKET_VMNET_TEMP_PREFIX ?= $(OUTDIR)/dependencies/lima-socket_vmnet/opt/finch
@@ -28,29 +29,34 @@ FINCH_ROOTFS_x86_DIGEST := $(or $(FINCH_ROOTFS_x86_DIGEST),"sha256:ed36fb7f48196
 LIMA_DEPENDENCY_FILE_NAME ?= lima-and-qemu.tar.gz
 .DEFAULT_GOAL := all
 
+WINGIT_TEMP_DIR := $(CURDIR)/wingit-temp
+WINGIT_x86_URL := $(or $(WINGIT_x86_URL),https://github.com/git-for-windows/git/releases/download/v2.42.0.windows.2/Git-2.42.0.2-64-bit.tar.bz2)
+WINGIT_x86_BASENAME ?= $(notdir $(WINGIT_x86_URL))
+WINGIT_x86_HASH := $(or $(WINGIT_x86_HASH),"sha256:c192e56f8ed3d364acc87ad04d1f5aa6ae03c23b32b67bf65fcc6f9b8f032e65")
+
 ifneq (,$(findstring arm64,$(ARCH)))
-    LIMA_ARCH = aarch64
-    LIMA_URL ?= https://deps.runfinch.com/aarch64/lima-and-qemu.macos-aarch64.1691201350.tar.gz
-    FINCH_OS_BASENAME := $(notdir $(FINCH_OS_AARCH64_URL))
-    FINCH_OS_IMAGE_URL := $(FINCH_OS_AARCH64_URL)
-    FINCH_OS_DIGEST ?= $(FINCH_OS_AARCH64_DIGEST)
-    # TODO: Use Finch rootfs in Finch on Windows testing
-    FINCH_ROOTFS_BASENAME := $(notdir $(FINCH_ROOTFS_AARCH64_URL))
-    FINCH_ROOTFS_URL ?= $(FINCH_ROOTFS_AARCH64_URL)
-    FINCH_ROOTFS_DIGEST ?= $(FINCH_ROOTFS_AARCH64_DIGEST)
-    HOMEBREW_PREFIX ?= /opt/homebrew
+	LIMA_ARCH = aarch64
+	LIMA_URL ?= https://deps.runfinch.com/aarch64/lima-and-qemu.macos-aarch64.1691201350.tar.gz
+	FINCH_OS_BASENAME := $(notdir $(FINCH_OS_AARCH64_URL))
+	FINCH_OS_IMAGE_URL := $(FINCH_OS_AARCH64_URL)
+	FINCH_OS_DIGEST ?= $(FINCH_OS_AARCH64_DIGEST)
+	# TODO: Use Finch rootfs in Finch on Windows testing
+	FINCH_ROOTFS_BASENAME := $(notdir $(FINCH_ROOTFS_AARCH64_URL))
+	FINCH_ROOTFS_URL ?= $(FINCH_ROOTFS_AARCH64_URL)
+	FINCH_ROOTFS_DIGEST ?= $(FINCH_ROOTFS_AARCH64_DIGEST)
+	HOMEBREW_PREFIX ?= /opt/homebrew
 
 else ifneq (,$(findstring x86_64,$(ARCH)))
-    LIMA_ARCH = x86_64
-    LIMA_URL ?= https://deps.runfinch.com/x86-64/lima-and-qemu.macos-x86_64.1691201350.tar.gz
-    FINCH_OS_BASENAME := $(notdir $(FINCH_OS_x86_URL))
-    FINCH_OS_IMAGE_URL := $(FINCH_OS_x86_URL)
-    FINCH_OS_DIGEST ?= $(FINCH_OS_x86_DIGEST)
-    # TODO: Use Finch rootfs in Finch on Windows testing
-    FINCH_ROOTFS_BASENAME := $(notdir $(FINCH_ROOTFS_x86_URL))
-    FINCH_ROOTFS_URL ?= $(FINCH_ROOTFS_x86_URL)
-    FINCH_ROOTFS_DIGEST ?= $(FINCH_ROOTFS_x86_DIGEST)
-    HOMEBREW_PREFIX ?= /usr/local
+	LIMA_ARCH = x86_64
+	LIMA_URL ?= https://deps.runfinch.com/x86-64/lima-and-qemu.macos-x86_64.1691201350.tar.gz
+	FINCH_OS_BASENAME := $(notdir $(FINCH_OS_x86_URL))
+	FINCH_OS_IMAGE_URL := $(FINCH_OS_x86_URL)
+	FINCH_OS_DIGEST ?= $(FINCH_OS_x86_DIGEST)
+	# TODO: Use Finch rootfs in Finch on Windows testing
+	FINCH_ROOTFS_BASENAME := $(notdir $(FINCH_ROOTFS_x86_URL))
+	FINCH_ROOTFS_URL ?= $(FINCH_ROOTFS_x86_URL)
+	FINCH_ROOTFS_DIGEST ?= $(FINCH_ROOTFS_x86_DIGEST)
+	HOMEBREW_PREFIX ?= /usr/local
 
 endif
 
@@ -81,7 +87,7 @@ BUILD_OS ?= $(OS)
 ifeq ($(BUILD_OS), Windows_NT)
 binaries: rootfs lima
 download: download.rootfs
-lima: lima-exe
+lima: lima-exe install.lima-dependencies-wsl2
 FINCH_IMAGE_LOCATION := $(FINCH_ROOTFS_LOCATION)
 FINCH_IMAGE_DIGEST := $(FINCH_ROOTFS_DIGEST)
 else
@@ -121,9 +127,72 @@ $(LIMA_DOWNLOAD_DIR)/$(LIMA_DEPENDENCY_FILE_NAME):
 .PHONY: download.lima-dependencies
 download.lima-dependencies: $(LIMA_DOWNLOAD_DIR)/$(LIMA_DEPENDENCY_FILE_NAME)
 
-.PHONE: install.lima-dependencies
+.PHONY: install.lima-dependencies
 install.lima-dependencies: download.lima-dependencies
 
+# Only redownload/extract if this file is missing (there's no particular reason for choosing this file instead of any other)
+$(LIMA_OUTDIR)/bin/ssh.exe:
+	mkdir -p $(DEPENDENCIES_DOWNLOAD_DIR)
+	mkdir -p $(OUTDIR)/bin
+
+	curl -L --fail $(WINGIT_x86_URL) > $(DEPENDENCIES_DOWNLOAD_DIR)/$(WINGIT_x86_BASENAME)
+	pwsh.exe -NoLogo -NoProfile -c ./verify_hash.ps1 "$(DEPENDENCIES_DOWNLOAD_DIR)\$(WINGIT_x86_BASENAME)" $(WINGIT_x86_HASH)
+	mkdir -p $(WINGIT_TEMP_DIR)
+	# this takes a long time because of an almost 4:1 compression ratio and needing to extract many small files
+	tar -xvjf "$(DEPENDENCIES_DOWNLOAD_DIR)\$(WINGIT_x86_BASENAME)" -C $(WINGIT_TEMP_DIR)
+
+	# Lima runtime dependencies
+	mkdir -p $(LIMA_OUTDIR)/bin
+
+	# From https://packages.msys2.org/package/gzip?repo=msys&variant=x86_64
+	cp $(WINGIT_TEMP_DIR)/usr/bin/gzip.exe $(LIMA_OUTDIR)/bin/
+	# From https://packages.msys2.org/package/msys2-runtime?repo=msys&variant=x86_64
+	cp $(WINGIT_TEMP_DIR)/usr/bin/cygpath.exe $(LIMA_OUTDIR)/bin/
+	# From https://packages.msys2.org/package/tar?repo=msys&variant=x86_64
+	cp $(WINGIT_TEMP_DIR)/usr/bin/tar.exe $(LIMA_OUTDIR)/bin/
+	# From https://packages.msys2.org/package/openssh?repo=msys&variant=x86_64
+	cp $(WINGIT_TEMP_DIR)/usr/bin/ssh.exe $(LIMA_OUTDIR)/bin/
+
+	# Dependency DLLs, extracted with https://github.com/lucasg/Dependencies
+	# Dependencies.exe -chain $(WINGIT_TEMP_DIR)\usr\bin\ssh.exe -depth 3 -json
+	# Depth 3 is only needed for ssh.exe, everything else only needs depth 1
+	# TODO: Automate
+
+	# Required by all MSYS2 programs, from https://github.com/msys2/msys2-runtime
+	cp $(WINGIT_TEMP_DIR)/usr/bin/msys-2.0.dll $(LIMA_OUTDIR)/bin/
+	# Required by tar.exe, from https://packages.msys2.org/package/libiconv?repo=msys&variant=x86_64
+	cp $(WINGIT_TEMP_DIR)/usr/bin/msys-iconv-2.dll $(LIMA_OUTDIR)/bin/
+	# Required by msys-iconv-2.dll, from https://packages.msys2.org/package/libintl?repo=msys&variant=x86_64
+	cp $(WINGIT_TEMP_DIR)/usr/bin/msys-intl-8.dll $(LIMA_OUTDIR)/bin/
+	# GCC exception handling, required for all programs that throw exceptions, from https://packages.msys2.org/package/gcc-libs?repo=msys&variant=x86_64
+	cp $(WINGIT_TEMP_DIR)/usr/bin/msys-gcc_s-seh-1.dll $(LIMA_OUTDIR)/bin/
+
+	# Required by ssh.exe, from https://packages.msys2.org/package/libopenssl?repo=msys&variant=x86_64
+	cp $(WINGIT_TEMP_DIR)/usr/bin/msys-crypto-3.dll $(LIMA_OUTDIR)/bin/
+	# Required by ssh.exe, from https://packages.msys2.org/package/zlib-devel?repo=msys&variant=x86_64
+	cp $(WINGIT_TEMP_DIR)/usr/bin/msys-z.dll $(LIMA_OUTDIR)/bin/
+	# Required by ssh.exe, from https://packages.msys2.org/package/libcrypt?repo=msys&variant=x86_64
+	cp $(WINGIT_TEMP_DIR)/usr/bin/msys-crypt-0.dll $(LIMA_OUTDIR)/bin/
+	# Required by heimdal-libs, from https://packages.msys2.org/package/libsqlite?repo=msys&variant=x86_64
+	cp $(WINGIT_TEMP_DIR)/usr/bin/msys-sqlite3-0.dll $(LIMA_OUTDIR)/bin/
+
+	# Required by ssh.exe, from https://packages.msys2.org/package/heimdal-libs?repo=msys&variant=x86_64
+	cp $(WINGIT_TEMP_DIR)/usr/bin/msys-asn1-8.dll $(LIMA_OUTDIR)/bin/
+	cp $(WINGIT_TEMP_DIR)/usr/bin/msys-com_err-1.dll $(LIMA_OUTDIR)/bin/
+	cp $(WINGIT_TEMP_DIR)/usr/bin/msys-gssapi-3.dll $(LIMA_OUTDIR)/bin/
+	cp $(WINGIT_TEMP_DIR)/usr/bin/msys-hcrypto-4.dll $(LIMA_OUTDIR)/bin/
+	cp $(WINGIT_TEMP_DIR)/usr/bin/msys-heimbase-1.dll $(LIMA_OUTDIR)/bin/
+	cp $(WINGIT_TEMP_DIR)/usr/bin/msys-heimntlm-0.dll $(LIMA_OUTDIR)/bin/
+	cp $(WINGIT_TEMP_DIR)/usr/bin/msys-hx509-5.dll $(LIMA_OUTDIR)/bin/
+	cp $(WINGIT_TEMP_DIR)/usr/bin/msys-krb5-26.dll $(LIMA_OUTDIR)/bin/
+	cp $(WINGIT_TEMP_DIR)/usr/bin/msys-roken-18.dll $(LIMA_OUTDIR)/bin/
+	cp $(WINGIT_TEMP_DIR)/usr/bin/msys-wind-0.dll $(LIMA_OUTDIR)/bin/
+
+	-@rm -rf $(WINGIT_TEMP_DIR)
+
+.PHONY: install.lima-dependencies-wsl2
+install.lima-dependencies-wsl2: $(LIMA_OUTDIR)/bin/ssh.exe
+
 .PHONY: lima-template
 lima-template: download
 	mkdir -p $(OUTDIR)/lima-template

diff --git a/verify_hash.ps1 b/verify_hash.ps1
@@ -0,0 +1,18 @@
+<#
+    Verify file hashes using pure PowerShell
+#>
+param (
+    # Dependency file path
+    [Parameter(Mandatory=$true)]
+    [string]$DependencyFilePath,
+
+    # Dependency hash
+    [Parameter(Mandatory=$true)]
+    [string]$DependencyHash = 'out.png'
+)
+
+if (!(Get-FileHash -Algorithm SHA256 "$DependencyFilePath").Hash -eq $DependencyHash) {
+    $host.SetShouldExit(-1); exit
+} else {
+    Write-Output "Verified $DependencyFilePath"
+}