diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f565f7..6955e01 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,13 +8,13 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: - go-version: "1.20" + go-version: "1.23" - name: golangci-lint - uses: golangci/golangci-lint-action@v3 + uses: golangci/golangci-lint-action@v6 with: - version: v1.55 + version: v1.61 env: VERBOSE: "true" @@ -23,9 +23,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: - go-version: "1.20" + go-version: "1.23" - name: Check if mods are tidy run: make check-tidy @@ -34,9 +34,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: - go-version: "1.20" + go-version: "1.23" - name: Run tests run: make test env: diff --git a/.gitignore b/.gitignore index 1c59c21..1149c77 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ .DS_Store .envrc Formula/* -Gemfile.lock bin builds sources diff --git a/.golangci.yml b/.golangci.yml index 1e1671d..5081642 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -7,7 +7,6 @@ linters-settings: gocyclo: min-complexity: 20 govet: - check-shadowing: true enable-all: true disable: - fieldalignment @@ -23,9 +22,9 @@ linters: disable-all: true enable: - bodyclose + - copyloopvar - dupl - errcheck - - exportloopref - funlen - gochecknoinits - goconst @@ -72,12 +71,12 @@ issues: - source: "`yaml:" linters: - lll - -run: - skip-dirs: + exclude-dirs: - builds - sources - tarballs + +run: timeout: 2m allow-parallel-runners: true modules-download-mode: readonly diff --git a/Gemfile b/Gemfile index b81a189..ae928cd 100644 --- a/Gemfile +++ b/Gemfile @@ -5,8 +5,5 @@ source 'http://rubygems.org/' gem 'ruby-macho' group :development do - gem 'byebug' gem 'rubocop' - gem 'rubocop-daemon' - gem 'solargraph', '~> 0.39.17' end diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..c54c16c --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,40 @@ +GEM + remote: http://rubygems.org/ + specs: + ast (2.4.2) + json (2.8.2) + language_server-protocol (3.17.0.3) + parallel (1.26.3) + parser (3.3.6.0) + ast (~> 2.4.1) + racc + racc (1.8.1) + rainbow (3.1.1) + regexp_parser (2.9.2) + rubocop (1.68.0) + json (~> 2.3) + language_server-protocol (>= 3.17.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.4, < 3.0) + rubocop-ast (>= 1.32.2, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.36.1) + parser (>= 3.3.1.0) + ruby-macho (4.1.0) + ruby-progressbar (1.13.0) + unicode-display_width (2.6.0) + +PLATFORMS + arm64-darwin + ruby + x86_64-darwin + +DEPENDENCIES + rubocop + ruby-macho + +BUNDLED WITH + 2.5.23 diff --git a/Makefile b/Makefile index 3c4b4bb..8ce9359 100644 --- a/Makefile +++ b/Makefile @@ -48,10 +48,12 @@ SHELL := env \ bootstrap: bootstrap-brew bootstrap-ruby bootstrap-ruby: - bundle install + env BUNDLE_WITHOUT=development bundle install bootstrap-brew: +ifndef IN_NIX_SHELL brew bundle --verbose +endif bootstrap-pip: $(PIP) install -r requirements-ci.txt @@ -69,7 +71,7 @@ $(TOOLDIR)/$(1): Makefile endef $(eval $(call tool,gofumpt,mvdan.cc/gofumpt@latest)) -$(eval $(call tool,golangci-lint,github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55)) +$(eval $(call tool,golangci-lint,github.com/golangci/golangci-lint/cmd/golangci-lint@v1.61)) $(eval $(call tool,gomod,github.com/Helcaraxan/gomod@latest)) .PHONY: tools @@ -132,6 +134,21 @@ format: $(TOOLDIR)/gofumpt gen: go generate $$(go list ./... | grep -v 'sources/' | grep -v 'builds/') +.PHONY: nix-flake-update +nix-flake-update: + nix flake update \ + && $(MAKE) flake-package-versions.txt + +.SILENT: flake-package-versions +flake-package-versions: + nix develop --command -- bash -c \ + 'nix derivation show \ + $$(echo $$PATH | tr ":" "\n" | grep "/nix/store" | sort -u) \ + | jq -r ".[].name" | sort -u' + +flake-package-versions.txt: flake.nix flake.lock + $(MAKE) flake-package-versions > flake-package-versions.txt + # # Dependencies # diff --git a/README.md b/README.md index 9be1227..e245639 100644 --- a/README.md +++ b/README.md @@ -36,18 +36,40 @@ The build produced does have some limitations: ## Requirements +Required with both Nix and Homebrew approaches: + - [Xcode](https://apps.apple.com/gb/app/xcode/id497799835?mt=12) -- [Homebrew](https://brew.sh/) -- Ruby 2.3.0 or later is needed to execute the build script itself. macOS comes - with Ruby, check your version with `ruby --version`. If it's too old, you can - install a newer version with: - ``` - brew install ruby - ``` -- All dependencies can all easily be installed by running: - ``` - make bootstrap - ``` + +### Nix + +The [Nix](https://nixos.org/) package manager is the preferred and most reliable +way to install all dependencies required to build Emacs, by way of a Nix flake +included in the project root. + +To install all required dependencies within the nix shell, run: + +``` +nix develop --command make bootstrap +``` + +### Homebrew + +If you do not have Nix installed, then the alternative way to manage and install +build-time dependencies is via [Homebrew](https://brew.sh/). + +Ruby 3.3.x or later is also needed to execute the build script. Earlier versions +may work, but are untested. Simplest way to install a recent Ruby version is via +Homebrew: + +``` +brew install ruby +``` + +And finally, to install all built-time dependencies, run: + +``` +make bootstrap +``` ## Status @@ -71,6 +93,24 @@ Nightly builds are built with GitHub Actions on GitHub-hosted runners, using ## Usage +### Nix + +Ensure [Flakes](https://nixos.wiki/wiki/Flakes) are enabled, and enter the flake +development environment with `nix develop`. Within this environment, you can +execute the `./build-emacs-for-macos --help` to get started. + +Or you can run the build script via `nix develop`: + +``` +nix develop --command ./build-emacs-for-macos --help +``` + +### Homebrew + +Run `make boostrap` to ensure all Ruby and Homebrew dependencies are installed. + +### Build Script + ``` Usage: ./build-emacs-for-macos [options] @@ -78,10 +118,13 @@ Branch, tag, and SHA are from the emacs-mirror/emacs/emacs Github repo, available here: https://github.com/emacs-mirror/emacs Options: + --info Print environment info and detected library paths, then exit + --preview Print preview details about build and exit. -j, --parallel COUNT Compile using COUNT parallel processes (detected: 16) --git-sha SHA Override detected git SHA of specified branch allowing builds of old commits + --[no-]use-nix Use Nix instead of Homebrew to find dependencies (default: enabled if EMACS_BUILD_USE_NIX is set to truthy value) --[no-]xwidgets Enable/disable XWidgets if supported (default: enabled) - --[no-]tree-sitter Enable/disable tree-sitter if supported (default: enabled) + --[no-]tree-sitter Enable/disable tree-sitter if supported(default: enabled) --[no-]native-comp Enable/disable native-comp (default: enabled if supported) --[no-]native-march Enable/disable -march=native CFLAG(default: disabled) --[no-]native-full-aot Enable/disable NATIVE_FULL_AOT / Ahead of Time compilation (default: disabled) diff --git a/build-emacs-for-macos b/build-emacs-for-macos index 8b3a7b3..e0b44d5 100755 --- a/build-emacs-for-macos +++ b/build-emacs-for-macos @@ -96,17 +96,26 @@ class OS @version ||= OSVersion.new end + def self.sdk_version + @sdk_version ||= SDKVersion.new + end + def self.arch @arch ||= `uname -m`.strip end end -class OSVersion +class AbstractVersion + attr_reader :version + def initialize - @version = - `sw_vers -productVersion`.match( - /(?\d+)(?:\.(?\d+)(?:\.(?\d+))?)?/ - ) + @version = load_version.match( + /(?\d+)(?:\.(?\d+)(?:\.(?\d+))?)?/ + ) + end + + def load_version + raise NotImplementedError end def to_s @@ -126,6 +135,21 @@ class OSVersion end end +class OSVersion < AbstractVersion + def load_version + `sw_vers -productVersion`.strip + end +end + +class SDKVersion < AbstractVersion + def load_version + ENV.fetch( + 'MACOSX_DEPLOYMENT_TARGET', + `xcrun --show-sdk-version 2>/dev/null`.strip + ).strip + end +end + class Build include Output include System @@ -142,12 +166,12 @@ class Build @root_dir = root_dir @ref = ref || 'master' @options = options - @gcc_info = GccInfo.new - end + @gcc_info = GccInfo.new(use_nix: options[:use_nix]) - def build load_plan(options[:plan]) if options[:plan] + end + def build unless meta[:sha] && meta[:date] fatal 'Failed to get commit info from GitHub.' end @@ -167,7 +191,7 @@ class Build CSourcesEmbedder.new(app, @source_dir).embed LibEmbedder.new( app, - brew_dir, + [brew_dir, '/nix/store'], extra_libs, relink_eln_files: options[:relink_eln] ).embed @@ -177,6 +201,61 @@ class Build archive_build(build_dir) if options[:archive] end + def print_info + # Force-enable native-comp to ensure all env vars are setup. + options[:native_comp] = true + + puts YAML.dump( + { + 'os' => OS.version.to_s, + 'sdk' => OS.sdk_version.to_s, + 'arch' => OS.arch, + 'gcc' => { + 'root' => gcc_info.root_dir, + 'lib' => gcc_info.lib_dir, + 'darwin_lib' => gcc_info.darwin_lib_dir, + 'target_lib' => gcc_info.target_lib_dir, + 'target_darwin_lib' => gcc_info.target_darwin_lib_dir, + 'sanitized_target_darwin_lib_dir' => + gcc_info.sanitized_target_darwin_lib_dir, + 'version' => gcc_info.major_version + }, + 'libgccjit' => { + 'root' => gcc_info.libgccjit_root_dir, + 'lib' => gcc_info.libgccjit_lib_dir, + 'version' => gcc_info.libgccjit_major_version + }, + 'env' => { + 'CC' => compile_env['CC'], + 'CFLAGS' => compile_env['CFLAGS']&.split, + 'LDFLAGS' => compile_env['LDFLAGS']&.split, + 'LIBRARY_PATH' => compile_env['LIBRARY_PATH']&.split(':'), + 'PKG_CONFIG_PATH' => compile_env['PKG_CONFIG_PATH']&.split(':'), + 'PATH' => compile_env['PATH']&.split(':') + } + } + ) + end + + def print_preview + puts YAML.dump( + { + 'build_name' => build_name, + 'emacs' => { + 'ref' => meta[:ref], + 'sha' => meta[:sha], + 'date' => meta[:date] + }, + 'os_version' => OS.version.to_s, + 'sdk_version' => OS.sdk_version.to_s, + 'arch' => OS.arch, + 'native_comp' => options[:native_comp], + 'gcc_version' => gcc_info.major_version, + 'libgccjit_version' => gcc_info.libgccjit_major_version + } + ) + end + private def load_plan(filename) @@ -217,11 +296,16 @@ class Build @github_src_repo ||= options[:github_src_repo] || DEFAULT_GITHUB_REPO end + def use_nix? + !!options[:use_nix] + end + def brew_dir @brew_dir ||= `brew --prefix`.chomp end def extra_libs + return [] if use_nix? return @extra_libs if @extra_libs libs = [ @@ -359,6 +443,11 @@ class Build def autogen FileUtils.cd(source_dir) do + if File.exist?('configure') + info 'configure script exists, skipping autogen.' + return + end + if File.exist?('autogen/copy_autogen') run_cmd 'autogen/copy_autogen' elsif File.exist?('autogen.sh') @@ -367,6 +456,135 @@ class Build end end + # rubocop:disable Naming/MethodName,Naming/VariableName + def env_CFLAGS + return @env_CFLAGS if @env_CFLAGS + + env = [] + + env << '-O2' + + if options[:native_comp] + env += [ + "-I#{File.join(gcc_info.root_dir, 'include')}", + "-I#{File.join(gcc_info.libgccjit_root_dir, 'include')}" + ] + end + + env << '-march=native' if options[:native_march] + + if options[:fd_setsize].respond_to?(:>=) && options[:fd_setsize] >= 1024 + env += [ + "-DFD_SETSIZE=#{options[:fd_setsize]}", + '-DDARWIN_UNLIMITED_SELECT' + ] + end + + if use_nix? && ENV['NIX_CFLAGS_COMPILE'] + env += ENV['NIX_CFLAGS_COMPILE'].split + end + + @env_CFLAGS = env + end + + def env_LDFLAGS + return @env_LDFLAGS if @env_LDFLAGS + + env = [] + + # Ensure library re-linking and code signing will work after building. + env << '-Wl,-headerpad_max_install_names' + + if options[:native_comp] + env += [ + "-L#{gcc_info.lib_dir}", + "-L#{gcc_info.darwin_lib_dir}", + "-L#{gcc_info.libgccjit_lib_dir}", + "-I#{File.join(gcc_info.root_dir, 'include')}", + "-I#{File.join(gcc_info.libgccjit_root_dir, 'include')}" + ] + end + + env += ENV['NIX_LDFLAGS'].split if use_nix? && ENV['NIX_LDFLAGS'] + + @env_LDFLAGS = env + end + + def env_LIBRARY_PATH + return @env_LIBRARY_PATH if @env_LIBRARY_PATH + + env = [] + + if options[:native_comp] + env += [ + gcc_info.lib_dir, + gcc_info.darwin_lib_dir, + gcc_info.libgccjit_lib_dir + ] + end + + env << '/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib' + + @env_LIBRARY_PATH = env + end + + def env_PKG_CONFIG_PATH + return [] if use_nix? + + @env_PKG_CONFIG_PATH ||= [ + File.join(brew_dir, 'lib/pkgconfig'), + File.join(brew_dir, 'share/pkgconfig'), + File.join(brew_dir, 'opt/expat/lib/pkgconfig'), + File.join(brew_dir, 'opt/libxml2/lib/pkgconfig'), + File.join(brew_dir, 'opt/ncurses/lib/pkgconfig'), + File.join(brew_dir, 'opt/zlib/lib/pkgconfig'), + File.join( + brew_dir, + 'Homebrew/Library/Homebrew/os/mac/pkgconfig', + OS.version.to_s + ) + ] + end + + def env_PATH + return [] if use_nix? + + @env_PATH ||= [ + File.join(brew_dir, 'opt/make/libexec/gnubin'), + File.join(brew_dir, 'opt/coreutils/libexec/gnubin'), + File.join(brew_dir, 'opt/gnu-sed/libexec/gnubin'), + File.join(brew_dir, 'bin'), + File.join(brew_dir, 'opt/texinfo/bin') + ] + end + # rubocop:enable Naming/MethodName,Naming/VariableName + + def compile_env + return @compile_env if @compile_env + + env = { + 'CC' => use_nix? ? 'clang' : '/usr/bin/clang', + 'PATH' => [ + env_PATH, ENV.fetch('PATH', nil) + ].flatten.compact.reject(&:empty?).join(':'), + 'PKG_CONFIG_PATH' => [ + env_PKG_CONFIG_PATH, + ENV.fetch('PKG_CONFIG_PATH', nil) + ].flatten.compact.reject(&:empty?).join(':') + } + + if options[:native_comp] + env['CFLAGS'] = [env_CFLAGS, ENV.fetch('CFLAGS', nil)] + .flatten.compact.reject(&:empty?).join(' ') + env['LDFLAGS'] = [env_LDFLAGS, ENV.fetch('LDFLAGS', nil)] + .flatten.compact.reject(&:empty?).join(' ') + env['LIBRARY_PATH'] = [env_LIBRARY_PATH, ENV.fetch('LIBRARY_PATH', nil)] + .flatten.compact.reject(&:empty?).join(':') + end + + @compile_env = env + end + def compile_source(source) target = File.join(source, 'nextstep') emacs_app = File.join(target, 'Emacs.app') @@ -384,71 +602,9 @@ class Build info 'Compiling with native-comp enabled' verify_native_comp gcc_info.verify_libgccjit - - ENV['CFLAGS'] = [ - "-I#{File.join(gcc_info.root_dir, 'include')}", - "-I#{File.join(gcc_info.libgccjit_root_dir, 'include')}", - '-O2', - (options[:native_march] ? '-march=native' : nil), - ENV.fetch('CFLAGS', nil) - ].compact.join(' ') - - ENV['LDFLAGS'] = [ - "-L#{gcc_info.lib_dir}", - "-L#{gcc_info.darwin_lib_dir}", - "-L#{gcc_info.libgccjit_lib_dir}", - "-I#{File.join(gcc_info.root_dir, 'include')}", - "-I#{File.join(gcc_info.libgccjit_root_dir, 'include')}", - # Ensure library re-linking and code signing will work after building. - '-Wl,-headerpad_max_install_names', - ENV.fetch('LDFLAGS', nil) - ].compact.join(' ') - - ENV['LIBRARY_PATH'] = [ - gcc_info.lib_dir, - gcc_info.darwin_lib_dir, - gcc_info.libgccjit_lib_dir, - ENV.fetch('LIBRARY_PATH', nil) - ].compact.join(':') end - if options[:fd_setsize].respond_to?(:>=) && options[:fd_setsize] >= 1024 - ENV['CFLAGS'] = [ - "-DFD_SETSIZE=#{options[:fd_setsize]}", - '-DDARWIN_UNLIMITED_SELECT', - ENV.fetch('CFLAGS', nil) - ].compact.join(' ') - end - - ENV['CC'] = 'clang' - ENV['PKG_CONFIG_PATH'] = [ - File.join(brew_dir, 'lib/pkgconfig'), - File.join(brew_dir, 'share/pkgconfig'), - File.join(brew_dir, 'opt/expat/lib/pkgconfig'), - File.join(brew_dir, 'opt/libxml2/lib/pkgconfig'), - File.join(brew_dir, 'opt/ncurses/lib/pkgconfig'), - File.join(brew_dir, 'opt/zlib/lib/pkgconfig'), - File.join( - brew_dir, - 'Homebrew/Library/Homebrew/os/mac/pkgconfig', - OS.version.to_s - ), - ENV.fetch('PKG_CONFIG_PATH', nil) - ].compact.join(':') - - ENV['PATH'] = [ - File.join(brew_dir, 'opt/make/libexec/gnubin'), - File.join(brew_dir, 'opt/coreutils/libexec/gnubin'), - File.join(brew_dir, 'opt/gnu-sed/libexec/gnubin'), - File.join(brew_dir, 'bin'), - File.join(brew_dir, 'opt/texinfo/bin'), - ENV.fetch('PATH', nil) - ].compact.join(':') - - ENV['LIBRARY_PATH'] = [ - ENV.fetch('LIBRARY_PATH', nil), - '/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib' - ].compact.join(':') + compile_env.each { |k, v| ENV[k] = v } local_lisp_path = [ ENV.fetch('EMACS_LOCAL_LISP_PATH', '').split(':'), @@ -477,7 +633,7 @@ class Build # Disable aligned_alloc on Mojave and below. See issue: # https://github.com/daviderestivo/homebrew-emacs-head/issues/15 - if OS.version.major <= 10 && OS.version.minor <= 14 + if OS.sdk_version.major <= 10 && OS.sdk_version.minor <= 14 info 'Force disabling of aligned_alloc on macOS Mojave (10.14.x) ' \ 'and earlier' disable_alligned_alloc @@ -649,7 +805,7 @@ class Build meta[:date]&.strftime('%Y-%m-%d'), meta[:sha][0..6], meta[:ref], - "macOS-#{OS.version}", + "macOS-#{OS.sdk_version}", OS.arch ].compact.map { |v| v.gsub(/[^\w_-]+/, '-') } @@ -972,6 +1128,16 @@ class AbstractEmbedder def resources_dir @resources_dir ||= File.join(app, 'Contents', 'Resources') end + + private + + def while_writable(file) + mode = File.stat(file).mode + File.chmod(0o775, file) + yield + ensure + File.chmod(mode, file) if File.exist?(file) + end end class CLIHelperEmbedder < AbstractEmbedder @@ -1036,14 +1202,14 @@ class CSourcesEmbedder < AbstractEmbedder end class LibEmbedder < AbstractEmbedder - attr_reader :lib_source + attr_reader :lib_sources attr_reader :extra_libs attr_reader :relink_eln_files - def initialize(app, lib_source, extra_libs = [], relink_eln_files: true) + def initialize(app, sources = [], extra_libs = [], relink_eln_files: true) super(app) - @lib_source = lib_source + @lib_sources = sources @extra_libs = extra_libs @relink_eln_files = relink_eln_files end @@ -1167,9 +1333,9 @@ class LibEmbedder < AbstractEmbedder debug "-- -- Resolved to: #{lib_filepath}" if linked_dylib != lib_filepath - # Only bundle libraries from lib_source. - unless lib_filepath.start_with?(lib_source) - debug "-- -- Skipping, not from lib_source: #{lib_source}" + # Only bundle libraries from lib_sources. + unless lib_sources.any? { |p| lib_filepath.start_with?(p) } + debug "-- -- Skipping, not from lib_sources: #{lib_sources.join(', ')}" next end @@ -1219,7 +1385,15 @@ class LibEmbedder < AbstractEmbedder next if dylib_id.nil? || dylib_id == '' while_writable(target) do - MachO::Tools.change_dylib_id(target, dylib_id) + file = MachO.open(target) + file.change_dylib_id(dylib_id) + + # Remove all rpaths except for @loader_path. Any other rpaths present in + # embedded libraries will potentially cause issues. + rpaths = file.rpaths.reject { |r| r == '@loader_path' } + rpaths.each { |r| file.delete_rpath(r) } + + file.write! end end @@ -1245,14 +1419,6 @@ class LibEmbedder < AbstractEmbedder while_writable(target_file) { mf.write! } if changed end end - - def while_writable(file) - mode = File.stat(file).mode - File.chmod(0o775, file) - yield - ensure - File.chmod(mode, file) if File.exist?(file) - end end class GccLibEmbedder < AbstractEmbedder @@ -1275,12 +1441,31 @@ class GccLibEmbedder < AbstractEmbedder fatal "No suitable GCC lib dir found in #{gcc_info.root_dir}" end - FileUtils.mkdir_p(File.dirname(target_dir)) - run_cmd('cp', '-pRL', source_dir, target_dir) - FileUtils.rm(Dir[File.join(target_dir, '**', '.DS_Store')], force: true) + FileUtils.mkdir_p(target_dir) + run_cmd( + 'rsync', '-rlptD', + # Exclude lib symlink which points at itself when using nix. + '--exclude', 'lib', + # Exclude gcc directory which holds apple-darwin libs, we copy those + # separately. + '--exclude', 'gcc', + File.join(source_dir, ''), target_dir + ) run_cmd('chmod', '-R', 'u+w', target_dir) - if source_darwin_dir != target_darwin_dir - run_cmd('mv', source_darwin_dir, target_darwin_dir) + tidy_lib_rpaths(target_dir) + + FileUtils.mkdir_p(target_darwin_dir) + run_cmd( + 'rsync', '-rlptD', + File.join(source_darwin_dir, ''), target_darwin_dir + ) + run_cmd('chmod', '-R', 'u+w', target_darwin_dir) + tidy_lib_rpaths(target_darwin_dir) + + FileUtils.rm(Dir[File.join(target_dir, '**', '.DS_Store')], force: true) + + if target_darwin_dir != sanitized_target_darwin_dir + run_cmd('mv', target_darwin_dir, sanitized_target_darwin_dir) end env_setup = ERB.new(NATIVE_COMP_ENV_VAR_TPL).result(gcc_info.get_binding) @@ -1305,10 +1490,10 @@ class GccLibEmbedder < AbstractEmbedder (devtools-dir "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib") (gcc-dir (expand-file-name - "<%= app_bundle_relative_lib_dir %>" + "<%= app_bundle_target_lib_dir %>" invocation-directory)) (darwin-dir (expand-file-name - "<%= app_bundle_relative_darwin_lib_dir %>" + "<%= app_bundle_target_darwin_lib_dir %>" invocation-directory)) (lib-paths (list))) @@ -1322,26 +1507,50 @@ class GccLibEmbedder < AbstractEmbedder (setenv "LIBRARY_PATH" (mapconcat 'identity lib-paths ":")))) ELISP + # Remove all rpaths from Mach-O library files except for @loader_path. + def tidy_lib_rpaths(directory) + Dir[File.join(directory, '**', '*.{dylib,so}')].each do |file_path| + next if File.symlink?(file_path) + + begin + mf = MachO.open(file_path) + rescue MachO::NotAMachOError + next + end + + rpaths = mf.rpaths.reject { |r| r == '@loader_path' } + next if rpaths.none? + + debug "Tidying up rpaths from: #{relative_path(file_path)}" + rpaths.each { |r| mf.delete_rpath(r) } + mf.write! + end + end + def embedded? Dir[File.join(target_dir, 'libgcc*')].any? end def target_dir - File.join(lib_dir, gcc_info.relative_lib_dir) + File.join(lib_dir, gcc_info.target_lib_dir) end - def source_darwin_dir - File.join(lib_dir, gcc_info.relative_darwin_lib_dir) + def target_darwin_dir + File.join(lib_dir, gcc_info.target_darwin_lib_dir) end - def target_darwin_dir - File.join(lib_dir, gcc_info.sanitized_relative_darwin_lib_dir) + def sanitized_target_darwin_dir + File.join(lib_dir, gcc_info.sanitized_target_darwin_lib_dir) end def source_dir gcc_info.lib_dir end + def source_darwin_dir + gcc_info.darwin_lib_dir + end + def relative_dir(path, root) Pathname.new(path).relative_path_from(Pathname.new(root)).to_s end @@ -1354,86 +1563,132 @@ end class GccInfo include Output + def initialize(use_nix: false) + @use_nix = use_nix + end + + def use_nix? + @use_nix + end + def root_dir - @root_dir ||= `brew --prefix gcc`.chomp + @root_dir ||= + if use_nix? + libgccjit_root_dir + else + `brew --prefix gcc`.chomp + end end def major_version - @major_version ||= File.basename(lib_dir) + @major_version ||= + if use_nix? + libgccjit_major_version + else + File.basename(lib_dir) + end end def lib_dir @lib_dir ||= - Dir[File.join(root_dir, 'lib/gcc/*/libgcc*')] - .map { |path| File.dirname(path) } - .select { |path| File.basename(path).match(/^\d+$/) } - .max_by { |path| File.basename(path).to_i } + if use_nix? + File.join(root_dir, 'lib') + else + Dir[File.join(root_dir, 'lib/gcc/*/libgcc*')] + .map { |path| File.dirname(path) } + .select { |path| File.basename(path).match(/^\d+$/) } + .max_by { |path| File.basename(path).to_i } + end end - def relative_lib_dir - @relative_lib_dir ||= relative_dir(lib_dir, File.join(root_dir, 'lib')) + def target_lib_dir + File.join('gcc', 'lib') end def darwin_lib_dir - @darwin_lib_dir ||= - Dir[File.join(lib_dir, 'gcc/*apple-darwin*/*')].max_by do |path| - [ - File.basename(File.dirname(path)).match(/darwin(\d+)$/)[1].to_i, - File.basename(path).split('.').map(&:to_i) - ] + return @darwin_lib_dir if @darwin_lib_dir + + search_path = File.join(lib_dir, 'gcc/*apple-darwin*/*') + + @darwin_lib_dir ||= Dir[search_path].max_by do |path| + vers = [] + + unless use_nix? + matches = File.basename(File.dirname(path)).match(/darwin(\d+)$/) + vers << matches[1].to_i if matches end + + vers << File.basename(path).split('.').map(&:to_i) + vers.flatten + end end - def relative_darwin_lib_dir - @relative_darwin_lib_dir ||= - relative_dir(darwin_lib_dir, File.join(root_dir, 'lib')) + def target_darwin_lib_dir + File.join('gcc', 'lib', 'apple-darwin') end # Sanitize folder name with full "MAJOR.MINOR.PATCH" version number to just # the MAJOR version. Apple's codesign CLI tool throws a "bundle format # unrecognized" error if there are any folders with two dots in their name # within the Emacs.app application bundle. - def sanitized_relative_darwin_lib_dir - @sanitized_relative_darwin_lib_dir ||= + def sanitized_target_darwin_lib_dir + @sanitized_target_darwin_lib_dir ||= File.join( - File.dirname(relative_darwin_lib_dir), - File.basename(relative_darwin_lib_dir).gsub('.', '_') + File.dirname(target_darwin_lib_dir), + File.basename(target_darwin_lib_dir).gsub('.', '_') ) end - def app_bundle_relative_lib_dir - @app_bundle_relative_lib_dir ||= + def app_bundle_target_lib_dir + @app_bundle_target_lib_dir ||= relative_dir( - File.join(embedder.lib_dir, relative_lib_dir), + File.join(embedder.lib_dir, target_lib_dir), embedder.invocation_dir ) end - def app_bundle_relative_darwin_lib_dir - @app_bundle_relative_darwin_lib_dir ||= + def app_bundle_target_darwin_lib_dir + @app_bundle_target_darwin_lib_dir ||= relative_dir( - File.join(embedder.lib_dir, sanitized_relative_darwin_lib_dir), + File.join(embedder.lib_dir, sanitized_target_darwin_lib_dir), embedder.invocation_dir ) end def libgccjit_root_dir - @libgccjit_root_dir ||= `brew --prefix libgccjit`.chomp + @libgccjit_root_dir ||= + if use_nix? + ENV['NIX_LIBGCCJIT_ROOT']&.strip + else + `brew --prefix libgccjit`.chomp + end end def libgccjit_major_version - @libgccjit_major_version ||= File.basename(libgccjit_lib_dir.to_s) + @libgccjit_major_version ||= + if use_nix? + # rubocop:disable Style/SafeNavigationChainLength + ENV['NIX_LIBGCCJIT_VERSION']&.strip&.split('.')&.first + # rubocop:enable Style/SafeNavigationChainLength + else + File.basename(libgccjit_lib_dir.to_s) + end end def libgccjit_lib_dir @libgccjit_lib_dir ||= - Dir[ - File.join(libgccjit_root_dir, 'lib/gcc/*/libgccjit*.dylib'), - File.join(libgccjit_root_dir, 'lib/gcc/*/libgccjit.so*') - ] - .map { |path| File.dirname(path) } - .select { |path| File.basename(path).match(/^\d+$/) } - .max_by { |path| File.basename(path).to_i } + if use_nix? + Dir[File.join(libgccjit_root_dir, 'lib/libgccjit*.dylib')] + .map { |path| File.dirname(path) }.first + else + Dir[ + File.join(libgccjit_root_dir, 'lib/gcc/*/libgccjit*.dylib'), + File.join(libgccjit_root_dir, 'lib/gcc/*/libgccjit.so*'), + ] + .map { |path| File.dirname(path) } + .select { |path| File.basename(path).match(/^\d+$/) } + .max_by { |path| File.basename(path).to_i } + end end def verify_libgccjit @@ -1446,6 +1701,11 @@ class GccInfo 'brew reinstall libgccjit' end + # No need to verify gcc vs libgccjit for Nix, as we can pull everything we + # need from the libgccjit package. On homebrew we need to pull parts from + # gcc and parts from libgccjit, hence we need to ensure versions match. + return if use_nix? + return if major_version == libgccjit_major_version fatal <<~TEXT @@ -1472,6 +1732,8 @@ class GccInfo end if __FILE__ == $PROGRAM_NAME + use_nix_default = !ENV.fetch('IN_NIX_SHELL', '').empty? + cli_options = { work_dir: File.expand_path(__dir__), native_full_aot: false, @@ -1480,6 +1742,7 @@ if __FILE__ == $PROGRAM_NAME parallel: Etc.nprocessors, rsvg: true, dbus: true, + use_nix: use_nix_default, xwidgets: true, tree_sitter: true, fd_setsize: 10_000, @@ -1502,6 +1765,16 @@ if __FILE__ == $PROGRAM_NAME Options: DOC + opts.on( + '--info', + 'Print environment info and detected library paths, then exit' + ) { |v| cli_options[:info] = v } + + opts.on( + '--preview', + 'Print preview details about build and exit.' + ) { |v| cli_options[:preview] = v } + opts.on( '-j', '--parallel COUNT', @@ -1515,6 +1788,12 @@ if __FILE__ == $PROGRAM_NAME 'branch allowing builds of old commits' ) { |v| cli_options[:git_sha] = v } + opts.on( + '--[no-]use-nix', + 'Use Nix instead of Homebrew to find dependencies ' \ + '(default: enabled if EMACS_BUILD_USE_NIX is set to truthy value)' + ) { |v| cli_options[:use_nix] = v } + opts.on( '--[no-]xwidgets', 'Enable/disable XWidgets if supported ' \ @@ -1653,7 +1932,15 @@ if __FILE__ == $PROGRAM_NAME Output.log_level = cli_options[:log_level] work_dir = cli_options.delete(:work_dir) - Build.new(work_dir, ARGV.shift, cli_options).build + build = Build.new(work_dir, ARGV.shift, cli_options) + + if cli_options[:info] + build.print_info + elsif cli_options[:preview] + build.print_preview + else + build.build + end rescue Error => e warn "ERROR: #{e.message}" exit 1 diff --git a/flake-package-versions.txt b/flake-package-versions.txt new file mode 100644 index 0000000..21cb8cd --- /dev/null +++ b/flake-package-versions.txt @@ -0,0 +1,67 @@ +DarwinTools-1 +autoconf-2.72 +bash-5.2p37 +brotli-1.1.0 +bzip2-1.0.8 +cairo-1.18.2 +cctools-binutils-darwin-1010.6 +cctools-binutils-darwin-wrapper-1010.6 +clang-16.0.6 +clang-wrapper-16.0.6 +coreutils-9.5 +curl-8.11.0 +dbus-1.14.10 +diffutils-3.10 +expat-2.6.4 +file-5.45 +findutils-4.10.0 +fontconfig-2.15.0 +freetype-2.13.3 +gawk-5.3.1 +gcc-13.3.0 +gcc-wrapper-13.3.0 +gdk-pixbuf-2.42.12 +gettext-0.21.1 +giflib-5.2.2 +git-2.47.0 +glib-2.82.1 +gnugrep-3.11 +gnumake-4.4.1 +gnused-4.9 +gnutar-1.35 +gnutls-3.8.6 +graphite2-1.3.14 +gzip-1.13 +harfbuzz-10.0.1 +jq-1.7.1 +krb5-1.21.3 +lcms2-2.16 +libdeflate-1.22 +libgccjit-13.3.0 +libiconv-107 +libidn2-2.3.7 +libjpeg-turbo-3.0.4 +libpng-apng-1.6.43 +libpsl-0.21.5 +librsvg-2.58.3 +libtasn1-4.19.0 +libtiff-4.7.0 +libwebp-1.4.0 +libxml2-2.13.4 +mailutils-3.17 +nettle-3.10 +nghttp2-1.64.0 +openssl-3.3.2 +patch-2.7.6 +pkg-config-wrapper-0.29.2 +python3-3.12.7 +rsync-3.3.0 +ruby-3.3.5 +sqlite-3.46.1 +texinfo-7.1.1 +time-1.9 +tree-sitter-0.24.3 +which-2.21 +xcbuild-0.1.1-unstable-2019-11-20 +xz-5.6.3 +zstd-1.5.6 diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..a4ccc74 --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1731603435, + "narHash": "sha256-CqCX4JG7UiHvkrBTpYC3wcEurvbtTADLbo3Ns2CEoL8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "8b27c1239e5c421a2bbc2c65d52e4a6fbf2ff296", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "24.11-beta", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..18f2c99 --- /dev/null +++ b/flake.nix @@ -0,0 +1,102 @@ +{ + description = "Development environment flake"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/24.11-beta"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + + # List of supported macOS SDK versions. + sdk_versions = [ "11" "12" "13" "14" "15" ]; + default_sdk_version = "11"; + + mkDevShell = { macos_version ? default_sdk_version }: + let + apple_sdk = pkgs.${"apple-sdk_${macos_version}"}; + in + pkgs.mkShell { + # Package list specifically excludes ncurses, so that we link + # against the system version of ncurses. This ensures emacs' TUI + # works out of the box without the user having to manually set + # TERMINFO in the shell before launching emacs. + packages = with pkgs; [ + apple_sdk + autoconf + bash + cairo + clang + coreutils + curl + darwin.DarwinTools # sw_vers + dbus + expat + findutils + gcc + gettext + giflib + git + gmp + gnumake + gnupatch + gnused + gnutar + gnutls + harfbuzz + jansson + jq + lcms2 + libffi + libgccjit + libiconv + libjpeg + libpng + librsvg + libtasn1 + libunistring + libwebp + libxml2 + mailutils + nettle + pkg-config + python3 + rsync + ruby_3_3 + sqlite + texinfo + time + tree-sitter + which + xcbuild + zlib + ]; + + shellHook = '' + export CC=clang + export MACOSX_DEPLOYMENT_TARGET="${macos_version}.0" + export DEVELOPER_DIR="${apple_sdk}" + export NIX_LIBGCCJIT_VERSION="${pkgs.libgccjit.version}" + export NIX_LIBGCCJIT_ROOT="${pkgs.libgccjit.outPath}" + export BUNDLE_WITHOUT=development + ''; + }; + + # Generate an attrset of shells for each macOS SDK version. + versionShells = builtins.listToAttrs ( + map (version: { + name = "macos${version}"; + value = mkDevShell { macos_version = version; }; + }) sdk_versions + ); + in + { + devShells = versionShells // { + default = mkDevShell {}; + }; + } + ); +} diff --git a/go.mod b/go.mod index 3122b4a..cc240ae 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/jimeh/build-emacs-for-macos -go 1.20 +go 1.23 require ( github.com/bearer/gon v0.0.36 diff --git a/go.sum b/go.sum index 773cb52..d4b4505 100644 --- a/go.sum +++ b/go.sum @@ -24,6 +24,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v35 v35.3.0 h1:fU+WBzuukn0VssbayTT+Zo3/ESKX9JYWjbZTLOTEyho= github.com/google/go-github/v35 v35.3.0/go.mod h1:yWB7uCcVWaUbUP74Aq3whuMySRMatyRmq5U9FTNlbio= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= diff --git a/pkg/dmgbuild/license.go b/pkg/dmgbuild/license.go index 996283e..92f0499 100644 --- a/pkg/dmgbuild/license.go +++ b/pkg/dmgbuild/license.go @@ -116,7 +116,6 @@ func NewLicense() License { return License{} } -//nolint:goconst func (s *License) Render() []string { var l []string diff --git a/pkg/osinfo/osinfo.go b/pkg/osinfo/osinfo.go index 31d35b8..823db27 100644 --- a/pkg/osinfo/osinfo.go +++ b/pkg/osinfo/osinfo.go @@ -1,41 +1,65 @@ package osinfo import ( + "os" "os/exec" "strconv" "strings" ) type OSInfo struct { - Name string `yaml:"name" json:"name"` - Version string `yaml:"version" json:"version"` - Arch string `yaml:"arch" json:"arch"` + Name string `yaml:"name" json:"name"` + Version string `yaml:"version" json:"version"` + SDKVersion string `yaml:"sdk_version" json:"sdk_version"` + Arch string `yaml:"arch" json:"arch"` } func New() (*OSInfo, error) { - version, err := exec.Command("sw_vers", "-productVersion").CombinedOutput() + version, err := exec.Command("sw_vers", "-productVersion").Output() if err != nil { return nil, err } + sdkVersion := os.Getenv("MACOSX_DEPLOYMENT_TARGET") + if sdkVersion == "" { + var ver []byte + ver, err = exec.Command("xcrun", "--show-sdk-version").Output() + if err != nil { + return nil, err + } + + sdkVersion = string(ver) + } + arch, err := exec.Command("uname", "-m").CombinedOutput() if err != nil { return nil, err } return &OSInfo{ - Name: "macOS", - Version: strings.TrimSpace(string(version)), - Arch: strings.TrimSpace(string(arch)), + Name: "macOS", + Version: strings.TrimSpace(string(version)), + SDKVersion: strings.TrimSpace(sdkVersion), + Arch: strings.TrimSpace(string(arch)), }, nil } -// DistinctVersion returns macOS version down to a distinct "major" -// version. For macOS 10.x, this will include the first two numeric parts of the -// version (10.15), while for 11.x and later, the first numeric part is enough -// (11). +// DistinctVersion returns macOS version down to a distinct "major" version. For +// macOS 10.x, this will include the first two numeric parts of the version +// (10.15), while for 11.x and later, the first numeric part is enough (11). func (s *OSInfo) DistinctVersion() string { - parts := strings.Split(s.Version, ".") + return s.distinctVersion(s.Version) +} + +// DistinctSDKVersion returns macOS version down to a distinct "major" version. +// For macOS 10.x, this will include the first two numeric parts of the version +// (10.15), while for 11.x and later, the first numeric part is enough (11). +func (s *OSInfo) DistinctSDKVersion() string { + return s.distinctVersion(s.SDKVersion) +} + +func (s *OSInfo) distinctVersion(version string) string { + parts := strings.Split(version, ".") if n, _ := strconv.Atoi(parts[0]); n >= 11 { return parts[0] diff --git a/pkg/plan/create.go b/pkg/plan/create.go index 50d59c0..277fab1 100644 --- a/pkg/plan/create.go +++ b/pkg/plan/create.go @@ -95,10 +95,17 @@ func Create(ctx context.Context, opts *Options) (*Plan, error) { //nolint:funlen releaseName = "Emacs." + version } + // Attempt to get the macOS SDK version from the environment, if it's not + // available, use the version from the system. + targetMacOSVersion := osInfo.DistinctSDKVersion() + if targetMacOSVersion == "" { + targetMacOSVersion = osInfo.DistinctVersion() + } + buildName := fmt.Sprintf( "Emacs.%s.%s.%s", absoluteVersion, - sanitize.String(osInfo.Name+"-"+osInfo.DistinctVersion()), + sanitize.String(osInfo.Name+"-"+targetMacOSVersion), sanitize.String(osInfo.Arch), ) diskImage := buildName + ".dmg" diff --git a/pkg/sign/emacs.go b/pkg/sign/emacs.go index c59e790..62c343d 100644 --- a/pkg/sign/emacs.go +++ b/pkg/sign/emacs.go @@ -123,7 +123,7 @@ func signCLIHelper(ctx context.Context, appBundle string, opts *Options) error { // app bundle itself. func elnFiles(emacsApp string) ([]string, error) { var files []string - walkDirFunc := func(path string, d fs.DirEntry, _err error) error { + walkDirFunc := func(path string, d fs.DirEntry, _ error) error { if d.Type().IsRegular() && strings.HasSuffix(path, ".eln") && !strings.Contains(path, ".app/Contents/Frameworks/") { files = append(files, path)