From 5948fbe2ae9e171afa1f19c9861870cad619472d Mon Sep 17 00:00:00 2001 From: Alex Lopez Date: Thu, 11 Apr 2024 05:17:48 -0700 Subject: [PATCH] Fix failing CI, local dev dockerfile setup, implement Centos 6 deprecation (#913) --- .circleci/config.yml | 10 ++++--- Gemfile.lock | 33 ++++++++++++--------- README.md | 43 ++++++++++++++++----------- docker_test_env/Dockerfile | 52 +++++++++++++++++++++++++++------ docker_test_env/docker.rb.patch | 11 +++++++ recipes/_install-linux.rb | 15 ++++++++++ spec/dd-agent_spec.rb | 22 +++++++++----- spec/shared_examples.rb | 9 ++++++ 8 files changed, 143 insertions(+), 52 deletions(-) create mode 100644 docker_test_env/docker.rb.patch diff --git a/.circleci/config.yml b/.circleci/config.yml index 821d61bf..99862c18 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,21 +2,23 @@ version: 2 jobs: kitchen-docker-tests: machine: - image: ubuntu-2004:202201-02 + image: ubuntu-2004:2024.01.2 resource_class: large environment: CHEF_LICENSE: accept # newer versions of Chef client need explicit license accept to run KITCHEN_LOCAL_YAML: kitchen.docker.yml RUBY_VERSION: '2.6.3' # ruby used to invoke kitchen, not the version used in the tests + DOCKER_BUILDKIT: 0 # Disable buildkit for compatibility with version of kitchen we use + shell: /bin/bash --login -eo pipefail steps: - checkout - run: name: Set Ruby version command: | echo "Using $RUBY_VERSION" - curl -sSL https://rvm.io/pkuczynski.asc | gpg --import - - # Update RVM. Using this path instead of https://get.rvm.io because the later uses the letsencrypt cert that breaks openssl 1.0 - curl -sSL https://raw.githubusercontent.com/rvm/rvm/master/binscripts/rvm-installer | bash -s stable + gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB + curl -sSL https://get.rvm.io | bash -s stable + source /home/circleci/.rvm/scripts/rvm rvm reload rvm install $RUBY_VERSION echo . $(rvm $RUBY_VERSION do rvm env --path) >> $BASH_ENV diff --git a/Gemfile.lock b/Gemfile.lock index 0c1c22e9..23d51f2c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GEM remote: https://rubygems.org/ specs: - activesupport (6.0.3.4) + activesupport (6.0.6.1) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) @@ -78,6 +78,8 @@ GEM mixlib-config (>= 2.2.12, < 3.0) mixlib-shellout (~> 2.0) tomlrb (~> 1.2) + chef-utils (18.4.12) + concurrent-ruby chef-zero (14.0.17) ffi-yajl (~> 2.2) hashie (>= 2.0, < 4.0) @@ -110,7 +112,7 @@ GEM cucumber-tag_expressions (1.1.1) descendants_tracker (0.0.4) thread_safe (~> 0.3, >= 0.3.1) - diff-lcs (1.4.4) + diff-lcs (1.5.1) docile (1.3.2) ed25519 (1.2.4) equalizer (0.0.11) @@ -134,8 +136,9 @@ GEM gherkin (5.1.0) gssapi (1.3.0) ffi (>= 1.0.1) - gyoku (1.3.1) + gyoku (1.4.0) builder (>= 2.1.2) + rexml (~> 3.0) hashie (3.6.0) highline (1.7.10) httpclient (2.8.3) @@ -166,7 +169,7 @@ GEM multi_json (~> 1.14) mini_portile2 (2.8.5) minitar (0.9) - minitest (5.14.2) + minitest (5.22.3) mixlib-archive (1.0.7) mixlib-log mixlib-authentication (2.1.1) @@ -194,8 +197,8 @@ GEM net-ssh (>= 2.6.5) net-ssh-gateway (>= 1.2.0) net-telnet (0.1.1) - nokogiri (1.15.5) - mini_portile2 (~> 2.8.2) + nokogiri (1.13.10) + mini_portile2 (~> 2.8.0) racc (~> 1.4) nori (2.6.0) octokit (4.18.0) @@ -226,7 +229,7 @@ GEM racc (1.7.3) rack (2.2.3) rainbow (3.0.0) - rake (13.0.1) + rake (13.2.1) rbnacl (4.0.2) ffi rbnacl-libsodium (1.0.16) @@ -239,7 +242,7 @@ GEM rspec-mocks (~> 3.9.0) rspec-core (3.9.3) rspec-support (~> 3.9.3) - rspec-expectations (3.9.2) + rspec-expectations (3.9.4) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.9.0) rspec-its (1.3.0) @@ -287,9 +290,9 @@ GEM net-ssh (>= 2.7) net-telnet (= 0.1.1) sfl - strings (0.2.0) + strings (0.2.1) strings-ansi (~> 0.2) - unicode-display_width (~> 1.5) + unicode-display_width (>= 1.5, < 3.0) unicode_utils (~> 1.4) strings-ansi (0.2.0) sync (0.5.0) @@ -297,8 +300,9 @@ GEM systemu (2.6.5) term-ansicolor (1.7.1) tins (~> 1.0) - test-kitchen (2.7.2) + test-kitchen (2.12.0) bcrypt_pbkdf (~> 1.0) + chef-utils (>= 16.4.35) ed25519 (~> 1.2) license-acceptance (>= 1.0.11, < 3.0) mixlib-install (~> 3.6) @@ -317,7 +321,7 @@ GEM tomlrb (1.3.0) treetop (1.6.11) polyglot (~> 0.3) - tty-box (0.6.0) + tty-box (0.7.0) pastel (~> 0.8) strings (~> 0.2.0) tty-cursor (~> 0.7) @@ -362,7 +366,7 @@ GEM wisper (2.0.1) wmi-lite (1.0.5) yaml (0.1.0) - zeitwerk (2.4.0) + zeitwerk (2.6.13) PLATFORMS ruby @@ -385,10 +389,11 @@ DEPENDENCIES rake rbnacl (~> 4.0.2) rbnacl-libsodium (~> 1.0.16) + rspec-expectations (< 3.12.4) rubocop (~> 0.80.1) test-kitchen virtus yaml BUNDLED WITH - 2.3.22 + 2.3.26 diff --git a/README.md b/README.md index 82e675e7..307e0a0d 100644 --- a/README.md +++ b/README.md @@ -80,23 +80,6 @@ The following Opscode cookbooks are dependencies: 5. Wait for the next scheduled `chef-client` run or trigger it manually. -### Dockerized environment - -To build a Docker environment, use the files under `docker_test_env`: - -``` -cd docker_test_env -docker build -t chef-datadog-container . -``` - -To run the container use: - -``` -docker run -d -v /dev/vboxdrv:/dev/vboxdrv --privileged=true chef-datadog-container -``` - -Then attach a console to the container or use the VScode remote-container feature to develop inside the container. - #### Datadog attributes The following methods are available for adding your [Datadog API and application keys][4]: @@ -412,6 +395,32 @@ To get the available versions of the integrations, see the integration-specific **Note**: For Chef Windows users, the `chef-client` must have read access to the `datadog.yaml` file when the `datadog-agent` binary available on the node is used by this resource. +## Development + +### Dockerized environment + +To build a Docker environment with which to run kitchen tests, use the files under `docker_test_env`: + +``` +cd docker_test_env +docker build -t chef-datadog-test-env . +``` + +To run the container use: + +``` +docker run -d -v /var/run/docker.sock:/var/run/docker.sock chef-datadog-test-env +``` + +Then attach a console to the container or use the VS Code remote-container feature to develop inside the container. + +To run kitchen-docker tests from within the container: + +``` +# Note: Also set KITCHEN_DOCKER_HOSTNAME=host.docker.internal if on MacOS or Windows +# Run this under a login shell (otherwise `bundle` won't be found) +KITCHEN_LOCAL_YAML=kitchen.docker.yml bundle exec rake circle +``` [1]: https://github.com/DataDog/chef-datadog/blob/master/attributes/default.rb [2]: https://github.com/DataDog/chef-datadog/releases/tag/v2.18.0 diff --git a/docker_test_env/Dockerfile b/docker_test_env/Dockerfile index 0ec95bc3..eeb0d07f 100644 --- a/docker_test_env/Dockerfile +++ b/docker_test_env/Dockerfile @@ -1,25 +1,59 @@ -# docker run --rm -it -v /dev/vboxdrv:/dev/vboxdrv ARG BASE_IMAGE=ubuntu:20.04 FROM $BASE_IMAGE ARG TZ=America/New_York -ARG RUBY_VERSION=2.6 +ARG RUBY_VERSION=2.6.3 ENV TZ=$TZ ENV RUBY_VERSION=$RUBY_VERSION ENV DEBIAN_FRONTEND=noninteractive RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN apt-get update && apt-get install -y wget virtualbox git zlib1g-dev curl +RUN apt-get update && apt-get install -y wget git zlib1g-dev curl gpg # RVM -RUN gpg --keyserver hkp://pool.sks-keyservers.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB +RUN gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB RUN curl -sSL https://get.rvm.io | bash -s stable -RUN /bin/bash -l -c "rvm requirements" + +RUN /bin/bash -l -c "rvm requirements" # || cat /usr/local/rvm/log/*/*.log && false RUN /bin/bash -l -c "rvm install ${RUBY_VERSION} && rvm cleanup all" -RUN /bin/bash -l -c "gem install bundler --no-document" +RUN /bin/bash -l -c "gem install bundler:2.3.26 --no-document" + +RUN /bin/bash -l -c "gem install nori:2.6.0 test-kitchen:2.7.2 octokit:4.18.0 semverse:3.0.0 chef:14.10.9 berkshelf:7.0.10 kitchen-vagrant:1.7.0 kitchen-docker:2.3.0" + +# build with --build-arg INSTALL_VAGRANT=1 to get vagrant support (not needed for kitchen-docker-tests) +# requires `-v /dev/vboxdrv:/dev/vboxdrv` when running +ARG INSTALL_VAGRANT +RUN if [ -n "${INSTALL_VAGRANT}" ]; then \ + apt-get install -y virtualbox && wget https://releases.hashicorp.com/vagrant/2.2.9/vagrant_2.2.9_x86_64.deb && dpkg -i vagrant_2.2.9_x86_64.deb && rm vagrant_2.2.9_x86_64.deb \ + ; fi + +# Install docker. Requires `-v /var/run/docker.sock:/var/run/docker.sock` when running to use the host's docker daemon +RUN \ + apt-get update && \ + apt-get install -y ca-certificates && \ + install -m 0755 -d /etc/apt/keyrings && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc && \ + chmod a+r /etc/apt/keyrings/docker.asc && \ + # Add the repository to Apt sources: + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null && \ + apt-get update && \ + apt-get install -y docker-ce-cli -RUN /bin/bash -l -c "gem install test-kitchen berkshelfbundler kitchen-vagrant" -RUN wget https://releases.hashicorp.com/vagrant/2.2.9/vagrant_2.2.9_x86_64.deb && dpkg -i vagrant_2.2.9_x86_64.deb && rm vagrant_2.2.9_x86_64.deb COPY entrypoint.sh /usr/bin/entrypoint.sh RUN chmod 755 /usr/bin/entrypoint.sh -ENTRYPOINT ["/usr/bin/entrypoint.sh"] + +# Apply patch kitchen-docker to make it possible to manually override the hostname for docker +# with the `KITCHEN_DOCKER_HOSTNAME` environment variable, which is necessary when working on non-native docker. +COPY docker.rb.patch / +RUN patch -p1 -i /docker.rb.patch + +# Helpful defaults to run kitchen tests +ENV DOCKER_BUILDKIT=0 \ + CIRCLE_NODE_TOTAL=1 \ + CIRCLE_NODE_INDEX=0 \ + CHEF_LICENSE=accept + +ENTRYPOINT ["/bin/bash", "-l", "-c", "/usr/bin/entrypoint.sh"] diff --git a/docker_test_env/docker.rb.patch b/docker_test_env/docker.rb.patch new file mode 100644 index 00000000..34eaa73a --- /dev/null +++ b/docker_test_env/docker.rb.patch @@ -0,0 +1,11 @@ +--- /usr/local/rvm/gems/ruby-2.6.3/gems/kitchen-docker-2.3.0/lib/kitchen/driver/docker.rb 2024-04-09 10:54:16.032712772 -0400 ++++ /docker.rb 2024-04-09 10:53:25.988909160 -0400 +@@ -103,6 +103,6 @@ + state[:ssh_key] = config[:private_key] + state[:image_id] = build_image(state) unless state[:image_id] + state[:container_id] = run_container(state) unless state[:container_id] +- state[:hostname] = remote_socket? ? socket_uri.host : 'localhost' ++ state[:hostname] = ENV['KITCHEN_DOCKER_HOSTNAME'] || (remote_socket? ? socket_uri.host : 'localhost') + state[:port] = container_ssh_port(state) + wait_for_sshd(state[:hostname], nil, :port => state[:port]) if config[:wait_for_sshd] + end \ No newline at end of file diff --git a/recipes/_install-linux.rb b/recipes/_install-linux.rb index 070e66f0..846835f8 100644 --- a/recipes/_install-linux.rb +++ b/recipes/_install-linux.rb @@ -45,6 +45,21 @@ action :upgrade end when 'rhel', 'fedora', 'amazon' + # Centos < 7 was deprecated on agent {6,7}.52 + agent_major_version = Chef::Datadog.agent_major_version(node) + if agent_major_version.to_i >= 6 && platform_family?('rhel') && node['platform_version'].to_i < 7 + agent_minor_version = Chef::Datadog.agent_minor_version(node) + if dd_agent_version && agent_minor_version && agent_minor_version >= 52 + # Error out with a useful message when the version was pinned to an unsupported one + Chef::Log.error("Agent versions #{agent_major_version}.52 and above not supported by current OS (RHEL < 7 equivalent).") + raise + else + # Set an upper bound for the package when the version was left unpinned + # Bounds like this one need to go on the package name, they're not supported on the version field + dd_agent_flavor = "#{dd_agent_flavor} < 1:#{agent_major_version}.52.0-1" + end + end + if (platform_family?('rhel') && node['platform_version'].to_i >= 8) || (platform_family?('fedora') && node['platform_version'].to_i >= 28) || (platform_family?('amazon') && node['platform_version'].to_i >= 2022) diff --git a/spec/dd-agent_spec.rb b/spec/dd-agent_spec.rb index ecea6113..7c41e884 100644 --- a/spec/dd-agent_spec.rb +++ b/spec/dd-agent_spec.rb @@ -40,6 +40,12 @@ def set_env_var(name, value) it_behaves_like 'rhellions datadog-agent' end +shared_examples_for 'rhellions no version set rhel<7' do + it_behaves_like 'common linux resources' + + it_behaves_like 'rhellions datadog-agent rhel<7' +end + shared_examples_for 'rhellions dnf no version set' do it_behaves_like 'common linux resources' @@ -103,11 +109,11 @@ def set_env_var(name, value) it_behaves_like 'debianoids no version set' end - context 'on RedHat-family distro above 6.x' do + context 'on RedHat-family distro above 7.x' do cached(:chef_run) do ChefSpec::SoloRunner.new( :platform => 'centos', - :version => '6.9' + :version => '7.7' ) do |node| node.normal['datadog'] = { 'api_key' => 'somethingnotnil' } node.normal['languages'] = { 'python' => { 'version' => '2.6.2' } } @@ -118,11 +124,11 @@ def set_env_var(name, value) it_behaves_like 'rhellions no version set' end - context 'on CentOS 5.11 distro' do + context 'on CentOS < 7 distro' do cached(:chef_run) do ChefSpec::SoloRunner.new( :platform => 'centos', - :version => '5.11' + :version => '6.10' ) do |node| node.normal['datadog'] = { 'api_key' => 'somethingnotnil' } node.normal['languages'] = { 'python' => { 'version' => '2.4.3' } } @@ -130,7 +136,7 @@ def set_env_var(name, value) end it_behaves_like 'repo recipe' - it_behaves_like 'rhellions no version set' + it_behaves_like 'rhellions no version set rhel<7' end context 'on Fedora distro' do @@ -484,7 +490,7 @@ def set_env_var(name, value) cached(:chef_run) do ChefSpec::SoloRunner.new( :platform => 'redhat', - :version => '6.10' + :version => '7.7' ) do |node| node.normal['datadog'] = { 'agent_major_version' => 6, @@ -914,7 +920,7 @@ def set_env_var(name, value) cached(:chef_run) do ChefSpec::SoloRunner.new( platform: 'centos', - version: '6.9' + version: '7.7' ) do |node| node.normal['datadog'] = { 'api_key' => 'somethingnotnil' } node.normal['languages'] = { 'python' => { 'version' => '2.6.2' } } @@ -959,7 +965,7 @@ def set_env_var(name, value) cached(:chef_run) do ChefSpec::SoloRunner.new( platform: 'centos', - version: '6.9' + version: '7.7' ) do |node| node.normal['datadog'] = { 'api_key' => 'somethingnotnil', diff --git a/spec/shared_examples.rb b/spec/shared_examples.rb index 7d7a9aff..b6c7cedf 100644 --- a/spec/shared_examples.rb +++ b/spec/shared_examples.rb @@ -108,6 +108,15 @@ end end +shared_examples_for 'rhellions datadog-agent rhel<7' do + it_behaves_like 'datadog-agent' + + # Centos < 7 was deprecated on Agent 7.52 + it 'installs the version-capped datadog-agent' do + expect(chef_run).to install_yum_package 'datadog-agent < 1:7.52.0-1' + end +end + shared_examples_for 'rhellions dnf datadog-agent' do it_behaves_like 'datadog-agent'