Merge pull request #41 from elastic/buildkite

Buildkite configuration

.buildkite/Dockerfile

@@ -0,0 +1,32 @@
+ARG PHP_VERSION=8.0-cli
+FROM php:${PHP_VERSION}
+
+WORKDIR /usr/src/app
+
+# Install git
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends git
+
+# Install zip extension for PHP
+RUN apt-get install -y \
+        libzip-dev \
+        zip \
+    && docker-php-ext-install zip
+
+# Print PHP version
+RUN php -v
+
+# Install composer
+RUN curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer
+
+# Print composer version
+RUN composer --version
+
+COPY composer.json .
+
+# Install dependencies with composer
+RUN composer install
+
+COPY . .
+
+CMD ["bash", ".buildkite/integration-tests.sh"]

.buildkite/certs/README.md

@@ -0,0 +1,50 @@
+# CI certificates
+
+This directory contains certificates that can be used to test against Elasticsearch in CI
+
+## Generating new certificates using the Certificate Authority cert and key
+
+The `ca.crt` and `ca.key` can be used to generate any other certificates that may be needed
+for CI. Perhaps the easiest way to do so is using 
+[`elasticsearch-certutil`](https://www.elastic.co/guide/en/elasticsearch/reference/current/certutil.html)
+
+Using the elasticsearch docker container, run the following from the `.ci/certs` directory
+
+```sh
+docker run \
+  -v "$PWD:/var/tmp" \
+  --rm docker.elastic.co/elasticsearch/elasticsearch:7.6.1 \
+  ./bin/elasticsearch-certutil cert \
+  --ca-cert /var/tmp/ca.crt --ca-key /var/tmp/ca.key --pem \
+  --out /var/tmp/bundle.zip
+```
+
+This will output a `bundle.zip` file containing a directory named `instance` containing 
+`instance.crt` and `instance.key` in PEM format.
+
+The CN Subject name can be changed using
+
+```sh
+docker run \
+  -v "$PWD:/var/tmp" \
+  --rm docker.elastic.co/elasticsearch/elasticsearch:7.6.1 \
+  ./bin/elasticsearch-certutil cert \
+  --ca-cert /var/tmp/ca.crt --ca-key /var/tmp/ca.key --pem \
+  --out /var/tmp/bundle.zip \
+  --name foo
+```
+
+The directory in `bundle.zip` will now be named `foo` and contain 
+`foo.crt` and `foo.key` in PEM format.
+
+Additional DNS and IP SAN entries can be added with `--dns` and `--ip`, respectively.
+
+```sh
+docker run \
+  -v "$PWD:/var/tmp" \
+  --rm docker.elastic.co/elasticsearch/elasticsearch:7.6.1 \
+  ./bin/elasticsearch-certutil cert \
+  --ca-cert /var/tmp/ca.crt --ca-key /var/tmp/ca.key --pem \
+  --out /var/tmp/bundle.zip \
+  --dns instance --dns localhost --dns es1 --ip 127.0.0.1 --ip ::1
+```

.buildkite/certs/ca.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDSjCCAjKgAwIBAgIVAJQLm8V2LcaCTHUcoIfO+KL63nG3MA0GCSqGSIb3DQEB
+CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu
+ZXJhdGVkIENBMB4XDTIwMDIyNjA1NTA1N1oXDTIzMDIyNTA1NTA1N1owNDEyMDAG
+A1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5lcmF0ZWQgQ0Ew
+ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDYyajkPvGtUOE5M1OowQfB
+kWVrWjo1+LIxzgCeRHp0YztLtdVJ0sk2xoSrt2uZpxcPepdyOseLTjFJex1D2yCR
+AEniIqcFif4G72nDih2LlbhpUe/+/MTryj8ZTkFTzI+eMmbQi5FFMaH+kwufmdt/
+5/w8YazO18SxxJUlzMqzfNUrhM8vvvVdxgboU7PWhk28wZHCMHQovomHmzclhRpF
+N0FMktA98vHHeRjH19P7rNhifSd7hZzoH3H148HVAKoPgqnZ6vW2O2YfAWOP6ulq
+cyszr57p8fS9B2wSdlWW7nVHU1JuKcYD67CxbBS23BeGFgCj4tiNrmxO8S5Yf85v
+AgMBAAGjUzBRMB0GA1UdDgQWBBSWAlip9eoPmnG4p4OFZeOUBlAbNDAfBgNVHSME
+GDAWgBSWAlip9eoPmnG4p4OFZeOUBlAbNDAPBgNVHRMBAf8EBTADAQH/MA0GCSqG
+SIb3DQEBCwUAA4IBAQA19qqrMTWl7YyId+LR/QIHDrP4jfxmrEELrAL58q5Epc1k
+XxZLzOBSXoBfBrPdv+3XklWqXrZjKWfdkux0Xmjnl4qul+srrZDLJVZG3I7IrITh
+AmQUmL9MuPiMnAcxoGZp1xpijtW8Qmd2qnambbljWfkuVaa4hcVRfrAX6TciIQ21
+bS5aeLGrPqR14h30YzDp0RMmTujEa1o6ExN0+RSTkE9m89Q6WdM69az8JW7YkWqm
+I+UCG3TcLd3TXmN1zNQkq4y2ObDK4Sxy/2p6yFPI1Fds5w/zLfBOvvPQY61vEqs8
+SCCcQIe7f6NDpIRIBlty1C9IaEHj7edyHjF6rtYb
+-----END CERTIFICATE-----

.buildkite/certs/ca.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpgIBAAKCAQEA2Mmo5D7xrVDhOTNTqMEHwZFla1o6NfiyMc4AnkR6dGM7S7XV
+SdLJNsaEq7drmacXD3qXcjrHi04xSXsdQ9sgkQBJ4iKnBYn+Bu9pw4odi5W4aVHv
+/vzE68o/GU5BU8yPnjJm0IuRRTGh/pMLn5nbf+f8PGGsztfEscSVJczKs3zVK4TP
+L771XcYG6FOz1oZNvMGRwjB0KL6Jh5s3JYUaRTdBTJLQPfLxx3kYx9fT+6zYYn0n
+e4Wc6B9x9ePB1QCqD4Kp2er1tjtmHwFjj+rpanMrM6+e6fH0vQdsEnZVlu51R1NS
+binGA+uwsWwUttwXhhYAo+LYja5sTvEuWH/ObwIDAQABAoIBAQC8QDGnMnmPdWJ+
+13FYY3cmwel+FXXjFDk5QpgK15A2rUz6a8XxO1d7d1wR+U84uH4v9Na6XQyWjaoD
+EyPQnuJiyAtgkZLUHoY244PGR5NsePEQlBSCKmGeF5w/j1LvP/2e9EmP4wKdQYJY
+nLxFNcgEBCFnFbKIU5n8fKa/klybCrwlBokenyBro02tqH4LL7h1YMRRrl97fv1V
+e/y/0WcMN+KnMglfz6haimBRV2yamCCHHmBImC+wzOgT/quqlxPfI+a3ScHxuA65
+3QyCavaqlPh+T3lXnN/Na4UWqFtzMmwgJX2x1zM5qiln46/JoDiXtagvV43L3rNs
+LhPRFeIRAoGBAPhEB7nNpEDNjIRUL6WpebWS9brKAVY7gYn7YQrKGhhCyftyaiBZ
+zYgxPaJdqYXf+DmkWlANGoYiwEs40QwkR/FZrvO4+Xh3n3dgtl59ZmieuoQvDsG+
+RYIj+TfBaqhewhZNMMl7dxz7DeyQhyRCdsvl3VqJM0RuOsIrzrhCIEItAoGBAN+K
+lgWI7swDpOEaLmu+IWMkGImh1LswXoZqIgi/ywZ7htZjPzidOIeUsMi+lrYsKojG
+uU3sBxASsf9kYXDnuUuUbGT5M/N2ipXERt7klUAA/f5sg1IKlTrabaN/HGs/uNtf
+Efa8v/h2VyTurdPCJ17TNpbOMDwX1qGM62tyt2CLAoGBAIHCnP8iWq18QeuQTO8b
+a3/Z9hHRL22w4H4MI6aOB6GSlxuTq6CJD4IVqo9IwSg17fnCy2l3z9s4IqWuZqUf
++XJOW8ELd2jdrT2qEOfGR1Z7UCVyqxXcq1vgDYx0zZh/HpalddB5dcJx/c8do2Ty
+UEE2PcHqYB9uNcvzNbLc7RtpAoGBALbuU0yePUTI6qGnajuTcQEPpeDjhRHWSFRZ
+ABcG1N8uMS66Mx9iUcNp462zgeP8iqY5caUZtMHreqxT+gWKK7F0+as7386pwElF
+QPXgO18QMMqHBIQb0vlBjJ1SRPBjSiSDTVEML1DljvTTOX7kEJHh6HdKrmBO5b54
+cqMQUo53AoGBAPVWRPUXCqlBz914xKna0ZUh2aesRBg5BvOoq9ey9c52EIU5PXL5
+0Isk8sWSsvhl3tjDPBH5WuL5piKgnCTqkVbEHmWu9s1T57Mw6NuxlPMLBWvyv4c6
+tB9brOxv0ui3qGMuBsBoDKbkNnwXyOXLyFg7O+H4l016A3mLQzJM+NGV
+-----END RSA PRIVATE KEY-----

.buildkite/certs/testnode.crt

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDYjCCAkqgAwIBAgIVAIClHav09e9XGWJrnshywAjUHTnXMA0GCSqGSIb3DQEB
+CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu
+ZXJhdGVkIENBMB4XDTIzMDMyODE3MDIzOVoXDTI2MDMyNzE3MDIzOVowEzERMA8G
+A1UEAxMIaW5zdGFuY2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCV
++t5/g6u2r3awCtzqp17KG0hRxzkVoJoF8DYzVh+Rv9ymxQW0C/U8dQihAjkZHaIA
+n49lSyNLkwWtmqQgPcimV4d6XuTYx2ahDixXYtjmoOSwH5dRtovKPCNKDPkUj9Vq
+NwMW0uB1VxniMKI4DnYFqBgHL9kQKhQqvas6Gx0X6ptGRCLYCtVxeFcau6nnkZJt
+urb+HNV5waOh0uTmsqnnslK3NjCQ/f030vPKxM5fOqOU5ajUHpZFJ6ZFmS32074H
+l+mZoRT/GtbnVtIg+CJXsWThF3/L4iBImv+rkY9MKX5fyMLJgmIJG68S90IQGR8c
+Z2lZYzC0J7zjMsYlODbDAgMBAAGjgYswgYgwHQYDVR0OBBYEFIDIcECn3AVHc3jk
+MpQ4r7Kc3WCsMB8GA1UdIwQYMBaAFJYCWKn16g+acbing4Vl45QGUBs0MDsGA1Ud
+EQQ0MDKCCWxvY2FsaG9zdIIIaW5zdGFuY2WHBH8AAAGHEAAAAAAAAAAAAAAAAAAA
+AAGCA2VzMTAJBgNVHRMEAjAAMA0GCSqGSIb3DQEBCwUAA4IBAQBtX3RQ5ATpfORM
+lrnhaUPGOWkjnb3p3BrdAWUaWoh136QhaXqxKiALQQhTtTerkXOcuquy9MmAyYvS
+9fDdGvLCAO8pPCXjnzonCHerCLGdS7f/eqvSFWCdy7LPHzTAFYfVWVvbZed+83TL
+bDY63AMwIexj34vJEStMapuFwWx05fstE8qZWIbYCL87sF5H/MRhzlz3ScAhQ1N7
+tODH7zvLzSxFGGEzCIKZ0iPFKbd3Y0wE6SptDSKhOqlnC8kkNeI2GjWsqVfHKsoF
+pDFmri7IfOucuvalXJ6xiHPr9RDbuxEXs0u8mteT5nFQo7EaEGdHpg1pNGbfBOzP
+lmj/dRS9
+-----END CERTIFICATE-----

.buildkite/certs/testnode.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAlfref4Ortq92sArc6qdeyhtIUcc5FaCaBfA2M1Yfkb/cpsUF
+tAv1PHUIoQI5GR2iAJ+PZUsjS5MFrZqkID3IpleHel7k2MdmoQ4sV2LY5qDksB+X
+UbaLyjwjSgz5FI/VajcDFtLgdVcZ4jCiOA52BagYBy/ZECoUKr2rOhsdF+qbRkQi
+2ArVcXhXGrup55GSbbq2/hzVecGjodLk5rKp57JStzYwkP39N9LzysTOXzqjlOWo
+1B6WRSemRZkt9tO+B5fpmaEU/xrW51bSIPgiV7Fk4Rd/y+IgSJr/q5GPTCl+X8jC
+yYJiCRuvEvdCEBkfHGdpWWMwtCe84zLGJTg2wwIDAQABAoIBAAEP7HYNNnDWdYMD
++WAtYM12X/W5s/wUP94juaBI4u4iZH2EZodlixEdZUCTXgq43WsDUhxX05s7cE+p
+H5DuSCHtoo2WHvGKAposwRDm2f3YVWQ2Xyb2ahNt69LYHHWrO+XQ60YYTa3r8Gn3
+7dFR3I016/jyn5DeEVaglvS1dfj2UG4ybR4KkMfcKd94X0rKvz3wzAhHIh+hwMtv
+sVk7V4vSnKf2mJXwIVECTolnEJEkCjWjjymgUJYKT8yN7JnAsHRcvMa6kWwIGrLp
+oQCEaJwYM6ynCRS989pLt3vA2iu5VkYhiHXJ9Ds/5b5yzhzmj+ymzKbFKrrUUrmn
++2Jp1K0CgYEAw8BchALsD/+JuoXjinA14MH7PZjIsXyhtPk+c4pk42iMNyg1J8XF
+Y/ITepLYsl2bZqQI1jOJdDqsTwIsva9r749lsmkYI3VOxhi7+qBK0sThR66C87lX
+iU2QpnZ9NloC6ort4a3MEvZ/gRQcXdBrNlNoza2p7PHAVDTnsdSrNKUCgYEAxCQV
+uo85oZyfnMufn/gcI9IeYOgiB0tO3a8cAFX2wQW1y935t6Z13ApUQc4EnCOH7ZBc
+td5kT+xGdRWnfPZ38FM1dd5MBdGE69s3q8pJDUExSgNLqaF6/5bD32qui66L3ugu
+eMjxrzqJsc2uQTPCs18SGsyRmf54DpY8HglOmUcCgYAGRDgx+a347SNJl1OrcOAo
+q80RMbzrAaRjmL8JD9se9I/YjC73cPtasbsx51WMkDaTWJj30nqJ//7YIKeyAtWf
+u6Vzyq19JRo6eTw7T7pVePwFQW7rwnks6hDBY3WqscL6IyxuVxP7X2zBgxVNY4ir
+Gox2WSLhdPPFPlRUewxoCQKBgAJvqE1u5fpZ5ame5dao0ECppXLyrymEB/C88g4X
+Az+WgJGNqkJbsO8QuccvdeMylcefmWcw4fIULzPZFwF4VjkH74wNPMh9t7buPBzI
+IGwnuSMAM3ph5RMzni8yNgTKIDaej6U0abwRcBBjS5zHtc1giusGS3CsNnWH7Cs7
+VlyVAoGBAK+prq9t9x3tC3NfCZH8/Wfs/X0T1qm11RiL5+tOhmbguWAqSSBy8OjX
+Yh8AOXrFuMGldcaTXxMeiKvI2cyybnls1MFsPoeV/fSMJbex7whdeJeTi66NOSKr
+oftUHvkHS0Vv/LicMEOufFGslb4T9aPJ7oyhoSlz9CfAutDWk/q/
+-----END RSA PRIVATE KEY-----

.buildkite/certs/testnode_no_san.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDIzCCAgugAwIBAgIVAMTO6uVx9dLox2t0lY4IcBKZXb5WMA0GCSqGSIb3DQEB
+CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu
+ZXJhdGVkIENBMB4XDTIwMDIyNjA1NTA1OVoXDTIzMDIyNTA1NTA1OVowEzERMA8G
+A1UEAxMIaW5zdGFuY2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDK
+YLTOikVENiN/qYupOsoXd7VYYnryyfCC/dK4FC2aozkbqjFzBdvPGAasoc4yEiH5
+CGeXMgJuOjk1maqetmdIsw00j4oHJviYsnGXzxxS5swhD7spcW4Uk4V4tAUzrbfT
+vW/2WW/yYCLe5phVb2chz0jL+WYb4bBmdfs/t6RtP9RqsplYAmVp3gZ6lt2YNtvE
+k9gz0TVk3DuO1TquIClfRYUjuywS6xDSvxJ8Jl91EfDWM8QU+9F+YAtiv74xl2U3
+P0wwMqNvMxf9/3ak3lTQGsgO4L6cwbKpVLMMzxSVunZz/sgl19xy3qHHz1Qr2MjJ
+/2c2J7vahUL4NPRkjJClAgMBAAGjTTBLMB0GA1UdDgQWBBS2Wn8E2VZv4oenY+pR
+O8G3zfQXhzAfBgNVHSMEGDAWgBSWAlip9eoPmnG4p4OFZeOUBlAbNDAJBgNVHRME
+AjAAMA0GCSqGSIb3DQEBCwUAA4IBAQAvwPvCiJJ6v9jYcyvYY8I3gP0oCwrylpRL
+n91UlgRSHUmuAObyOoVN5518gSV/bTU2SDrstcLkLFxHvnfpoGJoxsQEHuGxwDRI
+nhYNd62EKLerehNM/F9ILKmvTh8f6QPCzjUuExTXv+63l2Sr6dBS7FHsGs6UKUYO
+llM/y9wMZ1LCuZuBg9RhtgpFXRSgDM9Z7Begu0d/BPX9od/qAeZg9Arz4rwUiCN4
+IJOMEBEPi5q1tgeS0Fb1Grpqd0Uz5tZKtEHNKzLG+zSMmkneL62Nk2HsmEFZKwzg
+u2pU42UaUE596G6o78s1aLn9ICcElPHTjiuZNSiyuu9IzvFDjGQw
+-----END CERTIFICATE-----

.buildkite/certs/testnode_no_san.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAymC0zopFRDYjf6mLqTrKF3e1WGJ68snwgv3SuBQtmqM5G6ox
+cwXbzxgGrKHOMhIh+QhnlzICbjo5NZmqnrZnSLMNNI+KByb4mLJxl88cUubMIQ+7
+KXFuFJOFeLQFM623071v9llv8mAi3uaYVW9nIc9Iy/lmG+GwZnX7P7ekbT/UarKZ
+WAJlad4GepbdmDbbxJPYM9E1ZNw7jtU6riApX0WFI7ssEusQ0r8SfCZfdRHw1jPE
+FPvRfmALYr++MZdlNz9MMDKjbzMX/f92pN5U0BrIDuC+nMGyqVSzDM8Ulbp2c/7I
+Jdfcct6hx89UK9jIyf9nNie72oVC+DT0ZIyQpQIDAQABAoIBADAh7f7NjgnaInlD
+ds8KB3SraPsbeQhzlPtiqRJU4j/MIFH/GYG03AGWQkget67a9y+GmzSvlTpoKKEh
+6h2TXl9BDpv4o6ht0WRn1HJ5tM/Wyqf2WNpTew3zxCPgFPikkXsPrChYPzLTQJfp
+GkP/mfTFmxfAOlPZSp4j41zVLYs53eDkAegFPVfKSr1XNNJ3QODLPcIBfxBYsiC9
+oU+jRW8xYuj31cEl5k5UqrChJ1rm3mt6cguqXKbISuoSvi13gXI6DccqhuLAU+Kr
+ib2XYrRP+pWocZo/pM9WUVoNGtFxfY88sAQtvG6gDKo2AURtFyq84Ow0h9mdixV/
+gRIDPcECgYEA5nEqE3OKuG9WuUFGXvjtn4C0F6JjflYWh7AbX51S4F6LKrW6/XHL
+Rg4BtF+XReT7OQ6llsV8kZeUxsUckkgDLzSaA8lysNDV5KkhAWHfRqH//QKFbqZi
+JL9t3x63Qt81US8s2hQk3khPYTRM8ZB3xHiXvZYSGC/0x/DxfEO3QJECgYEA4NK5
+sxtrat8sFz6SK9nWEKimPjDVzxJ0hxdX4tRq/JdOO5RncawVqt6TNP9gTuxfBvhW
+MhJYEsQj8iUoL1dxo9d1eP8HEANNV0iX5OBvJNmgBp+2OyRSyr+PA55+wAxYuAE7
+QKaitOjW57fpArNRt2hQyiSzTuqUFRWTWJHCWNUCgYAEurPTXF6vdFGCUc2g61jt
+GhYYGhQSpq+lrz6Qksj9o9MVWE9zHh++21C7o+6V16I0RJGva3QoBMVf4vG4KtQt
+5tV2WG8LI+4P2Ey+G4UajP6U8bVNVQrUmD0oBBhcvfn5JY+1Fg6/pRpD82/U0VMz
+7AmpMWhDqNBMPiymkTk0kQKBgCuWb05cSI0ly4SOKwS5bRk5uVFhYnKNH255hh6C
+FGP4acB/WzbcqC7CjEPAJ0nl5d6SExQOHmk1AcsWjR3wlCWxxiK5PwNJwJrlhh1n
+reS1FKN0H36D4lFQpkeLWQOe4Sx7gKNeKzlr0w6Fx3Uwku0+Gju2tdTdAey8jB6l
+08opAoGAEe1AuR/OFp2xw6V8TH9UHkkpGxy+OrXI6PX6tgk29PgB+uiMu4RwbjVz
+1di1KKq2XecAilVbnyqY+edADxYGbSnci9x5wQRIebfMi3VXKtV8NQBv2as6qwtW
+JDcQUWotOHjpdvmfJWWkcBhbAKrgX8ukww00ZI/lC3/rmkGnBBg=
+-----END RSA PRIVATE KEY-----

.buildkite/functions/cleanup.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+#
+# Shared cleanup routines between different steps
+#
+# Please source .ci/functions/imports.sh as a whole not just this file
+#
+# Version 1.0.0
+# - Initial version after refactor
+
+function cleanup_volume {
+  if [[ "$(docker volume ls -q -f name=$1)" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m"
+    (docker volume rm "$1") || true
+  fi
+}
+function container_running {
+  if [[ "$(docker ps -q -f name=$1)" ]]; then
+    return 0;
+    else return 1;
+  fi
+}
+function cleanup_node {
+  if container_running "$1"; then
+    echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m"
+    (docker container rm --force --volumes "$1") || true
+  fi
+  if [[ -n "$1" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing volume $1-${suffix}-data\033[0m"
+    cleanup_volume "$1-${suffix}-data"
+  fi
+}
+function cleanup_network {
+  if [[ "$(docker network ls -q -f name=$1)" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing network $1\033[0m"
+    (docker network rm "$1") || true
+  fi
+}
+
+function cleanup_trap {
+  status=$?
+  set +x
+  if [[ "$DETACH" != "true" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m clean the network if not detached (start and exit)\033[0m"
+    cleanup_all_in_network "$1"
+  fi
+  # status is 0 or SIGINT
+  if [[ "$status" == "0" || "$status" == "130" ]]; then
+    echo -e "\n\033[32;1mSUCCESS run-tests\033[0m"
+    exit 0
+  else
+    echo -e "\n\033[31;1mFAILURE during run-tests\033[0m"
+    exit ${status}
+  fi
+};
+function cleanup_all_in_network {
+
+  if [[ -z "$(docker network ls -q -f name="^$1\$")" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m $1 is already deleted\033[0m"
+    return 0
+  fi
+  containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' $1)
+  while read -r container; do
+    cleanup_node "$container"
+  done <<< "$containers"
+  cleanup_network $1
+  echo -e "\033[32;1mSUCCESS:\033[0m Cleaned up and exiting\033[0m"
+};

.buildkite/functions/imports.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+#
+# Sets up all the common variables and imports relevant functions
+#
+# Version 1.0.1
+# - Initial version after refactor
+# - Validate STACK_VERSION asap
+
+function require_stack_version() {
+  if [[ -z $STACK_VERSION ]]; then
+    echo -e "\033[31;1mERROR:\033[0m Required environment variable [STACK_VERSION] not set\033[0m"
+    exit 1
+  fi
+}
+
+require_stack_version
+
+if [[ -z $es_node_name ]]; then
+  # only set these once
+  set -euo pipefail
+  export DETACH=${DETACH-false}
+  export CLEANUP=${CLEANUP-false}
+
+  export es_node_name=instance
+  export elastic_password=changeme
+  export elasticsearch_image=elasticsearch
+  export elasticsearch_url=https://elastic:${elastic_password}@${es_node_name}:9200
+  export external_elasticsearch_url=${elasticsearch_url/$es_node_name/localhost}
+  export elasticsearch_container="${elasticsearch_image}:${STACK_VERSION}"
+
+  export suffix=rest-test
+  export moniker=$(echo "$elasticsearch_container" | tr -C "[:alnum:]" '-')
+  export network_name=${moniker}${suffix}
+
+  export ssl_cert="${script_path}/certs/testnode.crt"
+  export ssl_key="${script_path}/certs/testnode.key"
+  export ssl_ca="${script_path}/certs/ca.crt"
+
+fi
+
+  export script_path=$(dirname $(realpath -s $0))
+  source $script_path/functions/cleanup.sh
+  source $script_path/functions/wait-for-container.sh
+  trap "cleanup_trap ${network_name}" EXIT
+
+
+if [[ "$CLEANUP" == "true" ]]; then
+  cleanup_all_in_network $network_name
+  exit 0
+fi
+
+echo -e "\033[34;1mINFO:\033[0m Creating network $network_name if it does not exist already \033[0m"
+docker network inspect "$network_name" > /dev/null 2>&1 || docker network create "$network_name"
+

.buildkite/functions/wait-for-container.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+#
+# Exposes a routine scripts can call to wait for a container if that container set up a health command
+#
+# Please source .ci/functions/imports.sh as a whole not just this file
+#
+# Version 1.0.1
+# - Initial version after refactor
+# - Make sure wait_for_contiainer is silent
+
+function wait_for_container {
+  set +x
+  until ! container_running "$1" || (container_running "$1" && [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "starting" ]]); do
+    echo ""
+    docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${1}
+    echo -e "\033[34;1mINFO:\033[0m waiting for node $1 to be up\033[0m"
+    sleep 20;
+  done;
+
+  # Always show logs if the container is running, this is very useful both on CI as well as while developing
+  if container_running $1; then
+    docker logs $1
+  fi
+
+  if ! container_running $1 || [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "healthy" ]]; then
+    cleanup_all_in_network $2
+    echo
+    echo -e "\033[31;1mERROR:\033[0m Failed to start $1 in detached mode beyond health checks\033[0m"
+    echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m"
+    return 1
+  else
+    echo
+    echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${1} on docker network: ${network_name}\033[0m"
+    return 0
+  fi
+}