Skip to content

Commit

Permalink
Merge pull request #351 from bmbouter/merge-0.1.1-to-0.1-branch
Browse files Browse the repository at this point in the history
Merge 0.1.1 to 0.1 branch
  • Loading branch information
bmbouter authored Jan 31, 2020
2 parents 57ebf80 + 322e7bf commit a72de28
Show file tree
Hide file tree
Showing 24 changed files with 787 additions and 564 deletions.
30 changes: 17 additions & 13 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
---
sudo: required
# https://docs.travis-ci.com/user/trusty-ci-environment/
dist: xenial
dist: bionic
language: python
python:
# Fedora has 3.6 available (python36.x86_64), but pulp container images
Expand All @@ -19,13 +19,6 @@ env:
- TEST=pulp
- TEST=docs
- TEST=bindings
matrix:
exclude:
- python: '3.6'
env: TEST=bindings
- python: '3.6'
env: TEST=docs
fast_finish: true
services:
- postgresql
- redis-server
Expand All @@ -52,28 +45,33 @@ after_failure:
- sudo kubectl logs -l app=pulp-worker --tail=10000
jobs:
include:
- stage: deploy-plugin-to-pypi
- stage: deploy
name: "Deploy plugin to pypi"
before_install: skip
install: skip
before_script: skip
script: bash .travis/publish_plugin_pypi.sh
if: tag IS present
- stage: publish-daily-client-gem
- stage: deploy
name: "Publish client to rubygems"
script: bash .travis/publish_client_gem.sh
env:
- TEST=bindings
if: type = cron
- stage: publish-daily-client-pypi
- stage: deploy
name: "Publish client to pypi"
script: bash .travis/publish_client_pypi.sh
env:
- TEST=bindings
if: type = cron
- stage: publish-client-gem
- stage: deploy
name: "Publish client to rubygems"
script: bash .travis/publish_client_gem.sh
env:
- TEST=bindings
if: tag IS present
- stage: publish-client-pypi
- stage: deploy
name: "Publish client to pypi"
script: bash .travis/publish_client_pypi.sh
env:
- TEST=bindings
Expand All @@ -82,6 +80,12 @@ jobs:
env:
- TEST=performance
if: type = cron
- stage: cherry-pick
if: type = cron
before_install: skip
install: skip
before_script: skip
script: bash .travis/cherrypick.sh
notifications:
irc:
channels:
Expand Down
32 changes: 20 additions & 12 deletions .travis/before_install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,10 @@ if [[ -n $(echo -e $COMMIT_MSG | grep -P "Required PR:.*" | grep -v "https") ]];
exit 1
fi

if [ "$TRAVIS_PULL_REQUEST" != "false" ] || [ -z "$TRAVIS_TAG" -a "$TRAVIS_BRANCH" != "master"]
if [ "$TRAVIS_PULL_REQUEST" != "false" ] || [ -z "$TRAVIS_TAG" -a "$TRAVIS_BRANCH" != "master" ]
then
export PULP_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore\/pull\/(\d+)' | awk -F'/' '{print $7}')
export PULP_SMASH_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/PulpQE\/pulp-smash\/pull\/(\d+)' | awk -F'/' '{print $7}')
export PULP_SMASH_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-smash\/pull\/(\d+)' | awk -F'/' '{print $7}')
export PULP_ROLES_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/ansible-pulp\/pull\/(\d+)' | awk -F'/' '{print $7}')
export PULP_BINDINGS_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-openapi-generator\/pull\/(\d+)' | awk -F'/' '{print $7}')
export PULP_OPERATOR_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-operator\/pull\/(\d+)' | awk -F'/' '{print $7}')
Expand Down Expand Up @@ -56,31 +56,39 @@ cd ..
git clone --depth=1 https://github.com/pulp/ansible-pulp.git
if [ -n "$PULP_ROLES_PR_NUMBER" ]; then
cd ansible-pulp
git fetch --depth=1 origin +refs/pull/$PULP_ROLES_PR_NUMBER/merge
git checkout FETCH_HEAD
git fetch --depth=1 origin pull/$PULP_ROLES_PR_NUMBER/head:$PULP_ROLES_PR_NUMBER
git checkout $PULP_ROLES_PR_NUMBER
cd ..
fi


git clone --depth=1 https://github.com/pulp/pulp-operator.git
if [ -n "$PULP_OPERATOR_PR_NUMBER" ]; then
cd pulp-operator
git fetch --depth=1 origin +refs/pull/$PULP_OPERATOR_PR_NUMBER/merge
git checkout FETCH_HEAD
git fetch --depth=1 origin pull/$PULP_OPERATOR_PR_NUMBER/head:$PULP_OPERATOR_PR_NUMBER
git checkout $PULP_OPERATOR_PR_NUMBER
RELEASE_VERSION=v0.9.0
curl -LO https://github.com/operator-framework/operator-sdk/releases/download/${RELEASE_VERSION}/operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu
chmod +x operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu && sudo mkdir -p /usr/local/bin/ && sudo cp operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu /usr/local/bin/operator-sdk && rm operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu
sudo operator-sdk build --image-builder=docker quay.io/pulp/pulp-operator:latest
cd ..
fi

git clone https://github.com/pulp/pulp-openapi-generator.git
if [ -n "$PULP_BINDINGS_PR_NUMBER" ]; then
cd pulp-openapi-generator
git fetch origin pull/$PULP_BINDINGS_PR_NUMBER/head:$PULP_BINDINGS_PR_NUMBER
git checkout $PULP_BINDINGS_PR_NUMBER
cd ..
fi


git clone --depth=1 https://github.com/pulp/pulpcore.git
git clone --depth=1 https://github.com/pulp/pulpcore.git --branch master

if [ -n "$PULP_PR_NUMBER" ]; then
cd pulpcore
git fetch --depth=1 origin +refs/pull/$PULP_PR_NUMBER/merge
git checkout FETCH_HEAD
git fetch --depth=1 origin pull/$PULP_PR_NUMBER/head:$PULP_PR_NUMBER
git checkout $PULP_PR_NUMBER
cd ..
fi

Expand All @@ -90,12 +98,12 @@ fi
# build (they will be installed as dependencies of the plugin).
if [ -z "$TRAVIS_TAG" ]; then

git clone --depth=1 https://github.com/PulpQE/pulp-smash.git
git clone --depth=1 https://github.com/pulp/pulp-smash.git

if [ -n "$PULP_SMASH_PR_NUMBER" ]; then
cd pulp-smash
git fetch --depth=1 origin +refs/pull/$PULP_SMASH_PR_NUMBER/merge
git checkout FETCH_HEAD
git fetch --depth=1 origin pull/$PULP_SMASH_PR_NUMBER/head:$PULP_SMASH_PR_NUMBER
git checkout $PULP_SMASH_PR_NUMBER
cd ..
fi

Expand Down
10 changes: 10 additions & 0 deletions .travis/before_script.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,16 @@ else
sed "s/localhost/$(hostname)/g" ../pulpcore/.travis/pulp-smash-config.json > ~/.config/pulp_smash/settings.json
fi


# set up pulp-fixtures docker container
if [[ "$TEST" == 'pulp' ]]; then
docker run -d -p 0.0.0.0:8000:80 quay.io/pulp/pulp-fixtures:latest
fi
cat ~/.config/pulp_smash/settings.json | \
jq "setpath([\"custom\",\"fixtures_origin\"]; \"http://$(hostname):8000/fixtures/\")" > temp.json
cat temp.json > ~/.config/pulp_smash/settings.json


if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' ]]; then
# Many tests require pytest/mock, but users do not need them at runtime
# (or to add plugins on top of pulpcore or pulp container images.)
Expand Down
1 change: 1 addition & 0 deletions .travis/check_commit.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ fi

for sha in `git log --format=oneline --no-merges "$RANGE" | cut '-d ' -f1`
do
pip install requests
python .travis/validate_commit_message.py $sha
VALUE=$?

Expand Down
121 changes: 121 additions & 0 deletions .travis/cherrypick.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
import os
import subprocess
import uuid
from git import Repo
from github import Github
from github.GithubException import UnknownObjectException

PR_LABEL = "Needs Cherry Pick"
STABLE_BRANCH = os.environ["STABLE_BRANCH"]
REPOSITORY = os.environ["TRAVIS_REPO_SLUG"]
GITHUB_USER = os.environ["GITHUB_USER"]
GITHUB_TOKEN = os.environ["GITHUB_TOKEN"]


def blobs_match(commit1, commit2):
"""Compare the file blobs of two commits."""
if len(commit1.files) != len(commit2.files):
return False

for cf in commit1.files:
files = [f for f in commit2.files if f.sha == cf.sha and f.filename == cf.filename]
if len(files) == 0:
return False

return True


def get_merged_commits(pr):
"""
Find the merged commits for a PR.
Github allows PRs to be merged with 3 different strategies: merge, rebase, and squashing. Github
doesn't record how a PR was merged in the API so we need to examine the merge commit to
determine how the pr was merged and then return the commits based on that.
"""
merge_commit = pr.base.repo.get_commit(pr.merge_commit_sha)

if len(merge_commit.parents) > 1:
# PR was just merged and we can use the PR's commits
return pr.get_commits()
elif blobs_match(merge_commit, pr.get_commits().reversed[0]):
# if the last PR commit is the same as the merge commit then the PR was rebased
commits = []
commit = merge_commit
for _ in pr.get_commits():
if len(commit.parents) > 1:
print(f"Ran into problem attempting to cherry pick {commit.sha}.")
exit(1)
commits.append(commit)
commit = commit.parents[0]
commits.reverse()
return commits
else:
# we have a squashed commit. we can just use the merge commit
return [merge_commit]


print(f"Processing cherry picks for {REPOSITORY}.")

# prepare our local repo to receive git cherry picks
repo = Repo(os.getcwd())
remote_url = f"https://{GITHUB_USER}:{GITHUB_TOKEN}@github.com/{REPOSITORY}.git"
remote = repo.create_remote("auth-origin", url=remote_url)
remote.fetch()
repo.git.checkout(STABLE_BRANCH)

g = Github(GITHUB_TOKEN)
grepo = g.get_repo(REPOSITORY)
(label,) = (l for l in grepo.get_labels() if l.name == PR_LABEL)
issues = grepo.get_issues(labels=[label], state="all")

cherrypicks = []

for issue in issues:
try:
pr = grepo.get_pull(issue.number)
except UnknownObjectException:
# this issue is not a PR. skip it.
continue

if not pr.merged:
print(f"Pull request {pr.number} not merged. Skipping.")
continue

print(f"Attempting to cherry-pick commits for {pr.html_url}.")

for commit in get_merged_commits(pr):
# looks like GitPython doesn't support cherry picks
ret = subprocess.run(["git", "cherry-pick", "-x", commit.sha], stderr=subprocess.PIPE)

if ret.returncode != 0:
print(f"Failed to cherry-pick commit {commit.sha}: {ret.stderr.decode('ascii')}")
exit(1)
else:
cherrypicks.append(issue)
print(f"Cherry-picked commit {commit.sha}.")

# check if we cherry picked anything
if len(cherrypicks) == 0:
print(f"No cherry picks detected.")
exit(0)

# push our changes
print(f"Attempting push changes to {REPOSITORY}.")
cherry_pick_branch = f"cherry-picks-{uuid.uuid4()}"
remote.push(refspec=f"{STABLE_BRANCH}:{cherry_pick_branch}")
print(f"Pushed cherry picks to {cherry_pick_branch}.")

# create a pull request
body = f"Cherry picking #{(', #').join(str(cp.number) for cp in cherrypicks)}."
pr = grepo.create_pull(f"Cherry picks to {STABLE_BRANCH}", body, STABLE_BRANCH, cherry_pick_branch)
print(f"Created pull request {pr.html_url}.")

# remove the cherry pick label from our PRs
for cp in cherrypicks:
labels = cp.labels
labels.remove(label)
cp.edit(labels=labels)
print(f"Removed label '{PR_LABEL}' from PR #{cp.number}.")

print("Cherry picking complete.")
5 changes: 5 additions & 0 deletions .travis/cherrypick.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pip install PyGithub GitPython yq

export STABLE_BRANCH=$(yq -r ".stable_branch" template_config.yml)

python ./.travis/cherrypick.py
25 changes: 10 additions & 15 deletions .travis/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,7 @@
set -euv

if [ "$TEST" = 'docs' ]; then

pip install -r ../pulpcore/doc_requirements.txt

pip install -r doc_requirements.txt
fi

Expand Down Expand Up @@ -48,37 +46,34 @@ else
fi


PLUGIN=pulp_file


# For pulpcore, and any other repo that might check out some plugin PR
if [ -n "$TRAVIS_TAG" ]; then
# Install the plugin only and use published PyPI packages for the rest
cat > vars/vars.yaml << VARSYAML
---
images:
- ${PLUGIN}-${TAG}:
image_name: $PLUGIN
- pulp_file-${TAG}:
image_name: pulp_file
tag: $TAG
plugins:
- ./$PLUGIN
- ./pulp_file
VARSYAML
else
cat > vars/vars.yaml << VARSYAML
---
images:
- ${PLUGIN}-${TAG}:
image_name: $PLUGIN
- pulp_file-${TAG}:
image_name: pulp_file
tag: $TAG
pulpcore: ./pulpcore
plugins:
- ./$PLUGIN
- ./pulp_file
VARSYAML
fi
ansible-playbook build.yaml
ansible-playbook -v build.yaml

cd $TRAVIS_BUILD_DIR/../pulp-operator
# Tell pulp-perator to deploy our image
# NOTE: With k3s 1.17, $TAG must be quoted. So that 3.0 does not become 3.
cat > deploy/crds/pulpproject_v1alpha1_pulp_cr.yaml << CRYAML
apiVersion: pulpproject.org/v1alpha1
kind: Pulp
Expand All @@ -90,8 +85,8 @@ spec:
access_mode: "ReadWriteOnce"
# We have a little over 40GB free on Travis VMs/instances
size: "40Gi"
image: $PLUGIN
tag: $TAG
image: pulp_file
tag: "${TAG}"
database_connection:
username: pulp
password: pulp
Expand Down
Loading

0 comments on commit a72de28

Please sign in to comment.