diff --git a/.travis.yml b/.travis.yml
index a193841..cb672f3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,29 +1,13 @@
----
-language: python
-python: "2.7"
 sudo: required
-dist: trusty
-before_install:
-  - sudo apt-get update -qq
+language: python
+services: docker
+
+cache: pip
 
 install:
-  - pip install ansible
-  - "echo localhost > inventory remote_user=root"
+  - pip install ansible docker
+  - ansible-galaxy install -r tests/requirements.yml -p tests/roles/
 
 script:
-  - "ansible-playbook -i inventory role.yml --connection=local --sudo"
-  - >
-    ansible-playbook -i inventory role.yml --connection=local --sudo
-    | grep -q 'changed=0.*failed=0'
-    && (echo 'Idempotence test: pass' && exit 0)
-    || (echo 'Idempotence test: fail' && exit 1)
-  - >
-    ansible-playbook -i inventory role.yml --connection=local --sudo
-    --tags unattended | grep -q 'changed=0.*failed=0'
-    && (echo 'Idempotence test: pass' && exit 0)
-    || (echo 'Idempotence test: fail' && exit 1)
-  - apt-config dump | grep 'Unattended-Upgrade "1"'
-  - sudo unattended-upgrades --dry-run
-
-notifications:
-  webhooks: https://galaxy.ansible.com/api/v1/notifications/
+  - ansible --version
+  - tests/test.sh
diff --git a/Vagrantfile b/Vagrantfile
deleted file mode 100644
index 37bb4f7..0000000
--- a/Vagrantfile
+++ /dev/null
@@ -1,34 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
-VAGRANTFILE_API_VERSION = "2"
-
-Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-  config.vm.define "debian7" do |debian|
-    debian.vm.box = "debian/wheezy64"
-  end
-
-  config.vm.define "debian8" do |debian|
-    debian.vm.box = "debian/jessie64"
-  end
-
-  config.vm.define "ubuntu1204" do |ubuntu|
-    ubuntu.vm.box = "ubuntu/precise64"
-  end
-
-  config.vm.define "ubuntu1404" do |ubuntu|
-    ubuntu.vm.box = "ubuntu/trusty64"
-  end
-
-  config.vm.define "ubuntu1604" do |ubuntu|
-    ubuntu.vm.box = "ubuntu/xenial64"
-  end
-
-  config.vm.provision :shell, inline: "sudo apt-get install -y python python-apt aptitude"
-  config.vm.provision "ansible" do |ansible|
-    ansible.playbook = "role.yml"
-    ansible.verbose = "vv"
-    ansible.sudo = true
-  end
-end
diff --git a/ansible.cfg b/ansible.cfg
deleted file mode 100644
index c2e2d9f..0000000
--- a/ansible.cfg
+++ /dev/null
@@ -1,3 +0,0 @@
-# Used for Vagrant and CI testing
-[defaults]
-roles_path = ../
diff --git a/role.yml b/role.yml
deleted file mode 100644
index 5a4a0e8..0000000
--- a/role.yml
+++ /dev/null
@@ -1,4 +0,0 @@
----
-- hosts: all
-  roles:
-    - ansible-role-unattended-upgrades
diff --git a/test-upgrades-on-virtual-machines.sh b/test-upgrades-on-virtual-machines.sh
deleted file mode 100755
index b058fbd..0000000
--- a/test-upgrades-on-virtual-machines.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-set -e
-
-SD=$(dirname $0)
-
-cd $SD
-
-vagrant destroy --force
-
-vagrant up
-
-VMS="
-debian7
-debian8
-ubuntu1204
-ubuntu1404
-ubuntu1604
-"
-
-for name in $VMS; do
-
-    echo ""
-    echo "============================================================"
-    echo "testing on $name"
-    echo "============================================================"
-    echo ""
-
-    vagrant ssh -c "sudo unattended-upgrades -d" $name
-done
diff --git a/tests/ansible.cfg b/tests/ansible.cfg
new file mode 100644
index 0000000..6abf5b7
--- /dev/null
+++ b/tests/ansible.cfg
@@ -0,0 +1,3 @@
+[defaults]
+roles_path = ../../
+retry_files_enabled = False
diff --git a/tests/inventory b/tests/inventory
new file mode 100644
index 0000000..42de3a1
--- /dev/null
+++ b/tests/inventory
@@ -0,0 +1 @@
+localhost ansible_connection=local ansible_python_interpreter="/usr/bin/env python"
diff --git a/tests/requirements.yml b/tests/requirements.yml
new file mode 100644
index 0000000..562771a
--- /dev/null
+++ b/tests/requirements.yml
@@ -0,0 +1,3 @@
+---
+- src: chrismeyersfsu.provision_docker
+  name: provision_docker
diff --git a/tests/test.sh b/tests/test.sh
new file mode 100755
index 0000000..84096bf
--- /dev/null
+++ b/tests/test.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Exit on any individual command failure
+set -e
+
+# Pretty colors.
+red='\033[0;31m'
+green='\033[0;32m'
+neutral='\033[0m'
+
+section() {
+  echo -e "\033[33;1m$1\033[0m"
+}
+
+fold_start() {
+  echo -e "travis_fold:start:$1\033[33;1m$2\033[0m"
+}
+
+fold_end() {
+  echo -e "\ntravis_fold:end:$1\r"
+}
+
+# Ensure we are in the tests dir
+cd "$( dirname "${BASH_SOURCE[0]}" )"
+
+section "Syntax check"
+ansible-playbook -i inventory --syntax-check test.yml
+section "Running role"
+ansible-playbook -i inventory test.yml
diff --git a/tests/test.yml b/tests/test.yml
new file mode 100644
index 0000000..ae1675f
--- /dev/null
+++ b/tests/test.yml
@@ -0,0 +1,65 @@
+---
+- name: Bring up Docker containers
+  hosts: localhost
+  gather_facts: false
+  vars:
+    inventory:
+      - name: ubuntu_rolling
+        image: "ubuntu:rolling"
+      - name: ubuntu_latest
+        image: "ubuntu:latest"
+      - name: ubuntu_trusty
+        image: "ubuntu:trusty"
+      - name: debian_testing
+        image: "debian:testing"
+      - name: debian_stable
+        image: "debian:stable"
+      - name: debian_oldstable
+        image: "debian:oldstable"
+  roles:
+    - role: provision_docker
+      provision_docker_inventory: "{{ inventory }}"
+      provision_docker_privileged: true
+      provision_docker_use_docker_connection: true
+
+- name: Test role
+  hosts: docker_containers
+  gather_facts: false
+  pre_tasks:
+    - name: Provision Python
+      raw: bash -c "test -e /usr/bin/python || (apt-get -y update && apt-get install -y python-simplejson)"
+      register: output
+      changed_when: output.stdout
+    - setup: # Gather facts
+  vars:
+    unattended_autofix_interrupted_dpkg: false
+    unattended_minimal_steps: true
+    unattended_install_on_shutdown: true
+    unattended_automatic_reboot: true
+  roles:
+    # Searched for in ../.. (see ansible.cfg)
+    - ansible-role-unattended-upgrades
+  tasks:
+    - name: Idempotency check
+      include_role:
+        name: ansible-role-unattended-upgrades
+      register: idempotency
+    - fail:
+        msg: Role failed idempotency check
+      when: idempotency.changed
+
+    - name: Get apt-config variables
+      shell: apt-config dump
+      register: aptconfig
+    - name: Check for registered variables
+      assert:
+        that: item in aptconfig.stdout
+      with_items:
+        - 'APT::Periodic::Unattended-Upgrade "1"'
+        - 'Unattended-Upgrade::AutoFixInterruptedDpkg "false"'
+        - 'Unattended-Upgrade::MinimalSteps "true"'
+        - 'Unattended-Upgrade::InstallOnShutdown "true"'
+        - 'Unattended-Upgrade::Automatic-Reboot "true"'
+
+    - name: Dry run unattended-upgrades
+      command: /usr/bin/unattended-upgrades --dry-run