diff --git a/.kitchen.yml b/.kitchen.yml index ae7e508..a5f67c1 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -63,5 +63,7 @@ suites: - { device: "/dev/sdc" } - { device: "/dev/sdd" } run_list: + - recipe[ceph_test::radosgw_zone] - recipe[ceph::all_in_one] - recipe[ceph_test::cephfs] + - recipe[ceph_test::radosgw_restart] diff --git a/README.md b/README.md index 892708c..7693963 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,27 @@ The ceph\_cephfs LWRP provides an easy way to mount CephFS. It will automaticall - :use\_fuse - whether to use ceph-fuse or the kernel client to mount the filesystem. ceph-fuse is updated more often, but the kernel client allows for subdirectory mounting. Defaults to true - :cephfs\_subdir - which CephFS subdirectory to mount. Defaults to '/'. An exception will be thrown if this option is set to anything other than '/' if use\_fuse is also true +### ceph\_radosgw + +The ceph\_radosgw LWRP provides an easy way to setup customized radosgw servers. The LWRP itself just adds a node attribute, and then the radosgw recipe does all the setup. Due to the way Chef processes the runlist, this resource has to be explicitly converged before compiling the radosgw recipe. Any recipe that calls this LWRP has to say `.run_action(:add)` after the do..end block. Alternatively, the node attributes can be set with a wrapper cookbook. + +#### Actions + +- :add - Adds the given information as a custom radosgw configuration. + +#### Parameters + +- :name - name attribute, not used +- :region - what region should be set in the ceph.conf for this radosgw +- :region\_root\_pool - what rados pool should be used to load region information +- :zone - what zone should be set in the ceph.conf for this radosgw +- :zone\_root\_pool - what rados pool should be used to load zone information +- :keyname - what name to use when generating the cephx key for this radosgw. Defaults to client.radosgw.#{zone}.#{hostname} +- :dns\_name - what dns name to use in ceph.conf and Apache +- :dns\_aliases - an optional array of other hostnames to handle requests for in Apache +- :socket\_path - the path to the socket that radosgw will use for fastcgi communications. Defaults to /var/run/ceph/ceph-radosgw.#{zone}.#{hostname} +- :print\_continue - whether to set the `print continue` option in ceph.conf. Defaults to nil, which doesn't set the option, true or false sets the option to the given setting. + ## LICENSE AND AUTHORS * Author: Kyle Bader diff --git a/attributes/radosgw.rb b/attributes/radosgw.rb index 11e39c2..4490d5a 100644 --- a/attributes/radosgw.rb +++ b/attributes/radosgw.rb @@ -27,8 +27,10 @@ default['ceph']['radosgw']['use_apache_fork'] = true default['ceph']['radosgw']['init_style'] = node['ceph']['init_style'] -default['ceph']['radosgw']['path'] = '/var/www' +default['ceph']['radosgw']['default'] = true # whether to deploy the default radosgw +default['ceph']['radosgw']['instances'] = {} # map from zone name to settings +default['ceph']['radosgw']['path'] = '/var/www' if node['platform_family'] == 'suse' default['ceph']['radosgw']['path'] = '/srv/www/ceph-radosgw' end diff --git a/providers/radosgw.rb b/providers/radosgw.rb new file mode 100644 index 0000000..9bea24c --- /dev/null +++ b/providers/radosgw.rb @@ -0,0 +1,46 @@ +# +# Author:: Kyle Bader +# Cookbook Name:: ceph +# Recipe:: radosgw +# +# Copyright 2011, DreamHost Web Hosting +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +use_inline_resources + +def whyrun_supported? + true +end + +action :add do + node.default['ceph']['is_radosgw'] = true + region = @new_resource.region + zone = @new_resource.zone + ::Chef::Log.info("Saving information for radosgw #{zone}") + + instance = {} + instance['rgw region'] = @new_resource.region + instance['rgw region root pool'] = @new_resource.region_root_pool || ".#{region}.rgw.root" + instance['rgw zone'] = @new_resource.zone + instance['rgw zone root pool'] = @new_resource.zone_root_pool || ".#{@new_resource.zone}.rgw.root" + keyname = @new_resource.keyname || "client.radosgw.#{zone}.#{node['hostname']}" + instance['keyring'] = "/etc/ceph/ceph.#{keyname}.keyring" + instance['rgw socket path'] = @new_resource.socket_path || "/var/run/ceph-radosgw/radosgw.#{zone}.#{node['hostname']}" + instance['rgw dns name'] = @new_resource.dns_name + instance['api_aliases'] = @new_resource.dns_aliases if @new_resource.dns_aliases + instance['rgw print continue'] = @new_resource.print_continue.to_s unless @new_resource.print_continue.nil? + + node.default['ceph']['radosgw']['instances'][keyname] = instance + +end diff --git a/recipes/radosgw.rb b/recipes/radosgw.rb index 0a5bb0d..65b96e3 100644 --- a/recipes/radosgw.rb +++ b/recipes/radosgw.rb @@ -46,24 +46,47 @@ include_recipe "ceph::radosgw_#{node['ceph']['radosgw']['webserver_companion']}" end -ceph_client 'radosgw' do - caps('mon' => 'allow rw', 'osd' => 'allow rwx') - owner 'root' - group node['apache']['group'] - mode 0640 -end +if node['ceph']['radosgw']['default'] + ceph_client 'radosgw' do + caps('mon' => 'allow rw', 'osd' => 'allow rwx') + owner 'root' + group node['apache']['group'] + mode 0640 + end -directory "/var/lib/ceph/radosgw/ceph-radosgw.#{node['hostname']}" do - recursive true - only_if { node['platform'] == 'ubuntu' } + directory "/var/lib/ceph/radosgw/ceph-radosgw.#{node['hostname']}" do + recursive true + only_if { node['platform'] == 'ubuntu' } + end + + # needed by https://github.com/ceph/ceph/blob/master/src/upstart/radosgw-all-starter.conf + file "/var/lib/ceph/radosgw/ceph-radosgw.#{node['hostname']}/done" do + action :create + only_if { node['platform'] == 'ubuntu' } + end end -# needed by https://github.com/ceph/ceph/blob/master/src/upstart/radosgw-all-starter.conf -file "/var/lib/ceph/radosgw/ceph-radosgw.#{node['hostname']}/done" do - action :create - only_if { node['platform'] == 'ubuntu' } +::Chef::Log.info("Found radosgw instances #{node['ceph']['radosgw']['instances'].keys}") +node['ceph']['radosgw']['instances'].sort.each do |keyname, instance| + ceph_client "radosgw.#{keyname}" do + keyname keyname + filename instance['keyring'] + caps('mon' => 'allow rw', 'osd' => 'allow rwx') + end + + keyid = keyname.sub(/^client\./, '') + + # sets the init script to start this instance + directory "/var/lib/ceph/radosgw/ceph-#{keyid}" do + recursive true + end + + file "/var/lib/ceph/radosgw/ceph-#{keyid}/done" do + action :create + end end +# sysvinit script fails to start instances that don't have rgw pools set up service 'radosgw' do case node['ceph']['radosgw']['init_style'] when 'upstart' diff --git a/recipes/radosgw_apache2.rb b/recipes/radosgw_apache2.rb index 8b266c0..581cca0 100644 --- a/recipes/radosgw_apache2.rb +++ b/recipes/radosgw_apache2.rb @@ -65,7 +65,24 @@ template 'rgw.conf.erb' server_name node['ceph']['radosgw']['api_fqdn'] admin_email node['ceph']['radosgw']['admin_email'] - ceph_rgw_addr node['ceph']['radosgw']['rgw_addr'] + rgw_addr node['ceph']['radosgw']['rgw_addr'] + api_aliases node['ceph']['radosgw']['api_aliases'] + socket "/var/run/ceph-radosgw/radosgw.#{ node['hostname']}" + only_if { node['ceph']['radosgw']['default'] } +end + +node['ceph']['radosgw']['instances'].sort.each do |keyname, instance| + Chef::Log.info("Applying radosgw apache vhost #{keyname}") + zone = instance['rgw zone'] + web_app "rgw-#{zone}" do + template 'rgw.conf.erb' + server_name instance['rgw dns name'] + admin_email node['ceph']['radosgw']['admin_email'] + rgw_addr node['ceph']['radosgw']['rgw_addr'] + api_aliases instance['api_aliases'] + socket instance['rgw socket path'] + zone instance['rgw zone'] + end end directory node['ceph']['radosgw']['path'] do diff --git a/resources/radosgw.rb b/resources/radosgw.rb new file mode 100644 index 0000000..6919c9a --- /dev/null +++ b/resources/radosgw.rb @@ -0,0 +1,22 @@ +actions :add +default_action :add + +attribute :name, :kind_of => String, :name_attribute => true + +# pool settings +attribute :region, :kind_of => String +attribute :region_root_pool, :kind_of => String, :default => nil # defaults to .#{region}.rgw.root +attribute :zone, :kind_of => String +attribute :zone_root_pool, :kind_of => String, :default => nil # defaults to .#{zone}.rgw.root + +# other radosgw settings +attribute :keyname, :kind_of => String, :default => nil # defaults to client.radosgw.#{zone}.#{hostname}, used to generate key +attribute :dns_name, :kind_of => String +attribute :dns_aliases, :kind_of => Array, :default => nil # turns into ServerAlias lines +attribute :socket_path, :kind_of => String, :default => nil # defaults to /var/run/ceph-radosgw/ceph-radosgw.#{zone}.#{hostname} +attribute :print_continue, :kind_of => [TrueClass, FalseClass], :default => nil + +def initialize(*args) + super + @action = :add +end diff --git a/templates/default/ceph.conf.erb b/templates/default/ceph.conf.erb index d17fe32..32aa4fb 100644 --- a/templates/default/ceph.conf.erb +++ b/templates/default/ceph.conf.erb @@ -46,7 +46,16 @@ <%= k %> = <%= v %> <% end %> <% end -%> -<% end -%> + +<% node['ceph']['radosgw']['instances'].sort.each do |name, instance| -%> +[<%= name %>] + host = <%= node['hostname'] %> + log file = /var/log/ceph/radosgw-<%= instance['rgw zone'] %>.log +<%- instance.sort.reject{ |k,v| (! k.is_a? String) || (! v.is_a? String) }.each do |k,v| %> + <%= k %> = <%= v %> +<%- end %> +<%- end # radosgw.instances.each -%> +<% end # if is_rgw -%> <% node['ceph']['config-sections'].sort.each do |name, sect| %> [<%= name %>] diff --git a/templates/default/rgw.conf.erb b/templates/default/rgw.conf.erb index 4a5ebd5..1e58d16 100644 --- a/templates/default/rgw.conf.erb +++ b/templates/default/rgw.conf.erb @@ -1,16 +1,17 @@ +<% fcgi = "/s3gw#{ "-#{@params[:zone]}" if @params[:zone] }.fcgi" -%> <% if node['ceph']['radosgw']['rgw_port'] -%> -FastCgiExternalServer <%= node['ceph']['radosgw']['path'] %>/s3gw.fcgi -host 127.0.0.1:<%= node['ceph']['radosgw']['rgw_port'] %> +FastCgiExternalServer <%= node['ceph']['radosgw']['path'] %><%= fcgi %> -host 127.0.0.1:<%= node['ceph']['radosgw']['rgw_port'] %> <% else -%> -FastCgiExternalServer <%= node['ceph']['radosgw']['path'] %>/s3gw.fcgi -socket /var/run/ceph-radosgw/radosgw.<%= node['hostname'] %> +FastCgiExternalServer <%= node['ceph']['radosgw']['path'] %><%= fcgi %> -socket <%= @params[:socket] %> <% end -%> LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" \"%{Host}i\"" proxy_combined LogFormat "%{X-Forwarded-For}i %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" \"%{Host}i\"" proxy_debug -> +> ServerName <%= @params[:server_name] %> -<% if node['ceph']['radosgw']['api_aliases'] -%> -<% node['ceph']['radosgw']['api_aliases'].each do |api_alias| -%> +<% if @params[:api_aliases] -%> +<% @params[:api_aliases].each do |api_alias| -%> ServerAlias <%= api_alias %> <% end -%> <% end -%> @@ -18,7 +19,7 @@ LogFormat "%{X-Forwarded-For}i %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{Use DocumentRoot <%= node['ceph']['radosgw']['path'] %> RewriteEngine On - RewriteRule ^/([a-zA-Z0-9-_.]*)([/]?.*) /s3gw.fcgi?page=$1¶ms=$2&%{QUERY_STRING} [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L] + RewriteRule ^/([a-zA-Z0-9-_.]*)([/]?.*) <%= fcgi %>?page=$1¶ms=$2&%{QUERY_STRING} [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L] > diff --git a/test/cookbooks/ceph_test/recipes/radosgw_restart.rb b/test/cookbooks/ceph_test/recipes/radosgw_restart.rb new file mode 100644 index 0000000..93a1d29 --- /dev/null +++ b/test/cookbooks/ceph_test/recipes/radosgw_restart.rb @@ -0,0 +1,36 @@ +# +# Author:: Kyle Bader +# Cookbook Name:: ceph_test +# Recipe:: radosgw_restart +# +# Copyright 2011, DreamHost Web Hosting +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# restarts radosgw +# during a normal run, Ceph would have the custom radosgw pools already running +# during a test run, the pool metadata hasn't been fully installed by the time +# the alt radosgw starts up, so it fails to start. +# we can't restart in bats, because bats patiently waits for the +# spawned children processes, including radosgw, to finish +bash 'restart radosgw services' do + code <<-EOF + if [ -e /etc/init/shutdown.conf ]; then + service radosgw-all stop + service radosgw-all-starter start + else + service radosgw stop + service radosgw start + fi + EOF +end diff --git a/test/cookbooks/ceph_test/recipes/radosgw_zone.rb b/test/cookbooks/ceph_test/recipes/radosgw_zone.rb new file mode 100644 index 0000000..28ab53b --- /dev/null +++ b/test/cookbooks/ceph_test/recipes/radosgw_zone.rb @@ -0,0 +1,32 @@ +# +# Author:: Kyle Bader +# Cookbook Name:: ceph_test +# Recipe:: radosgw_zone +# +# Copyright 2011, DreamHost Web Hosting +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ceph_radosgw 'us-test' do + region 'default' + zone 'default' + region_root_pool '.rgw.root' + zone_root_pool '.rgw.root' + dns_name 'ceph.test' + dns_aliases ['*.ceph.test'] + keyname 'client.radosgw.us-test' + socket_path '/var/run/ceph-radosgw/radosgw.us-test' + print_continue true +end.run_action(:add) + +package 'curl' # used for the tests diff --git a/test/integration/aio/bats/ceph-running.bats b/test/integration/aio/bats/ceph-running.bats index 578d7e4..fb908cb 100644 --- a/test/integration/aio/bats/ceph-running.bats +++ b/test/integration/aio/bats/ceph-running.bats @@ -17,3 +17,6 @@ @test "apache is running and listening" { netstat -ln | grep -E '^\S+\s+\S+\s+\S+\s+\S+:80\s+' } +@test "apache responds" { + wget http://localhost --header=host:localhost -O- | grep ListAllMyBucketsResult > /dev/null +} diff --git a/test/integration/aio/bats/radosgw_zone.bats b/test/integration/aio/bats/radosgw_zone.bats new file mode 100644 index 0000000..75b6089 --- /dev/null +++ b/test/integration/aio/bats/radosgw_zone.bats @@ -0,0 +1,42 @@ +@test "radosgw for test zone is configured" { + # check that the settings look right + settings=`cat /etc/ceph/ceph.conf | awk '/^\[/ { $1 ~ /\[client.radosgw.us-test\]/ ? mode=1 : mode=0 } /^ +/ {if (mode==1) print $0;}'` + test -n "$settings" + echo "$settings" | grep 'rgw region = default' > /dev/null + echo "$settings" | grep 'rgw region root pool = .rgw.root' > /dev/null + echo "$settings" | grep 'rgw zone = default' > /dev/null + echo "$settings" | grep 'rgw zone root pool = .rgw.root' > /dev/null + echo "$settings" | grep 'keyring = /etc/ceph/ceph.client.radosgw.us-test.keyring' > /dev/null + echo "$settings" | grep 'rgw socket path = /var/run/ceph-radosgw/radosgw.us-test' > /dev/null + echo "$settings" | grep 'rgw dns name = ceph.test' > /dev/null + echo "$settings" | grep -v 'api_aliases' > /dev/null + echo "$settings" | grep -v 'rgw dns aliases' > /dev/null + echo "$settings" | grep 'rgw print continue = true' > /dev/null +} + +@test "radosgw for test zone has keys" { + test -e /etc/ceph/ceph.client.radosgw.us-test.keyring +} +@test "radosgw for test zone should start" { + test -e /var/lib/ceph/radosgw/ceph-radosgw.us-test +} +@test "radosgw for test zone is running" { + ps -ef | grep 'radosg[w].*radosgw.us-test' > /dev/null + test -e /var/run/ceph-radosgw/radosgw.us-test +} + +@test "radosgw for test zone has apache config" { + filename=/etc/apache2/sites-enabled/rgw-default.conf + test -e $filename + grep 'FastCgiExternalServer /var/www/s3gw-default.fcgi -socket /var/run/ceph-radosgw/radosgw.us-test' $filename > /dev/null + grep 'ServerName ceph.test' $filename > /dev/null + grep 'ServerAlias \*.ceph.test' $filename > /dev/null + grep 'RewriteRule.*s3gw-default.fcgi' $filename > /dev/null +} + +@test "radosgw for test zone responds via apache" { + wget http://localhost --header=host:ceph.test -O- | grep ListAllMyBucketsResult > /dev/null +} +@test "radosgw for test zone subdomain responds via apache" { + curl http://localhost --header host:sub.ceph.test | grep NoSuchBucket > /dev/null +}