Feature - zoned radosgw

.kitchen.yml

@@ -63,5 +63,7 @@ suites:
         - { device: "/dev/sdc" }
         - { device: "/dev/sdd" }
   run_list:
+    - recipe[ceph_test::radosgw_zone]
     - recipe[ceph::all_in_one]
     - recipe[ceph_test::cephfs]
+    - recipe[ceph_test::radosgw_restart]

README.md

@@ -121,6 +121,27 @@ The ceph\_cephfs LWRP provides an easy way to mount CephFS. It will automaticall
 - :use\_fuse - whether to use ceph-fuse or the kernel client to mount the filesystem. ceph-fuse is updated more often, but the kernel client allows for subdirectory mounting. Defaults to true
 - :cephfs\_subdir - which CephFS subdirectory to mount. Defaults to '/'. An exception will be thrown if this option is set to anything other than '/' if use\_fuse is also true
 
+### ceph\_radosgw
+
+The ceph\_radosgw LWRP provides an easy way to setup customized radosgw servers. The LWRP itself just adds a node attribute, and then the radosgw recipe does all the setup. Due to the way Chef processes the runlist, this resource has to be explicitly converged before compiling the radosgw recipe. Any recipe that calls this LWRP has to say `.run_action(:add)` after the do..end block. Alternatively, the node attributes can be set with a wrapper cookbook.
+
+#### Actions
+
+- :add - Adds the given information as a custom radosgw configuration.
+
+#### Parameters
+
+- :name - name attribute, not used
+- :region - what region should be set in the ceph.conf for this radosgw
+- :region\_root\_pool - what rados pool should be used to load region information
+- :zone - what zone should be set in the ceph.conf for this radosgw
+- :zone\_root\_pool - what rados pool should be used to load zone information
+- :keyname - what name to use when generating the cephx key for this radosgw. Defaults to client.radosgw.#{zone}.#{hostname}
+- :dns\_name - what dns name to use in ceph.conf and Apache
+- :dns\_aliases - an optional array of other hostnames to handle requests for in Apache
+- :socket\_path - the path to the socket that radosgw will use for fastcgi communications. Defaults to /var/run/ceph/ceph-radosgw.#{zone}.#{hostname}
+- :print\_continue - whether to set the `print continue` option in ceph.conf. Defaults to nil, which doesn't set the option, true or false sets the option to the given setting.
+
 ## LICENSE AND AUTHORS
 
 * Author: Kyle Bader <[email protected]>

attributes/radosgw.rb

@@ -27,8 +27,10 @@
 default['ceph']['radosgw']['use_apache_fork'] = true
 default['ceph']['radosgw']['init_style'] = node['ceph']['init_style']
 
-default['ceph']['radosgw']['path'] = '/var/www'
+default['ceph']['radosgw']['default'] = true   # whether to deploy the default radosgw
+default['ceph']['radosgw']['instances'] = {}   # map from zone name to settings
 
+default['ceph']['radosgw']['path'] = '/var/www'
 if node['platform_family'] == 'suse'
   default['ceph']['radosgw']['path'] = '/srv/www/ceph-radosgw'
 end

providers/radosgw.rb

@@ -0,0 +1,46 @@
+#
+# Author:: Kyle Bader <[email protected]>
+# Cookbook Name:: ceph
+# Recipe:: radosgw
+#
+# Copyright 2011, DreamHost Web Hosting
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use_inline_resources
+
+def whyrun_supported?
+  true
+end
+
+action :add do
+  node.default['ceph']['is_radosgw'] = true
+  region = @new_resource.region
+  zone = @new_resource.zone
+  ::Chef::Log.info("Saving information for radosgw #{zone}")
+
+  instance = {}
+  instance['rgw region'] = @new_resource.region
+  instance['rgw region root pool'] = @new_resource.region_root_pool || ".#{region}.rgw.root"
+  instance['rgw zone'] = @new_resource.zone
+  instance['rgw zone root pool'] = @new_resource.zone_root_pool || ".#{@new_resource.zone}.rgw.root"
+  keyname = @new_resource.keyname || "client.radosgw.#{zone}.#{node['hostname']}"
+  instance['keyring'] = "/etc/ceph/ceph.#{keyname}.keyring"
+  instance['rgw socket path'] = @new_resource.socket_path || "/var/run/ceph-radosgw/radosgw.#{zone}.#{node['hostname']}"
+  instance['rgw dns name'] = @new_resource.dns_name
+  instance['api_aliases'] = @new_resource.dns_aliases if @new_resource.dns_aliases
+  instance['rgw print continue'] = @new_resource.print_continue.to_s unless @new_resource.print_continue.nil?
+
+  node.default['ceph']['radosgw']['instances'][keyname] = instance
+
+end

recipes/radosgw.rb

@@ -46,24 +46,47 @@
   include_recipe "ceph::radosgw_#{node['ceph']['radosgw']['webserver_companion']}"
 end
 
-ceph_client 'radosgw' do
-  caps('mon' => 'allow rw', 'osd' => 'allow rwx')
-  owner 'root'
-  group node['apache']['group']
-  mode 0640
-end
+if node['ceph']['radosgw']['default']
+  ceph_client 'radosgw' do
+    caps('mon' => 'allow rw', 'osd' => 'allow rwx')
+    owner 'root'
+    group node['apache']['group']
+    mode 0640
+  end
 
-directory "/var/lib/ceph/radosgw/ceph-radosgw.#{node['hostname']}" do
-  recursive true
-  only_if { node['platform'] == 'ubuntu' }
+  directory "/var/lib/ceph/radosgw/ceph-radosgw.#{node['hostname']}" do
+    recursive true
+    only_if { node['platform'] == 'ubuntu' }
+  end
+
+  # needed by https://github.com/ceph/ceph/blob/master/src/upstart/radosgw-all-starter.conf
+  file "/var/lib/ceph/radosgw/ceph-radosgw.#{node['hostname']}/done" do
+    action :create
+    only_if { node['platform'] == 'ubuntu' }
+  end
 end
 
-# needed by https://github.com/ceph/ceph/blob/master/src/upstart/radosgw-all-starter.conf
-file "/var/lib/ceph/radosgw/ceph-radosgw.#{node['hostname']}/done" do
-  action :create
-  only_if { node['platform'] == 'ubuntu' }
+::Chef::Log.info("Found radosgw instances #{node['ceph']['radosgw']['instances'].keys}")
+node['ceph']['radosgw']['instances'].sort.each do |keyname, instance|
+  ceph_client "radosgw.#{keyname}" do
+    keyname keyname
+    filename instance['keyring']
+    caps('mon' => 'allow rw', 'osd' => 'allow rwx')
+  end
+
+  keyid = keyname.sub(/^client\./, '')
+
+  # sets the init script to start this instance
+  directory "/var/lib/ceph/radosgw/ceph-#{keyid}" do
+    recursive true
+  end
+
+  file "/var/lib/ceph/radosgw/ceph-#{keyid}/done" do
+    action :create
+  end
 end
 
+# sysvinit script fails to start instances that don't have rgw pools set up
 service 'radosgw' do
   case node['ceph']['radosgw']['init_style']
   when 'upstart'

recipes/radosgw_apache2.rb

@@ -65,7 +65,24 @@
   template 'rgw.conf.erb'
   server_name node['ceph']['radosgw']['api_fqdn']
   admin_email node['ceph']['radosgw']['admin_email']
-  ceph_rgw_addr node['ceph']['radosgw']['rgw_addr']
+  rgw_addr node['ceph']['radosgw']['rgw_addr']
+  api_aliases node['ceph']['radosgw']['api_aliases']
+  socket "/var/run/ceph-radosgw/radosgw.#{ node['hostname']}"
+  only_if { node['ceph']['radosgw']['default'] }
+end
+
+node['ceph']['radosgw']['instances'].sort.each do |keyname, instance|
+  Chef::Log.info("Applying radosgw apache vhost #{keyname}")
+  zone = instance['rgw zone']
+  web_app "rgw-#{zone}" do
+    template 'rgw.conf.erb'
+    server_name instance['rgw dns name']
+    admin_email node['ceph']['radosgw']['admin_email']
+    rgw_addr node['ceph']['radosgw']['rgw_addr']
+    api_aliases instance['api_aliases']
+    socket instance['rgw socket path']
+    zone instance['rgw zone']
+  end
 end
 
 directory node['ceph']['radosgw']['path'] do

resources/radosgw.rb

@@ -0,0 +1,22 @@
+actions :add
+default_action :add
+
+attribute :name, :kind_of => String, :name_attribute => true
+
+# pool settings
+attribute :region, :kind_of => String
+attribute :region_root_pool, :kind_of => String, :default => nil  # defaults to .#{region}.rgw.root
+attribute :zone, :kind_of => String
+attribute :zone_root_pool, :kind_of => String, :default => nil  # defaults to .#{zone}.rgw.root
+
+# other radosgw settings
+attribute :keyname, :kind_of => String, :default => nil  # defaults to client.radosgw.#{zone}.#{hostname}, used to generate key
+attribute :dns_name, :kind_of => String
+attribute :dns_aliases, :kind_of => Array, :default => nil  # turns into ServerAlias lines
+attribute :socket_path, :kind_of => String, :default => nil # defaults to /var/run/ceph-radosgw/ceph-radosgw.#{zone}.#{hostname}
+attribute :print_continue, :kind_of => [TrueClass, FalseClass], :default => nil
+
+def initialize(*args)
+  super
+  @action = :add
+end

templates/default/ceph.conf.erb

@@ -46,7 +46,16 @@
   <%= k %> = <%= v %>
   <% end %>
 <% end -%>
-<% end -%>
+
+<% node['ceph']['radosgw']['instances'].sort.each do |name, instance| -%>
+[<%= name %>]
+  host = <%= node['hostname'] %>
+  log file = /var/log/ceph/radosgw-<%= instance['rgw zone'] %>.log
+<%- instance.sort.reject{ |k,v| (! k.is_a? String) || (! v.is_a? String) }.each do |k,v| %>
+  <%= k %> = <%= v %>
+<%- end %>
+<%- end # radosgw.instances.each -%>
+<% end # if is_rgw -%>
 
 <% node['ceph']['config-sections'].sort.each do |name, sect| %>
 [<%= name %>]

templates/default/rgw.conf.erb

@@ -1,24 +1,25 @@
+<% fcgi = "/s3gw#{ "-#{@params[:zone]}" if @params[:zone] }.fcgi" -%>
 <% if node['ceph']['radosgw']['rgw_port'] -%>
-FastCgiExternalServer <%= node['ceph']['radosgw']['path'] %>/s3gw.fcgi -host 127.0.0.1:<%= node['ceph']['radosgw']['rgw_port'] %>
+FastCgiExternalServer <%= node['ceph']['radosgw']['path'] %><%= fcgi %> -host 127.0.0.1:<%= node['ceph']['radosgw']['rgw_port'] %>
 <% else -%>
-FastCgiExternalServer <%= node['ceph']['radosgw']['path'] %>/s3gw.fcgi -socket /var/run/ceph-radosgw/radosgw.<%= node['hostname'] %>
+FastCgiExternalServer <%= node['ceph']['radosgw']['path'] %><%= fcgi %> -socket <%= @params[:socket] %>
 <% end -%>
 
 LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" \"%{Host}i\"" proxy_combined
 LogFormat "%{X-Forwarded-For}i %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" \"%{Host}i\"" proxy_debug
 
-<VirtualHost <%= node['ceph']['radosgw']['rgw_addr'] %>>
+<VirtualHost <%= @params[:rgw_addr] %>>
   ServerName <%= @params[:server_name] %>
-<% if node['ceph']['radosgw']['api_aliases'] -%>
-<%   node['ceph']['radosgw']['api_aliases'].each do |api_alias| -%>
+<% if @params[:api_aliases] -%>
+<%   @params[:api_aliases].each do |api_alias| -%>
   ServerAlias <%= api_alias %>
 <%   end -%>
 <% end -%>
   ServerAdmin <%= node["ceph"]["radosgw"]["admin_email"] %>
   DocumentRoot <%= node['ceph']['radosgw']['path'] %>
 
   RewriteEngine On
-  RewriteRule ^/([a-zA-Z0-9-_.]*)([/]?.*) /s3gw.fcgi?page=$1&params=$2&%{QUERY_STRING} [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L]
+  RewriteRule ^/([a-zA-Z0-9-_.]*)([/]?.*) <%= fcgi %>?page=$1&params=$2&%{QUERY_STRING} [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L]
 
   <IfModule mod_fastcgi.c>
     <Directory <%= node['ceph']['radosgw']['path'] %>>

test/cookbooks/ceph_test/recipes/radosgw_restart.rb

@@ -0,0 +1,36 @@
+#
+# Author:: Kyle Bader <[email protected]>
+# Cookbook Name:: ceph_test
+# Recipe:: radosgw_restart
+#
+# Copyright 2011, DreamHost Web Hosting
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# restarts radosgw
+# during a normal run, Ceph would have the custom radosgw pools already running
+# during a test run, the pool metadata hasn't been fully installed by the time
+# the alt radosgw starts up, so it fails to start.
+# we can't restart in bats, because bats patiently waits for the
+# spawned children processes, including radosgw, to finish
+bash 'restart radosgw services' do
+  code <<-EOF
+  if [ -e /etc/init/shutdown.conf ]; then
+    service radosgw-all stop
+    service radosgw-all-starter start
+  else
+    service radosgw stop
+    service radosgw start
+  fi
+  EOF
+end

test/cookbooks/ceph_test/recipes/radosgw_zone.rb

@@ -0,0 +1,32 @@
+#
+# Author:: Kyle Bader <[email protected]>
+# Cookbook Name:: ceph_test
+# Recipe:: radosgw_zone
+#
+# Copyright 2011, DreamHost Web Hosting
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ceph_radosgw 'us-test' do
+  region 'default'
+  zone 'default'
+  region_root_pool '.rgw.root'
+  zone_root_pool '.rgw.root'
+  dns_name 'ceph.test'
+  dns_aliases ['*.ceph.test']
+  keyname 'client.radosgw.us-test'
+  socket_path '/var/run/ceph-radosgw/radosgw.us-test'
+  print_continue true
+end.run_action(:add)
+
+package 'curl'   # used for the tests

test/integration/aio/bats/ceph-running.bats

@@ -17,3 +17,6 @@
 @test "apache is running and listening" {
   netstat -ln | grep -E '^\S+\s+\S+\s+\S+\s+\S+:80\s+'
 }
+@test "apache responds" {
+  wget http://localhost --header=host:localhost -O- | grep ListAllMyBucketsResult > /dev/null
+}

test/integration/aio/bats/radosgw_zone.bats

@@ -0,0 +1,42 @@
+@test "radosgw for test zone is configured" {
+  # check that the settings look right
+  settings=`cat /etc/ceph/ceph.conf | awk '/^\[/ { $1 ~ /\[client.radosgw.us-test\]/ ? mode=1 : mode=0 } /^ +/ {if (mode==1) print $0;}'`
+  test -n "$settings"
+  echo "$settings" | grep 'rgw region = default' > /dev/null
+  echo "$settings" | grep 'rgw region root pool = .rgw.root' > /dev/null
+  echo "$settings" | grep 'rgw zone = default' > /dev/null
+  echo "$settings" | grep 'rgw zone root pool = .rgw.root' > /dev/null
+  echo "$settings" | grep 'keyring = /etc/ceph/ceph.client.radosgw.us-test.keyring' > /dev/null
+  echo "$settings" | grep 'rgw socket path = /var/run/ceph-radosgw/radosgw.us-test' > /dev/null
+  echo "$settings" | grep 'rgw dns name = ceph.test' > /dev/null
+  echo "$settings" | grep -v 'api_aliases' > /dev/null
+  echo "$settings" | grep -v 'rgw dns aliases' > /dev/null
+  echo "$settings" | grep 'rgw print continue = true' > /dev/null
+}
+
+@test "radosgw for test zone has keys" {
+  test -e /etc/ceph/ceph.client.radosgw.us-test.keyring
+}
+@test "radosgw for test zone should start" {
+  test -e /var/lib/ceph/radosgw/ceph-radosgw.us-test
+}
+@test "radosgw for test zone is running" {
+  ps -ef | grep 'radosg[w].*radosgw.us-test' > /dev/null
+  test -e /var/run/ceph-radosgw/radosgw.us-test
+}
+
+@test "radosgw for test zone has apache config" {
+  filename=/etc/apache2/sites-enabled/rgw-default.conf
+  test -e $filename
+  grep 'FastCgiExternalServer /var/www/s3gw-default.fcgi -socket /var/run/ceph-radosgw/radosgw.us-test' $filename > /dev/null
+  grep 'ServerName ceph.test' $filename > /dev/null
+  grep 'ServerAlias \*.ceph.test' $filename > /dev/null
+  grep 'RewriteRule.*s3gw-default.fcgi' $filename > /dev/null
+}
+
+@test "radosgw for test zone responds via apache" {
+  wget http://localhost --header=host:ceph.test -O- | grep ListAllMyBucketsResult > /dev/null
+}
+@test "radosgw for test zone subdomain responds via apache" {
+  curl http://localhost --header host:sub.ceph.test | grep NoSuchBucket > /dev/null
+}