Merge branch 'v1.22.0'

ansible/roles/ckan/tasks/main.yml

@@ -187,7 +187,7 @@
   - modules
 
 - name: Ensure SqlAlchemy version
-  pip: name="sqlalchemy" virtualenv="{{ virtual_environment }}" version="0.7.10" extra_args='--exists-action=w --upgrade --force-reinstall'
+  pip: name="sqlalchemy" virtualenv="{{ virtual_environment }}" version="0.9.6" extra_args='--exists-action=w --upgrade --force-reinstall'
   tags:
   - ckan
   - spatial

ansible/roles/ckan/tasks/translations.yml

@@ -91,7 +91,7 @@
   - translation_table
 
 - name: Create translation page
-  script: translation_table.py "{{ cache_path }}" "{{ www_root }}/resources/translation-table.html" "{{ virtual_environment }}/src/ckan/ckan/i18n/!LANGUAGE!/LC_MESSAGES/ckan.po" "{{ cache_path }}/translations/ckan-2.2.!LANGUAGE!.po" "fi,sv"
+  script: translation_table.py "{{ cache_path }}" "{{ www_root }}/resources/translation-table.html" "{{ virtual_environment }}/src/ckan/ckan/i18n/!LANGUAGE!/LC_MESSAGES/ckan.po" "{{ cache_path }}/translations/ckan-{{ ckan_version }}.!LANGUAGE!.po" "fi,sv"
   tags:
   - translations
   - ckan

ansible/roles/ckan/vars/main.yml

@@ -56,20 +56,16 @@ ckan_source_path: "{{ virtual_environment }}/src/ckan"
 spatial_source_path: "{{ virtual_environment }}/src/ckanext-spatial"
 
 ckan_patches:
-  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/6f77aea762cdbc4d66ad70af0b2f1524a3a64218.patch", file: "fix_multilingual.patch" } # ytp specific
-  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/0791caace461315d2cbffbcbf551ef878f23f76c.patch", file: "fix_hardcoded_languages.patch" } # ytp specific
-  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/f772576d3c480f654c39ecaeb72feb423450f200.patch", file: "fix_invalid_date_indexing.patch" } # ytp specific
-  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/5e686b87289fbf9824dfa42468186b8cd7ba48a2.patch", file: "fix_multilingual_search.patch" } # ytp specific
-  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/7641df3acf1370642ea193c93be2a64fd4da261b.patch", file: "fix_collection_type_translation.patch" } # ytp specific
-  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/77f966c54b578beb812e486372e768110a48cc98.patch", file: "fix_autocomplete_input_delay.patch" } # ytp specific
-  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/c989a39ec9bd1657b6dfc5a9147ffb31f524ae33.patch", file: "upgrade_image-upload.js"} # in ckan 2.3
-  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/b441619d98adafbfe5a1da33cec66614566949e5.patch", file: "fix-rdf.patch"} # ytp specific
-  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/7ad11f098bb03cefdf3639b6d96e3127a6d1c833.patch", file: "fix-group_show.patch"} # in ckan 2.3
-  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/a98cb6c3ee3f82a8dc6b0fc40db1686b3c72e50a.patch", file: "refactor-multilingual-warnings.patch" } # in ckan 2.3
-  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/8175a352f960d83027e2476489e5034cf79e43be.patch", file: "usability-fix-for-image-upload.patch" } # ytp specific
-  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/81cf433a8f968f0f993da7c668699b485213f466.patch", file: "fix-double-encoding-in-autocomplete.patch"} # ytp specific
-  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/ae856c3dd9aa896163f80d330a3274636a5c4e74.patch", file: "fix-template-error-in-resources.patch"} # ytp specific
-  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/b39b1e8ac3eaaa013db4f2c4af804eadaad83e3f.patch", file: "add-event-callbacks-to-autocomplete.patch"} # ytp specific
-  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/8f85ef4ad90df1c9df6dd553b21140d0b48916eb.patch", file: "enable-url-quoting"} # pullreq in ckan master
+  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/e7936fe8707a847387661248b5b0d9d3ddc526ae.patch", file: "fix_hardcoded_languages.patch" } # ytp specific
+  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/4d596cca05a55fdcba9fbc2af1ecaf46b843b737.patch", file: "fix_invalid_date_indexing.patch" } # ytp specific
+  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/7367b56bc023da854aa31e05ff63ef8307c06f41.patch", file: "fix_multilingual_search.patch" } # ytp specific
+  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/9e80b58f24dea9d93ec84aac258f5436dfcfdd9c.patch", file: "fix_collection_type_translation.patch" } # ytp specific
+  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/5b7a63d86291987646ff5275c068f41eab21b128.patch", file: "fix_autocomplete_input_delay.patch" } # ytp specific
+  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/cbbca2047c68c377ef9408fc4df157b1acb1b02c.patch", file: "usability-fix-for-image-upload.patch" } # ytp specific
+  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/8f7ec3f9fb6a35bb4a7e52784920ed99b7c7e245.patch", file: "fix-double-encoding-in-autocomplete.patch"} # ytp specific
+  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/cde3c21a8b8612b88ef61b47488465b2926a6840.patch", file: "fix-template-error-in-resources.patch"} # ytp specific
+  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/eb59ffde3a702077ffbc8de8e3ff425b29974d6c.patch", file: "add-event-callbacks-to-autocomplete.patch"} # ytp specific
+  - { patch: "https://github.com/yhteentoimivuuspalvelut/ckan/commit/9497130896c969982a3b8b5e9685295428fd7d2b.patch", file: "enable-url-quoting"} # pullreq in ckan master
+
 service_database_enabled: false
 auto_author_enabled: true

ansible/roles/jetty/templates/schema.xml.j2

@@ -16,9 +16,7 @@
  limitations under the License.
 -->
 
-<!-- Based on https://raw2.github.com/ckan/ckan/release-v{{ ckan_version }}/ckan/config/solr/schema.xml -->
-
-<schema name="ckan" version="2.0">
+<schema name="ckan" version="2.3">
 
 <types>
     <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
@@ -95,12 +93,15 @@
     <field name="ratings_average" type="float" indexed="true" stored="false" />
     <field name="tags" type="string" indexed="true" stored="true" multiValued="true"/>
     <field name="groups" type="string" indexed="true" stored="true" multiValued="true"/>
+    <field name="organization" type="string" indexed="true" stored="true" multiValued="false"/>
 
     <field name="capacity" type="string" indexed="true" stored="true" multiValued="false"/>
 
+    <field name="res_name" type="textgen" indexed="true" stored="true" multiValued="true" />
     <field name="res_description" type="textgen" indexed="true" stored="true" multiValued="true"/>
     <field name="res_format" type="string" indexed="true" stored="true" multiValued="true"/>
     <field name="res_url" type="string" indexed="true" stored="true" multiValued="true"/>
+    <field name="res_type" type="string" indexed="true" stored="true" multiValued="true"/>
 
     <!-- catchall field, containing all other searchable text fields (implemented
          via copyField further on in this schema  -->
@@ -137,6 +138,7 @@
     <dynamicField name="*_date" type="date" indexed="true" stored="true" multiValued="false"/>
 
     <dynamicField name="extras_*" type="text" indexed="true" stored="true" multiValued="false"/>
+    <dynamicField name="res_extras_*" type="text" indexed="true" stored="true" multiValued="true"/>
     <dynamicField name="vocab_*" type="string" indexed="true" stored="true" multiValued="true"/>
     <dynamicField name="*" type="string" indexed="true"  stored="false"/>
 </fields>
@@ -150,6 +152,7 @@
 <copyField source="download_url" dest="urls"/>
 <copyField source="res_url" dest="urls"/>
 <copyField source="extras_*" dest="text"/>
+<copyField source="res_extras_*" dest="text"/>
 <copyField source="vocab_*" dest="text"/>
 <copyField source="urls" dest="text"/>
 <copyField source="name" dest="text"/>
@@ -159,6 +162,8 @@
 <copyField source="notes" dest="text"/>
 <copyField source="tags" dest="text"/>
 <copyField source="groups" dest="text"/>
+<copyField source="organization" dest="text"/>
+<copyField source="res_name" dest="text"/>
 <copyField source="res_description" dest="text"/>
 <copyField source="maintainer" dest="text"/>
 <copyField source="author" dest="text"/>

ansible/vars/common.yml

@@ -84,7 +84,7 @@ virtual_environment: /usr/lib/ckan/default
 
 harvest_sources: false
 
-ckan_version: "2.2"
+ckan_version: "2.3"
 
 email_domain: "localhost"
 email_domain_access: "localhost"

modules/ckanext-ytp-main/ckanext/ytp/common/public/javascript/ytp_form.js

@@ -295,7 +295,7 @@ $(document).ready(function(){
             organizations.filter(function(index, element){
                 return element.textContent.toLowerCase().indexOf(search_str) >= 0
             }).each(function(index, element){
-                var text = $(element).find('a').get(0).innerText
+                var text = $(element).find('a').get(0).textContent;
                 var str_index = text.toLowerCase().indexOf(search_str);
                 if ( str_index >= 0){
                     count++;

modules/ckanext-ytp-main/ckanext/ytp/dataset/controller.py

@@ -10,7 +10,6 @@
 from pylons import config
 
 import ckan.lib.navl.dictization_functions as dict_fns
-import ckan.lib.package_saver as package_saver
 from genshi.template import MarkupTemplate
 
 import sqlalchemy
@@ -84,9 +83,7 @@ def edit(self, id, data=None, errors=None, error_summary=None):
         package_type = self._get_package_type(id)
         context = {'model': model, 'session': model.Session,
                    'user': c.user or c.author, 'auth_user_obj': c.userobj,
-                   'save': 'save' in request.params,
-                   'moderated': config.get('moderated'),
-                   'pending': True}
+                   'save': 'save' in request.params}
 
         if context['save'] and not data:
             return self._save_edit(id, context, package_type=package_type)
@@ -115,36 +112,42 @@ def edit(self, id, data=None, errors=None, error_summary=None):
 
         try:
             check_access('package_update', context)
-        except NotAuthorized:
+        except NotAuthorized, e:
             abort(401, _('User %r not authorized to edit %s') % (c.user, id))
         # convert tags if not supplied in data
         if data and not data.get('tag_string'):
             data['tag_string'] = ', '.join(h.dict_list_reduce(
                 c.pkg_dict.get('tags', {}), 'name'))
         errors = errors or {}
-        vars = {'data': data, 'errors': errors,
-                'error_summary': error_summary, 'action': 'edit'}
+        form_snippet = self._package_form(package_type=package_type)
+        form_vars = {'data': data, 'errors': errors,
+                     'error_summary': error_summary, 'action': 'edit',
+                     'dataset_type': package_type,
+                     }
         c.errors_json = h.json.dumps(errors)
 
         self._setup_template_variables(context, {'id': id},
                                        package_type=package_type)
         c.related_count = c.pkg.related_count
 
         # we have already completed stage 1
-        vars['stage'] = ['active']
-        if data.get('state', '').startswith('draft'):
-            vars['stage'] = ['active', 'complete']
-
-        # TODO: This check is to maintain backwards compatibility with the
-        # old way of creating custom forms. This behaviour is now deprecated.
-        if hasattr(self, 'package_form'):
-            c.form = render(self.package_form, extra_vars=vars)
-        else:
-            c.form = render(self._package_form(package_type=package_type),
-                            extra_vars=vars)
-
-        return render(self._edit_template(package_type),
-                      extra_vars={'stage': vars['stage']})
+        form_vars['stage'] = ['active']
+        if data.get('state', '').startswith('draft') and len(data.get('resources')) == 0:
+            form_vars['stage'] = ['active', 'complete']
+
+        edit_template = self._edit_template(package_type)
+        c.form = ckan.lib.render.deprecated_lazy_render(
+            edit_template,
+            form_snippet,
+            lambda: render(form_snippet, extra_vars=form_vars),
+            'use of c.form is deprecated. please see '
+            'ckan/templates/package/edit.html for an example '
+            'of the new way to include the form snippet'
+        )
+        return render(edit_template,
+                      extra_vars={'form_vars': form_vars,
+                                  'form_snippet': form_snippet,
+                                  'dataset_type': package_type})
 
     # original ckan new resource
     def new_resource(self, id, data=None, errors=None, error_summary=None):
@@ -165,7 +168,7 @@ def new_resource(self, id, data=None, errors=None, error_summary=None):
             # see if we have any data that we are trying to save
             data_provided = False
             for key, value in data.iteritems():
-                if ((value or isinstance(value, cgi.FieldStorage)) and key != 'resource_type'):
+                if (value or isinstance(value, cgi.FieldStorage)) and key != 'resource_type':
                     data_provided = True
                     break
 
@@ -234,11 +237,7 @@ def new_resource(self, id, data=None, errors=None, error_summary=None):
                 # add more resources
                 redirect(h.url_for(controller='package',
                                    action='new_resource', id=id))
-        errors = errors or {}
-        error_summary = error_summary or {}
-        vars = {'data': data, 'errors': errors,
-                'error_summary': error_summary, 'action': 'new'}
-        vars['pkg_name'] = id
+
         # get resources for sidebar
         context = {'model': model, 'session': model.Session,
                    'user': c.user or c.author, 'auth_user_obj': c.userobj}
@@ -251,6 +250,15 @@ def new_resource(self, id, data=None, errors=None, error_summary=None):
         except NotAuthorized:
             abort(401, _('Unauthorized to create a resource for this package'))
 
+        package_type = pkg_dict['type'] or 'dataset'
+
+        errors = errors or {}
+        error_summary = error_summary or {}
+        vars = {'data': data, 'errors': errors,
+                'error_summary': error_summary, 'action': 'new',
+                'resource_form_snippet': self._resource_form(package_type),
+                'dataset_type': package_type}
+        vars['pkg_name'] = id
         # required for nav menu
         vars['pkg_dict'] = pkg_dict
         template = 'package/new_resource_not_draft.html'
@@ -262,6 +270,7 @@ def new_resource(self, id, data=None, errors=None, error_summary=None):
     # taken from original ckan resource edit
     def resource_edit(self, id, resource_id, data=None, errors=None,
                       error_summary=None):
+
         if request.method == 'POST' and not data:
             data = data or clean_dict(dict_fns.unflatten(tuplize_dict(parse_params(
                 request.POST))))
@@ -316,30 +325,30 @@ def resource_edit(self, id, resource_id, data=None, errors=None,
         if not data:
             data = resource_dict
 
+        package_type = pkg_dict['type'] or 'dataset'
+
         errors = errors or {}
         error_summary = error_summary or {}
         vars = {'data': data, 'errors': errors,
-                'error_summary': error_summary, 'action': 'new'}
+                'error_summary': error_summary, 'action': 'new',
+                'resource_form_snippet': self._resource_form(package_type),
+                'dataset_type': package_type}
         return render('package/resource_edit.html', extra_vars=vars)
 
     def read(self, id, format='html'):
-        ''' Copied and overriden from CKAN's package controller '''
-
         if not format == 'html':
-            ctype, extension, loader = \
+            ctype, extension = \
                 self._content_type_from_extension(format)
             if not ctype:
                 # An unknown format, we'll carry on in case it is a
                 # revision specifier and re-constitute the original id
                 id = "%s.%s" % (id, format)
-                ctype, format, loader = "text/html; charset=utf-8", "html", \
-                    MarkupTemplate
+                ctype, format = "text/html; charset=utf-8", "html"
         else:
-            ctype, format, loader = self._content_type_from_accept()
+            ctype, format = self._content_type_from_accept()
 
         response.headers['Content-Type'] = ctype
 
-        package_type = self._get_package_type(id.split('@')[0])
         context = {'model': model, 'session': model.Session,
                    'user': c.user or c.author, 'for_view': True,
                    'auth_user_obj': c.userobj}
@@ -381,23 +390,27 @@ def read(self, id, format='html'):
 
         # can the resources be previewed?
         for resource in c.pkg_dict['resources']:
+            # Backwards compatibility with preview interface
             resource['can_be_previewed'] = self._resource_preview(
                 {'resource': resource, 'package': c.pkg_dict})
 
+            resource_views = get_action('resource_view_list')(
+                context, {'id': resource['id']})
+            resource['has_views'] = len(resource_views) > 0
+
+        package_type = c.pkg_dict['type'] or 'dataset'
         self._setup_template_variables(context, {'id': id},
                                        package_type=package_type)
 
-        package_saver.PackageSaver().render_package(c.pkg_dict, context)
-
         template = self._read_template(package_type)
         template = template[:template.index('.') + 1] + format
 
         try:
-            return render(template, loader_class=loader)
+            return render(template,
+                          extra_vars={'dataset_type': package_type})
         except ckan.lib.render.TemplateNotFound:
             msg = _("Viewing {package_type} datasets in {format} format is "
-                    "not supported (template file {file} not found).".format(package_type=package_type,
-                                                                             format=format, file=template))
+                    "not supported (template file {file} not found).".format(package_type=package_type, format=format, file=template))
             abort(404, msg)
 
         assert False, "We should never get here"