Merge branch 'hotfeature-rest_jwt'

application/config/version.php

@@ -29,14 +29,14 @@
  *
  * @var string
  */
-$config['version'] = '4.6.0';
+$config['version'] = '4.7.0';
 
 /**
  * Version release date.
  *
  * @var string
  */
-$config['release_date'] = '2020-10-01';
+$config['release_date'] = '2020-10-09';
 
 /**
  * Link to the code repository downloads page.

modules/rest_api/config/rest.example.php

@@ -79,9 +79,15 @@
     'resource_options' => [
       // Grants full access to all reports. Client configs can override this.
       'reports' => [],
-      // Grant access to elasticsearch. Provide empty array to enable all
-      // end-points. Configure the clients which can access each index in
-      // the clients config entry.
+      // Grant access to elasticsearch via the listed endpoints. Either a
+      // simple array of endpoint names, or a associative array keyed by name
+      // containing config in the values. Set config option limit_to_website
+      // to TRUE to limit to data accessible to this website. Set
+      // limit_to_own_data to TRUE to restrict to the user's own data. Each
+      // endpoint needs to be added to the 'elasticsearch' configuration entry
+      // to define how it maps to Elasticsearch. If using directClient
+      // authentication, also configure the clients which can access each index
+      // in the clients config entry.
       'elasticsearch' => ['es'],
     ],
   ],
@@ -97,6 +103,11 @@
     'resource_options' => [
       // Grants full access to all reports. Client configs can override this.
       'reports' => ['featured' => TRUE, 'limit_to_own_data' => TRUE],
+      // Grant access to Elasticsearch but in this case, apply website and user ID filters.
+      // Limit to own data can be overridden by adding claim http://indicia.org.uk/allow_full_dataset=true.
+      // Best practice is to set both of these to TRUE, then in the Indicia settings enable
+      // the option to allow users to access all data if appropriate for the website.
+      'elasticsearch' => ['es' => ['limit_to_website' => TRUE, 'limit_to_own_data' => TRUE]],
     ],
   ],
 ];

modules/rest_api/config/rest.jwt-only.php

@@ -0,0 +1,65 @@
+<?php
+
+/**
+ * Indicia, the OPAL Online Recording Toolkit.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/gpl.html.
+ *
+ * @author Indicia Team
+ * @license http://www.gnu.org/licenses/gpl.html GPL
+ * @link https://github.com/indicia-team/warehouse/
+ */
+
+/**
+ * Authentication methods allowed.
+ */
+$config['authentication_methods'] = [
+  'jwtUser' => [
+    // TRUE to allow CORS from any domain, or provide an array of domain regexes.
+    'allow_cors' => TRUE,
+    'resource_options' => [
+      // Grants full access to all reports. Client configs can override this.
+      'reports' => ['featured' => TRUE, 'limit_to_own_data' => TRUE],
+      // Grant access to Elasticsearch but in this case, apply website and user ID filters.
+      'elasticsearch' => ['es' => ['limit_to_website' => TRUE, 'limit_to_own_data' => TRUE]],
+    ],
+  ],
+];
+
+/**
+ * Should authorisation tokens be allowed in the query parameters rather than the
+ * authorisation header? Recommended for development servers only.
+ */
+$config['allow_auth_tokens_in_url'] = FALSE;
+
+/**
+ * If this warehouse is configured to work with an Elasticsearch instance then
+ * the REST API can act as a proxy to avoid having to expose all the public
+ * APIs. The proxy can point to index aliases to limit the search filter.
+ */
+$config['elasticsearch'] = [
+  'es' => [
+    'open' => FALSE,
+    'index' => 'my-index',
+    'url' => 'http://my.elastic.url:9200',
+    'allowed' => [
+      'get' => [
+        '/^_search/' => 'GET requests to the search API (/_search?...)',
+        '/^_mapping/' => 'GET requests to the mappings API (/_mapping?...)',
+      ],
+      'post' => [
+        '/^_search/' => 'POST requests to the search API (/_search?...)',
+        '/^doc\/.*\/_update/' => 'POSTed document updates',
+      ],
+    ],
+  ],
+];

modules/rest_api/controllers/services/rest.php

@@ -162,13 +162,31 @@ class Rest_Controller extends Controller {
    */
   private $authenticated = FALSE;
 
+  /**
+   * Name of the authentication method.
+   *
+   * @var string
+   */
+  private $authMethod;
+
   /**
    * Config settings relating to the selected auth method.
    *
    * @var array
    */
   private $authConfig;
 
+  /**
+   * Allow override of default ES filters on record created_by_id
+   *
+   * When using user based auth (jwtUser or oAuth2User), configuration can
+   * included limit_to_own_data which applies an automatic user filter unless
+   * the request access token includes a claim that alldata access is allowed.
+   *
+   * @var bool
+   */
+  private $allowAllData = FALSE;
+
   /**
    * Config settings relating to the authenticated client if any.
    *
@@ -745,11 +763,20 @@ public function __construct() {
    * Outputs help text to describe the available API resources.
    */
   public function index() {
-    // A temporary array to simulate the arguments, which we can use to check
-    // for versioning.
-    $arguments = [$this->uri->last_segment()];
-    $this->checkVersion($arguments);
-    RestObjects::$apiResponse->index($this->resourceConfig);
+    try {
+      if (!file_exists(MODPATH . 'rest_api/config/rest.php')) {
+        RestObjects::$apiResponse->fail('Internal Server Error', 500,
+          'Missing config file. See https://indicia-docs.readthedocs.io/en/latest/administrating/warehouse/modules/rest-api.html for more info.');
+      }
+      // A temporary array to simulate the arguments, which we can use to check
+      // for versioning.
+      $arguments = [$this->uri->last_segment()];
+      $this->checkVersion($arguments);
+      RestObjects::$apiResponse->index($this->resourceConfig);
+    }
+    catch (RestApiAbort $e) {
+      // No action if a proper abort.
+    }
   }
 
   /**
@@ -826,6 +853,9 @@ public function token() {
    * @throws exception
    */
   public function __call($name, $arguments) {
+    if (!file_exists(MODPATH . 'rest_api/config/rest.php')) {
+      $this->fail('Internal Server Error', 500, 'Missing config file.');
+    }
     $tm = microtime(TRUE);
     try {
       // Undo router's conversion of hyphens and underscores.
@@ -1081,6 +1111,73 @@ private function getColumnsTemplate(&$postObj) {
     }
   }
 
+   /**
+   * A cached lookup of the websites that are available for a sharing mode.
+   *
+   * @param integer $websiteId
+   *   ID of the website that is receiving the shared data.
+   *
+   * @return array
+   *   List of website IDs that will share their data.
+   */
+  private function getSharedWebsiteList($websiteId, $sharing = 'reporting') {
+    $tag = "website-shares-$websiteId";
+    $cacheId = "$tag-$sharing";
+    $cache = Cache::instance();
+    if ($cached = $cache->get($cacheId)) {
+      return explode(',', $cached);
+    }
+    $qry = $this->db->select('to_website_id')
+      ->from('index_websites_website_agreements')
+      ->where([
+        "receive_for_$sharing" => 't',
+        'from_website_id' => $websiteId
+      ])
+      ->get()->result();
+    $ids = array();
+    foreach ($qry as $row) {
+      $ids[] = $row->to_website_id;
+    }
+    // Tag all cache entries for this website so they can be cleared together
+    // when changes are saved. Also note the cached entry is an imploded string
+    // so we benefit from sharing cache hits with the reporting engine.
+    $cache->set($cacheId, implode(',', $ids), $tag);
+    return $ids;
+  }
+
+  /**
+   * Adds permissions filters to ES search, based on website ID and user ID.
+   *
+   * If the authentication method configuration (e.g. jwtUser) includes the
+   * option limit_to_website in the settings for the Elasticsearch endpoint,
+   * then automatically adds a terms filter on metadata.website.id. Also,
+   * if the settings include limit_to_own_data for the endpoint, then adds a
+   * terms filter on metadata.created_by_id. This can be overridden by
+   * including the claim http://indicia.org.uk/alldata in the JWT access token.
+   */
+  private function applyEsPermissionsQuery(&$postObj) {
+    $filters = [];
+    if (!empty($this->esConfig['limit_to_own_data']) && !$this->allowAllData && RestObjects::$clientUserId) {
+      $filters[] = ['term' => ['metadata.created_by_id' => RestObjects::$clientUserId]];
+    }
+    if (!empty($this->esConfig['limit_to_website']) && RestObjects::$clientWebsiteId) {
+      // @todo Support for other sharing modes in JWT claims.
+      $filters[] = ['terms' => ['metadata.website.id' => $this->getSharedWebsiteList(RestObjects::$clientWebsiteId)]];
+    }
+    if (count($filters) > 0) {
+      if (!isset($postObj->query)) {
+        $postObj->query = new stdClass();
+      }
+      if (!isset($postObj->query->bool)) {
+        $postObj->query->bool = new stdClass();
+      }
+      if (!isset($postObj->query->bool->must)) {
+        $postObj->query->bool->must = [];
+      }
+      $postObj->query->bool->must = array_merge($postObj->query->bool->must, $filters);
+    }
+  }
+
   /**
    * Calculate the data to post to an Elasticsearch search.
    *
@@ -1095,7 +1192,7 @@ private function getColumnsTemplate(&$postObj) {
    * @return string
    *   Data to post.
    */
-  private function getEsPostData($postObj, $format, $file) {
+  private function getEsPostData($postObj, $format, $file, $isSearch) {
     if ($this->pagingMode === 'scroll' && $this->pagingModeState === 'nextPage') {
       // A subsequent hit on a scrolled request.
       $postObj = [
@@ -1111,6 +1208,9 @@ private function getEsPostData($postObj, $format, $file) {
     elseif ($this->pagingMode === 'composite' && isset($file['after_key'])) {
       $postObj->aggs->_rows->composite->after = $file['after_key'];
     }
+    if ($isSearch) {
+      $this->applyEsPermissionsQuery($postObj);
+    }
     if ($format === 'csv') {
       $csvTemplate = $this->getEsCsvTemplate();
       $fields = [];
@@ -1361,7 +1461,7 @@ private function proxyToEs($url) {
     else {
       echo $this->getEsOutputHeader($format);
     }
-    $postData = $this->getEsPostData($postObj, $format, $file);
+    $postData = $this->getEsPostData($postObj, $format, $file, preg_match('/\/_search/', $url));
     $actualUrl = $this->getEsActualUrl($url);
     $session = curl_init($actualUrl);
     if (!empty($postData) && $postData !== '[]') {
@@ -3220,14 +3320,27 @@ private function authenticate() {
         // Try this authentication method.
         call_user_func(array($this, "authenticateUsing$method"));
         if ($this->authenticated) {
+          $this->authMethod = $method;
           // Double checking required for Elasticsearch proxy.
           if ($this->elasticProxy) {
-            if (empty($cfg['resource_options']['elasticsearch']) || !in_array($this->elasticProxy, $cfg['resource_options']['elasticsearch'])) {
+            if (empty($cfg['resource_options']['elasticsearch'])) {
               kohana::log('debug', "Elasticsearch request to $this->elasticProxy not enabled for $method");
               RestObjects::$apiResponse->fail('Unauthorized', 401, 'Unable to authorise');
             }
-            if (!empty($this->clientConfig) && empty($this->clientConfig['elasticsearch']) ||
-                !in_array($this->elasticProxy, $this->clientConfig['elasticsearch'])) {
+            if (in_array($this->elasticProxy, $cfg['resource_options']['elasticsearch'])) {
+              // Simple array of ES endpoints with no config.
+              $this->esConfig = [];
+            }
+            elseif (array_key_exists($this->elasticProxy, $cfg['resource_options']['elasticsearch'])) {
+              // Endpoints are keys with array values holding config.
+              $this->esConfig = $cfg['resource_options']['elasticsearch'][$this->elasticProxy];
+            }
+            else {
+              kohana::log('debug', "Elasticsearch request to $this->elasticProxy not enabled for $method");
+              RestObjects::$apiResponse->fail('Unauthorized', 401, 'Unable to authorise');
+            }
+            if (!empty($this->clientConfig) && (empty($this->clientConfig['elasticsearch']) ||
+                !in_array($this->elasticProxy, $this->clientConfig['elasticsearch']))) {
               kohana::log('debug', "Elasticsearch request to $this->elasticProxy not enabled for client");
               RestObjects::$apiResponse->fail('Unauthorized', 401, 'Unable to authorise');
             }
@@ -3372,6 +3485,10 @@ private function authenticateUsingJwtUser() {
       if (empty($payloadValues['iss']) || empty($payloadValues['http://indicia.org.uk/user:id'])) {
         RestObjects::$apiResponse->fail('Bad request', 400);
       }
+      // Check for claim that stops ES filtering to just user's own records.
+      if (!empty($payloadValues['http://indicia.org.uk/alldata'])) {
+        $this->allowAllData = TRUE;
+      }
       $website = $this->getWebsiteByUrl($payloadValues['iss']);
       if (!$website || empty($website->public_key)) {
         kohana::log('debug', 'Website has no public key');

modules/rest_api/i18n/en_GB/rest_api.php

@@ -38,7 +38,8 @@
       '<li>Generate a public/private key pair and store the private key in the Warehouse website settings.</li>' .
       '<li>Provide a JWT token signed with the public key which provides the following claims:<ul>' .
       '  <li>iss - the website URL</li>' .
-      '  <li>http://indicia.org.uk/user:id</li> set to the warehouse ID of the user issuing the request.</li>' .
+      '  <li>http://indicia.org.uk/user:id - set to the warehouse ID of the user issuing the request.</li>' .
+      '  <li>http://indicia.org.uk/alldata - set to true if claiming that the user is allowed to access all the website records, not just their own.</li>' .
       '</ul></ul>';
 $lang['jwtUserHelpHeader'] = 'Set the authorisation header to "Bearer <JWT token>"';
 $lang['genericHelpHeader'] = 'Specify an authorisation header with a list of token name/value pairs, using colons as a ' .

modules/rest_api/libraries/RestApiResponse.php

@@ -237,19 +237,16 @@ private function indexHtml($resourceConfig) {
     if ($es) {
       echo '<h3>Elasticsearch end-points</h3>';
       foreach ($es as $endpoint => $esConfig) {
-        // Also allow if authentication provided.
-        if ($esConfig['open'] === TRUE) {
-          echo '<h4>' . url::base() . "index.php/services/rest/$endpoint</h4>";
-          echo '<table class="table table-bordered table-responsive"><caption>Allowed methods</caption>';
-          echo '<thead><tr><th>HTTP method</th><th>Expression</th><th>Description</th></tr></thead>';
-          echo '<tbody>';
-          foreach ($esConfig['allowed'] as $method => $patterns) {
-            foreach ($patterns as $expr => $desc) {
-              echo "<tr><td>$method</td><td>$expr</td><td>$desc</desc></tr>";
-            }
+        echo '<h4>' . url::base() . "index.php/services/rest/$endpoint</h4>";
+        echo '<table class="table table-bordered table-responsive"><caption>Allowed methods</caption>';
+        echo '<thead><tr><th>HTTP method</th><th>Expression</th><th>Description</th></tr></thead>';
+        echo '<tbody>';
+        foreach ($esConfig['allowed'] as $method => $patterns) {
+          foreach ($patterns as $expr => $desc) {
+            echo "<tr><td>$method</td><td>$expr</td><td>$desc</desc></tr>";
           }
-          echo '</tbody></table>';
         }
+        echo '</tbody></table>';
       }
     }
     echo str_replace('{{ base }}', url::base(), $this->htmlFooter);
@@ -581,7 +578,7 @@ private function outputArrayAsHtml($array, $options = array()) {
           $this->outputArrayAsHtml($value, $options);
         } else {
           // a simple value to output. If it contains an internal link then process it to hide user/secret data.
-          if (preg_match('/http(s)?:\/\//', $value)) {
+          if (preg_match('/^http(s)?:\/\//', $value)) {
             $parts = explode('?', $value);
             $displayUrl = $parts[0];
             if (count($parts)>1) {