From fd256e6ca3797f69d7980308a23663325114e415 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Fri, 5 Feb 2016 11:12:32 -0800 Subject: [PATCH 01/44] 7.x-1.3 --- CHANGELOG.txt | 18 ++++++++++++++++++ README.md | 1 + .../capx_auto_nodetitle.info | 2 +- .../capx_issue_collector.info | 2 +- stanford_capx.info | 2 +- 5 files changed, 22 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 21f40fdc..fa784526 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,24 @@ Stanford CAPx x.x-x.x, xxxx-xx-xx --------------------------------- +Stanford CAPx 7.x-1.3, 2016-02-05 +--------------------------------- + +- Bugfix: Removed hook_install views save and install time check for private files dir. +- Bugfix: Reduced duplicate warning messages. +- Bugfix: Fix UI for field collection mapping where it was only showing one. +- Added variable cache clear. +- Added block module as a dependency. +- Removed empty error messages caused by the new autonodetitle module. +- Bugfix: Fixes to taxonomy term saving. +- BASIC-1639: protection against images that were unable to be saved to disk. +- Publish orphans that were automatically unpublished. +- Bugfix: Fixes for orphans that are not on the API any more. +- CAPX-167: Expand and collapse all for the data browser. +- CAPX-113: Floating scrollable mapping sidebar. +- CAPX-113: Fixes for page offset and updates to data browser quick links. +- CAPX-89: Skip API call to count the sunet ids as we already have them. + Stanford CAPx 7.x-1.2, 2015-09-10 --------------------------------- diff --git a/README.md b/README.md index 68894dca..57b72257 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # Stanford CAPx +#### Version 1.3 Stanford CAP Extensible module builds on some great work. This module provides an interface for administrators to pull information directly from the CAP API into Drupal. This allows profile owners to continue to manage their profile information on the CAP web service and have that information automatically reflected into a Drupal website. diff --git a/modules/capx_auto_nodetitle/capx_auto_nodetitle.info b/modules/capx_auto_nodetitle/capx_auto_nodetitle.info index 97581e9a..6256954d 100644 --- a/modules/capx_auto_nodetitle/capx_auto_nodetitle.info +++ b/modules/capx_auto_nodetitle/capx_auto_nodetitle.info @@ -2,7 +2,7 @@ name = CAPX Auto Node Title Support description = Allows support for the auto_nodetitle module core = 7.x package = Stanford -version = 7.x-1.2-dev +version = 7.x-1.3 project = capx_auto_nodetitle dependencies[] = auto_nodetitle dependencies[] = stanford_capx diff --git a/modules/capx_issue_collector/capx_issue_collector.info b/modules/capx_issue_collector/capx_issue_collector.info index 92f390a1..61217b9d 100644 --- a/modules/capx_issue_collector/capx_issue_collector.info +++ b/modules/capx_issue_collector/capx_issue_collector.info @@ -1,6 +1,6 @@ name = Stanford CAPx Issue Collector description = Provides a feedback link to the CAPx Working Group Jira project. core = 7.x -version = 7.x-1.2-dev +version = 7.x-1.3 package = Stanford CAPx ; scripts[] = capx_issue_collector.js diff --git a/stanford_capx.info b/stanford_capx.info index 6968e212..bab90fea 100644 --- a/stanford_capx.info +++ b/stanford_capx.info @@ -3,7 +3,7 @@ description = Provides the ability to import profiles into existing entity types core = 7.x package = Stanford CAPx project = stanford_capx -version = 7.x-1.x +version = 7.x-1.3 dependencies[] = entity dependencies[] = block From 1ebfdf9482fce51c778b62e53232a346511d10de Mon Sep 17 00:00:00 2001 From: Daniel Carvalhinho Date: Sun, 17 Apr 2016 23:17:39 -0300 Subject: [PATCH 02/44] BugFix - Changing approach on Orphans Lookup so it loads the profiles in smaller chunks. --- .../Orphans/Lookups/LookupWorkgroupOrphans.php | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/includes/CAPx/Drupal/Importer/Orphans/Lookups/LookupWorkgroupOrphans.php b/includes/CAPx/Drupal/Importer/Orphans/Lookups/LookupWorkgroupOrphans.php index a00d5d55..822d471f 100644 --- a/includes/CAPx/Drupal/Importer/Orphans/Lookups/LookupWorkgroupOrphans.php +++ b/includes/CAPx/Drupal/Importer/Orphans/Lookups/LookupWorkgroupOrphans.php @@ -42,10 +42,18 @@ public function execute($orphaner) { // workgroup directly. Unfortunately, that means that we need to make // another request to the server for each work group. - $client->setLimit(99999); + // Setting limit of items per call to 100 so we don't overload the service. + $client->setLimit(100); $response = $client->api('profile')->search("privGroups", $groups); $results = $response['values']; + // Pull every existing page from CAPx one by one and merge to the results. + for ($page = 2; $page <= $response['totalPages']; $page++) { + $client->setPage($page); + $response = $client->api('profile')->search("privGroups", $groups); + $results = array_merge($results, $response['values']); + } + drupal_alter('capx_orphan_profile_results', $results); // Loop through the results and unset the profiles from the passed in list. From 0eccaf1a71c4008651a7fd6a91f6e28d0c52e605 Mon Sep 17 00:00:00 2001 From: Daniel Carvalhinho Date: Mon, 18 Apr 2016 15:43:16 -0300 Subject: [PATCH 03/44] Changing the hardcoded value for a variable_get as per Shea's suggestion --- .../Importer/Orphans/Lookups/LookupWorkgroupOrphans.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/includes/CAPx/Drupal/Importer/Orphans/Lookups/LookupWorkgroupOrphans.php b/includes/CAPx/Drupal/Importer/Orphans/Lookups/LookupWorkgroupOrphans.php index 822d471f..7970c55f 100644 --- a/includes/CAPx/Drupal/Importer/Orphans/Lookups/LookupWorkgroupOrphans.php +++ b/includes/CAPx/Drupal/Importer/Orphans/Lookups/LookupWorkgroupOrphans.php @@ -42,8 +42,10 @@ public function execute($orphaner) { // workgroup directly. Unfortunately, that means that we need to make // another request to the server for each work group. - // Setting limit of items per call to 100 so we don't overload the service. - $client->setLimit(100); + // Setting limit of items per call to use the batch limit variable + // so we don't overload the service. + $limit = variable_get('stanford_capx_batch_limit', 100); + $client->setLimit($limit); $response = $client->api('profile')->search("privGroups", $groups); $results = $response['values']; From 0a8175a1c9fe5957c1337a7ea59a33af280b6f39 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Fri, 22 Apr 2016 12:26:40 -0700 Subject: [PATCH 04/44] Sunet list adoption program space gap adjustments --- .../Comparisons/CompareOrgCodesSunet.php | 1 + .../Comparisons/CompareWorkgroupsSunet.php | 1 + .../Orphans/EntityImporterOrphans.php | 19 ++++++++++--------- .../Lookups/LookupMissingFromSunetList.php | 4 +++- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/includes/CAPx/Drupal/Importer/Orphans/Comparisons/CompareOrgCodesSunet.php b/includes/CAPx/Drupal/Importer/Orphans/Comparisons/CompareOrgCodesSunet.php index bde4ae20..11ce76db 100644 --- a/includes/CAPx/Drupal/Importer/Orphans/Comparisons/CompareOrgCodesSunet.php +++ b/includes/CAPx/Drupal/Importer/Orphans/Comparisons/CompareOrgCodesSunet.php @@ -34,6 +34,7 @@ public function execute($orphaner) { } $sunetOptions = explode(",", $options['sunet_id']); + $sunetOptions = array_map('trim', $sunetOptions); foreach ($orphans['orgCodes'] as $index => $profileId) { if (in_array($sunetIds[$profileId], $sunetOptions)) { diff --git a/includes/CAPx/Drupal/Importer/Orphans/Comparisons/CompareWorkgroupsSunet.php b/includes/CAPx/Drupal/Importer/Orphans/Comparisons/CompareWorkgroupsSunet.php index c5a1e4ad..53e7fede 100644 --- a/includes/CAPx/Drupal/Importer/Orphans/Comparisons/CompareWorkgroupsSunet.php +++ b/includes/CAPx/Drupal/Importer/Orphans/Comparisons/CompareWorkgroupsSunet.php @@ -34,6 +34,7 @@ public function execute($orphaner) { } $sunetOptions = explode(",", $options['sunet_id']); + $sunetOptions = array_map('trim', $sunetOptions); foreach ($orphans['privGroups'] as $index => $profileId) { if (in_array($sunetIds[$profileId], $sunetOptions)) { diff --git a/includes/CAPx/Drupal/Importer/Orphans/EntityImporterOrphans.php b/includes/CAPx/Drupal/Importer/Orphans/EntityImporterOrphans.php index 7beea960..acad4340 100644 --- a/includes/CAPx/Drupal/Importer/Orphans/EntityImporterOrphans.php +++ b/includes/CAPx/Drupal/Importer/Orphans/EntityImporterOrphans.php @@ -86,9 +86,7 @@ public function execute() { // If the action is set to do nothing to orphaned profiles, do nothing. $options = $this->getImporterOptions(); - if ($options['orphan_action'] == 'nothing') { - return; - } + $action = $options['orphan_action']; $importer = $this->getImporter(); $options = $this->getImporterOptions(); @@ -144,7 +142,11 @@ public function execute() { $orphaned = array_merge($orphaned, $orphans["missing"]); } - $this->processOrphans($orphaned, $importer); + if ($action !== "nothing") { + $this->processOrphans($orphaned, $importer); + } + + // Always want to do this in case the action has changed. $this->processAdopted($profiles, $orphaned); } @@ -393,6 +395,7 @@ public function processAdopted($profiles, $orphans) { foreach ($profiles as $id) { $profile = CAPx::getEntityByProfileId($entityType, $bundleType, $id); + $profileWrapped = entity_metadata_wrapper($entityType, $profile); if (CAPx::profileIsOrphan($profile) && !in_array($id, $orphans)) { // Profile is not an orphan. Lets enable it again. @@ -417,14 +420,12 @@ public function processAdopted($profiles, $orphans) { // If the action was to unpublish let's re-publish the node. if ($options['orphan_action'] == "unpublish") { - // Wrap it up. - $profile = entity_metadata_wrapper($entityType, $profile); - $profile->status->set(1); - $profile->save(); + $profileWrapped->status->set(1); + $profileWrapped->save(); } // Log the adoption. - watchdog('EntityImporterOrphans', "%title was adopted and is no longer an orphan in the importer %importername.", array("%title" => $profile->label(), "%importername" => $importerName), WATCHDOG_NOTICE, ''); + watchdog('EntityImporterOrphans', "%title was adopted and is no longer an orphan in the importer %importername.", array("%title" => $profileWrapped->label(), "%importername" => $importerName), WATCHDOG_NOTICE, ''); } } diff --git a/includes/CAPx/Drupal/Importer/Orphans/Lookups/LookupMissingFromSunetList.php b/includes/CAPx/Drupal/Importer/Orphans/Lookups/LookupMissingFromSunetList.php index 3086303f..09f0a48b 100644 --- a/includes/CAPx/Drupal/Importer/Orphans/Lookups/LookupMissingFromSunetList.php +++ b/includes/CAPx/Drupal/Importer/Orphans/Lookups/LookupMissingFromSunetList.php @@ -32,6 +32,7 @@ public function execute($orphaner) { $profiles = $orphaner->getProfiles(); $orphans = $orphaner->getOrphans(); $ids = explode(",", $options['sunet_id']); + $ids = array_map('trim', $ids); $found = array(); $sunetIds = array(); @@ -42,7 +43,8 @@ public function execute($orphaner) { // Loop through the profiles looking for ids. If we cannot find one in the // list of ids then we have an orphan. foreach ($profiles as $index => $profileId) { - if (!in_array($sunetIds[$profileId], $ids)) { + $sunet = $sunetIds[$profileId]; + if (!in_array($sunet, $ids)) { $found[$sunetIds[$profileId]] = $profileId; } } From 12b52e88437ed2242549a62892d0480ca2938a2a Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Fri, 22 Apr 2016 12:26:40 -0700 Subject: [PATCH 05/44] Sunet list adoption program space gap adjustments From fcbdadbb58803f96d8e1332499fe261e90681224 Mon Sep 17 00:00:00 2001 From: Zach Chandler Date: Thu, 28 Apr 2016 19:37:43 -0700 Subject: [PATCH 06/44] Added JSONPath for Lab Website Line 221, tested on Sites. Thanks Shea for helping me figure out the syntax! --- stanford_capx.blocks.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/stanford_capx.blocks.inc b/stanford_capx.blocks.inc index ad3ebd1b..baa87a24 100644 --- a/stanford_capx.blocks.inc +++ b/stanford_capx.blocks.inc @@ -218,6 +218,7 @@ function stanford_capx_data_browser_launch_block() { $rows[] = array('Graduation year', '$.education.*.yearIssued'); $rows[] = array('Personal info links title', '$.internetLinks.*.label.text'); $rows[] = array('Personal info links url', '$.internetLinks.*.url'); + $rows[] = array('Lab website url', '$.internetLinks.[?(@.type="Lab Site")].url'); $rows[] = array('Fields of interest', '$.professionalInterests.text'); $rows[] = array('Mailing address', '$.primaryContact.address'); $rows[] = array('Mailing address city', '$.primaryContact.city'); From 5522ce2465c1dc823a5cbd47ee62c8a726a608f2 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Mon, 16 May 2016 17:33:24 -0700 Subject: [PATCH 07/44] HSDO-325: Started references abstraction --- includes/CAPx/Drupal/Mapper/EntityMapper.php | 39 ++- .../CAPx/Drupal/Mapper/MapperAbstract.php | 2 - .../EntityReferenceFieldProcessor.php | 34 +++ .../FieldProcessors/FieldProcessor.php | 5 +- stanford_capx.forms.inc | 275 ++++++++++++------ 5 files changed, 260 insertions(+), 95 deletions(-) create mode 100644 includes/CAPx/Drupal/Processors/FieldProcessors/EntityReferenceFieldProcessor.php diff --git a/includes/CAPx/Drupal/Mapper/EntityMapper.php b/includes/CAPx/Drupal/Mapper/EntityMapper.php index 6cdc9307..8f8303f0 100644 --- a/includes/CAPx/Drupal/Mapper/EntityMapper.php +++ b/includes/CAPx/Drupal/Mapper/EntityMapper.php @@ -36,6 +36,7 @@ public function execute($entity, $data) { // is logged to watchdog so handle any errors in each of these by throwing // only one error up a level. + // FIELDS. try { $this->mapFields($data); } @@ -43,6 +44,7 @@ public function execute($entity, $data) { $this->setError($e); } + // PROPERTIES. try { $this->mapProperties($data); } @@ -50,6 +52,7 @@ public function execute($entity, $data) { $this->setError($e); } + // FIELD COLLECTIONS. try { // Field Collections are special. Special means more code. They get their // own mapProcess even though they are sort of a field. @@ -59,6 +62,14 @@ public function execute($entity, $data) { $this->setError($e); } + // REFERENCES. + try { + $this->mapReferences(); + } + catch (\Exception $e) { + $this->setError($e); + } + // Even if there were errors we should have an entity by this point. return $entity; } @@ -181,7 +192,7 @@ public function mapProperties($data) { try { $info = $this->getRemoteDataByJsonPath($data, $remoteDataPath); } - catch(\Exception $e) { + catch (\Exception $e) { $error = TRUE; $message = 'There was an exception when trying to get data by @path. Exception message is: @message.'; $message_vars = array( @@ -228,7 +239,7 @@ public function mapFieldCollections($data) { try { $collections = $this->getConfigSetting('fieldCollections'); } - catch(\Exception $e) { + catch (\Exception $e) { // No collections. Just return. return; } @@ -262,6 +273,30 @@ public function mapFieldCollections($data) { $this->setEntity($entity); } + /** + * Process entity reference fields uniquely. + * + * Reference fields are a special field and need to be handled differently. + * Allow for the ability to look for an item to attach to from the API + * that has already been imported. + * + */ + public function mapReferences() { + + try { + $references = $this->getConfigSetting('references'); + } + catch (\Exception $e) { + // No references. Just return. + return; + } + + foreach ($references as $fieldName => $importer_machine_name) { + + } + + } + /** * Checks that fields used in this mapper still in place. * diff --git a/includes/CAPx/Drupal/Mapper/MapperAbstract.php b/includes/CAPx/Drupal/Mapper/MapperAbstract.php index 413fb829..97402f11 100644 --- a/includes/CAPx/Drupal/Mapper/MapperAbstract.php +++ b/includes/CAPx/Drupal/Mapper/MapperAbstract.php @@ -97,8 +97,6 @@ public function getConfig() { public function addConfig($settings) { $config = $this->getConfig(); -// $mapper = $this->getMapper(); - $settings['fieldCollections'] = array(); if (isset($settings['collections'])) { diff --git a/includes/CAPx/Drupal/Processors/FieldProcessors/EntityReferenceFieldProcessor.php b/includes/CAPx/Drupal/Processors/FieldProcessors/EntityReferenceFieldProcessor.php new file mode 100644 index 00000000..1fff429b --- /dev/null +++ b/includes/CAPx/Drupal/Processors/FieldProcessors/EntityReferenceFieldProcessor.php @@ -0,0 +1,34 @@ + + */ + +namespace CAPx\Drupal\Processors\FieldProcessors; + +class EntityReferenceFieldProcessor extends FieldTypeProcessor { + + /** + * Default implementation of put. + * + * @see FieldProcessorAbstract::put() + */ + public function put($data) { + $data = $this->findProfileID($data); + parent::put($data); + } + + /** + * Prepares CAP API data to feet to Drupal field. + * + * @param array $data + * CAP API field data. + * + * @return array + * Prepared data. + */ + public function findProfileID($data) { + return 82; + } + +} diff --git a/includes/CAPx/Drupal/Processors/FieldProcessors/FieldProcessor.php b/includes/CAPx/Drupal/Processors/FieldProcessors/FieldProcessor.php index d3c4a8f7..6186b7df 100644 --- a/includes/CAPx/Drupal/Processors/FieldProcessors/FieldProcessor.php +++ b/includes/CAPx/Drupal/Processors/FieldProcessors/FieldProcessor.php @@ -37,7 +37,6 @@ * number_float number * taxonomy_term_reference options_select * - * * text text_textfield * text_long text_textarea * text_with_summary text_textarea_with_summary @@ -119,6 +118,10 @@ public function field($type) { $processor = new TextAreaFieldProcessor($entity, $fieldName, $type); break; + case "entityreference": + $processor = new EntityReferenceFieldProcessor($entity, $fieldName, $type); + break; + default: $processor = $this; } diff --git a/stanford_capx.forms.inc b/stanford_capx.forms.inc index ea5dde06..df8e606e 100644 --- a/stanford_capx.forms.inc +++ b/stanford_capx.forms.inc @@ -319,6 +319,9 @@ function stanford_capx_mapper_form_get_entity_mapping_form($form, &$form_state, /** * [stanford_capx_mapper_form_get_field_mapping_form description] + * + * @todo: Refactor this stupidly long function. + * * @param [type] $form [description] * @param [type] $form_state [description] * @return [type] [description] @@ -327,6 +330,7 @@ function stanford_capx_mapper_form_get_field_mapping_form($form, &$form_state, $ $entity_types = capx_entity_get_info(); $field_collections = array(); + $references = array(); global $CAP_RESTRICTED_FIELD_PROPERTIES; // Add some styles and javascript @@ -430,6 +434,7 @@ function stanford_capx_mapper_form_get_field_mapping_form($form, &$form_state, $ continue; } + // Bundles. foreach ($entity_info['bundles'] as $bundle_machine_name => $bundle_info) { // Skip bundles that dont match a loaded mapper. @@ -455,6 +460,7 @@ function stanford_capx_mapper_form_get_field_mapping_form($form, &$form_state, $ ), ); + // Loop through all the fields on this entity / bundle. foreach ($fields as $field_machine_name => $field_info) { // Field collections need their own fieldset and fields. @@ -463,6 +469,13 @@ function stanford_capx_mapper_form_get_field_mapping_form($form, &$form_state, $ continue; } + // Entity References are special and need special handling. + if (strpos($field_info['type'], 'list') !== FALSE) { + $references[$entity_machine_name][$bundle_machine_name][$field_machine_name] = $field_info; + continue; + } + + // Get field base and instance information. $field_info_field = field_info_field($field_machine_name); $field_instance_info = field_info_instance($entity_machine_name, $field_machine_name, $bundle_machine_name); @@ -475,43 +488,8 @@ function stanford_capx_mapper_form_get_field_mapping_form($form, &$form_state, $ '#theme_wrappers' => array('stanford_capx_columns_to_the_right'), ); - // Do not give all options for the file field. File field processor - // will take care of most. - if ($field_info_field['type'] == "file" || $field_info_field['type'] == "image") { - $form['field-mapping'][$field_container][$field_machine_name][0] = array( - '#type' => 'textfield', - '#title' => t('File Information'), - '#description' => t('In your JSON Path selector please pass in the whole array of image information. eg: $.profilePhotos.bigger'), - '#default_value' => @$mapper->fields[$field_machine_name][0], - ); - - if ($field_instance_info['required']) { - $form['field-mapping'][$field_container][$field_machine_name]['#attributes']['class'][] = "required"; - } - - continue; - } - - // Relation module doesn't have this column key set. - if (!isset($field_info_field['columns'])) { - continue; - } - - foreach ($field_info_field['columns'] as $column_key => $column_info) { - if (!in_array($column_key, $CAP_RESTRICTED_FIELD_PROPERTIES)) { - - $form['field-mapping'][$field_container][$field_machine_name][$column_key] = array( - '#type' => 'textfield', - '#title' => t("@var", array("@var" => $column_key)), - '#description' => !empty($column_info['description']) ? t("@var", array("@var" => $column_info['description'])) : "", - '#default_value' => isset($mapper->fields[$field_machine_name][$column_key]) ? $mapper->fields[$field_machine_name][$column_key] : "", - ); - - if ($field_instance_info['required']) { - $form['field-mapping'][$field_container][$field_machine_name]['#attributes']['class'][] = "required"; - } - } - } + // Get this forms fields for the entity/bundle fields. + stanford_capx_mapper_form_get_fields_fields($form['field-mapping'][$field_container][$field_machine_name], $form_state, $field_info_field, $field_instance_info, $field_machine_name, $mapper); } @@ -528,7 +506,7 @@ function stanford_capx_mapper_form_get_field_mapping_form($form, &$form_state, $ $instances = field_info_instances('field_collection_item', $field_machine_name); $collection_container = $entity_machine_name . "-" . $bundle_machine_name . "-collections"; - // Fieldset for all fields on this entity -> bundle -> collection field + // Fieldset for all fields on this entity -> bundle -> collection field. $form['field-mapping'][$collection_container][$field_machine_name] = array( '#type' => 'fieldset', '#title' => t('@name Field Collection', array("@name" => $field_machine_name)), @@ -556,34 +534,57 @@ function stanford_capx_mapper_form_get_field_mapping_form($form, &$form_state, $ '#theme_wrappers' => array('stanford_capx_columns_to_the_right'), ); - // Do not give all options for the file field. File field processor - // will take care of most. - if ($fc_field_info['type'] == "file" || $fc_field_info['type'] == "image") { - $form['field-mapping'][$collection_container][$field_machine_name][$fc_field_name][0] = array( - '#type' => 'textfield', - '#title' => t('File Info'), - '#description' => t('In your JSON Path selector please pass in the whole array of image information. eg: $.profilePhotos.bigger'), - '#default_value' => @$mapper->collections[$field_machine_name][$fc_field_name][0], - ); - continue; - } + // Run through the field. + stanford_capx_mapper_form_get_fields_fields($form['field-mapping'][$collection_container][$field_machine_name][$fc_field_name], $form_state, $fc_field_info, $instance_info, $field_machine_name, $mapper); + + } + + } + } + } - foreach ($fc_field_info['columns'] as $column_key => $column_info) { - if (!in_array($column_key, $CAP_RESTRICTED_FIELD_PROPERTIES)) { - $form['field-mapping'][$collection_container][$field_machine_name][$fc_field_name][$column_key] = array( - '#type' => 'textfield', - '#title' => t("@var", array("@var" => $column_key)), - '#description' => t(@$column_info['description']), - '#default_value' => @$mapper->collections[$field_machine_name][$fc_field_name][$column_key], - ); + // ENTITY REFERENCES + // ---------------------------------------------------------------------- - if ($instance_info['required']) { - $form['field-mapping'][$collection_container][$field_machine_name][$fc_field_name][$column_key]["#attributes"]["class"] = "required"; - } + foreach ($references as $entity_machine_name => $bundles) { + foreach ($bundles as $bundle_machine_name => $fields) { - } - } - } + // Create the container. + $reference_container = $entity_machine_name . "-" . $bundle_machine_name . "-references"; + + // Fieldset for all fields on this entity -> bundle -> reference field. + $form['field-mapping'][$reference_container] = array( + '#type' => 'fieldset', + '#title' => t('Entity References', array("@name" => $field_machine_name)), + '#description' => t(''), + '#collapsed' => TRUE, + '#collapsible' => TRUE, + '#states' => array( + 'visible' => array( + 'select[name="entity-type"]' => array('value' => $entity_machine_name), + 'select[name="bundle-' . $entity_machine_name . '"]' => array('value' => $bundle_machine_name), + ), + ), + ); + + // Now loop through the fields. + foreach ($fields as $field_machine_name => $field_info) { + + // Get field base and instance information. + $field_info_field = field_info_field($field_machine_name); + $field_instance_info = field_info_instance($entity_machine_name, $field_machine_name, $bundle_machine_name); + + $form['field-mapping'][$reference_container][$field_machine_name] = array( + '#type' => 'container', + '#title' => t("@var", array("@var" => $field_instance_info['label'])), + '#description' => t("@var", array("@var" => $field_instance_info['description'])), + '#field_info' => $field_info_field, + '#field_instance_info' => $field_instance_info, + '#theme_wrappers' => array('stanford_capx_columns_to_the_right'), + ); + + // Get this forms fields for the entity/bundle fields. + stanford_capx_mapper_form_get_fields_fields($form['field-mapping'][$reference_container][$field_machine_name], $form_state, $field_info_field, $field_instance_info, $field_machine_name, $mapper); } } @@ -592,6 +593,92 @@ function stanford_capx_mapper_form_get_field_mapping_form($form, &$form_state, $ return $form; } +/** + * [stanford_capx_mapper_form_get_fields_fields description] + * @param [type] $form [description] + * @param [type] $form_state [description] + * @param [type] $field_info_field [description] + * @param [type] $field_container [description] + * @param [type] $mapper [description] + * @return [type] [description] + */ +function stanford_capx_mapper_form_get_fields_fields(&$form_tree, &$form_state, $field_info_field, $field_instance_info, $field_machine_name, $mapper) { + + global $CAP_RESTRICTED_FIELD_PROPERTIES; + + // Relation module doesn't have this column key set. + if (!isset($field_info_field['columns'])) { + return; + } + + // FILE FIELD + // --------------------------------------------------------------------------- + // Do not give all options for the file field. File field processor + // will take care of most. + if ($field_info_field['type'] == "file" || $field_info_field['type'] == "image") { + $form_tree[0] = array( + '#type' => 'textfield', + '#title' => t('File Information'), + '#description' => t('In your JSON Path selector please pass in the whole array of image information. eg: $.profilePhotos.bigger'), + '#default_value' => @$mapper->fields[$field_machine_name][0], + ); + + if ($field_instance_info['required']) { + $form_tree['#attributes']['class'][] = "required"; + } + return; + } + + // ENTITY REFERENCE FIELDS + // --------------------------------------------------------------------------- + if ($field_info_field['type'] == "entityreference") { + + $importers = CAPxImporter::getAllEntityImporters(); + $options = array('none' => t("- None -")); + $options += array_map('stanford_capx_key_name_array_map', $importers); + + $form_tree[0] = array( + '#type' => 'select', + '#options' => $options, + '#title' => t("Referencing Importer"), + '#description' => t("Select the importer with the reference you would like to attach this item to."), + '#default_value' => $mapper->references[$field_machine_name][0], + ); + + return; + } + + + // ALL OTHER FIELDS + // --------------------------------------------------------------------------- + + foreach ($field_info_field['columns'] as $column_key => $column_info) { + if (!in_array($column_key, $CAP_RESTRICTED_FIELD_PROPERTIES)) { + $form_tree[$column_key] = array( + '#type' => 'textfield', + '#title' => t("@var", array("@var" => $column_key)), + '#description' => !empty($column_info['description']) ? t("@var", array("@var" => $column_info['description'])) : "", + '#default_value' => isset($mapper->fields[$field_machine_name][$column_key]) ? $mapper->fields[$field_machine_name][$column_key] : "", + ); + if ($field_instance_info['required']) { + $form_tree['#attributes']['class'][] = "required"; + } + } + } + +} + +/** + * [stanford_capx_key_name_array_map description] + * @param [type] $keys [description] + * @param [type] $importers [description] + * @return [type] [description] + */ +function stanford_capx_key_name_array_map($importer) { + return $importer->getImporter()->title; +} + + /** * [stanford_capx_mapper_machine_name_exits description] * @param [type] $name [description] @@ -747,7 +834,7 @@ function stanford_capx_mapper_form_validate($form, $form_state) { } // Set the error messages after everything has been looped through. - stanford_capx_mapper_form_validate_element_set_messages(); + // stanford_capx_mapper_form_validate_element_set_messages(); // Ensure all required fields were posted. Because of the way the form works // we cannot place a required on to each of the form element fields as there @@ -874,32 +961,32 @@ function stanford_capx_mapper_form_validate_element($element, $parents = array() /** * Sets the message for the element validation. */ -function stanford_capx_mapper_form_validate_element_set_messages() { - $form_error = &drupal_static("stanford_capx_mapper_form_validate_element_error", FALSE); - $form_field_warnings = &drupal_static("stanford_capx_mapper_form_validate_element_warnings", FALSE); - - // Set the error message once when the recursive calls have finished. - if ($form_error) { - $field_warnings_list = theme('item_list', array( - 'title' => '', - 'items' => $form_field_warnings, - 'type' => 'ul', - 'attributes' => array() - )); - $browser_link = l(t('Data Browser'), - 'admin/config/capx/data-browser', array( - 'attributes' => array('target' => '_blank') - )); - - drupal_set_message( - t('The following selectors did not match an element in the profile schema but may be valid. If you know the selector is valid please disregard this message. Try using the !databrowser to find a valid selector.!warnings', - array( - '!warnings' => $field_warnings_list, - '!databrowser' => $browser_link, - ) - ), 'warning'); - } -} +// function stanford_capx_mapper_form_validate_element_set_messages() { +// $form_error = &drupal_static("stanford_capx_mapper_form_validate_element_error", FALSE); +// $form_field_warnings = &drupal_static("stanford_capx_mapper_form_validate_element_warnings", FALSE); + +// // Set the error message once when the recursive calls have finished. +// if ($form_error) { +// $field_warnings_list = theme('item_list', array( +// 'title' => '', +// 'items' => $form_field_warnings, +// 'type' => 'ul', +// 'attributes' => array() +// )); +// $browser_link = l(t('Data Browser'), +// 'admin/config/capx/data-browser', array( +// 'attributes' => array('target' => '_blank') +// )); + +// drupal_set_message( +// t('The following selectors did not match an element in the profile schema but may be valid. If you know the selector is valid please disregard this message. Try using the !databrowser to find a valid selector.!warnings', +// array( +// '!warnings' => $field_warnings_list, +// '!databrowser' => $browser_link, +// ) +// ), 'warning'); +// } +// } /** * Submit handler for the mapper form. Create and save the mapper entity. @@ -924,6 +1011,7 @@ function stanford_capx_mapper_form_submit($form, $form_state) { $settings['fields'] = array(); $settings['properties'] = array(); $settings['collections'] = array(); + $settings['references'] = array(); $entity = $values['entity-type']; $bundle = $values['bundle-' . $entity]; @@ -943,6 +1031,13 @@ function stanford_capx_mapper_form_submit($form, $form_state) { $settings['fields'] = $fields; } + // If there are collections... + $references = @$values['field-mapping'][$entity . "-" . $bundle . "-references"]; + // We have values! + if (!empty($references)) { + $settings['references'] = $references; + } + // If there are collections... $collections = @$values['field-mapping'][$entity . "-" . $bundle . "-collections"]; From 17391f1c4fc189c97bdcc50577ec9e5cf365633b Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Tue, 17 May 2016 15:22:05 -0700 Subject: [PATCH 08/44] HSDO-325: 1-1 entity relationship working with importer and mappings --- includes/CAPx/Drupal/Mapper/EntityMapper.php | 31 ++++++++- .../Processors/EntityReferenceProcessor.php | 67 +++++++++++++++++++ .../EntityReferenceFieldProcessor.php | 46 ++++++++----- .../FieldProcessors/LinkFieldProcessor.php | 19 ++++++ 4 files changed, 144 insertions(+), 19 deletions(-) create mode 100644 includes/CAPx/Drupal/Processors/EntityReferenceProcessor.php diff --git a/includes/CAPx/Drupal/Mapper/EntityMapper.php b/includes/CAPx/Drupal/Mapper/EntityMapper.php index 8f8303f0..028af659 100644 --- a/includes/CAPx/Drupal/Mapper/EntityMapper.php +++ b/includes/CAPx/Drupal/Mapper/EntityMapper.php @@ -7,8 +7,11 @@ namespace CAPx\Drupal\Mapper; use CAPx\Drupal\Processors\FieldProcessors\FieldProcessor; +use CAPx\Drupal\Processors\FieldProcessors\EntityReferenceFieldProcessor; use CAPx\Drupal\Processors\PropertyProcessors\PropertyProcessor; use CAPx\Drupal\Processors\FieldCollectionProcessor; +use CAPx\Drupal\Processors\EntityReferenceProcessor; +use CAPx\Drupal\Util\CAPx; use CAPx\Drupal\Util\CAPxMapper; use CAPx\Drupal\Util\CAPxImporter; @@ -291,10 +294,36 @@ public function mapReferences() { return; } - foreach ($references as $fieldName => $importer_machine_name) { + // Nothing to do here. + if (empty($references)) { + return; + } + + $entity = $this->getEntity(); + + // Loop through each reference field and try to match up with another + // entity from another importer. + foreach ($references as $fieldName => $values) { + $target = array_pop($values); + $processor = new EntityReferenceProcessor($entity, $this->getImporter(), $target); + $referenceEntity = $processor->execute(); + + // No possible references available. End here. + if (empty($referenceEntity)) { + continue; + } + + // wrap it up. + $referenceEntity = entity_metadata_wrapper($this->getEntityType(), $referenceEntity); + + // Map the reference. + $entityReferenceFieldProcessor = new EntityReferenceFieldProcessor($entity, $fieldName); + $entityReferenceFieldProcessor->put($referenceEntity); } + // Set the entity to apply any changes this function may have had. + $this->setEntity($entity); } /** diff --git a/includes/CAPx/Drupal/Processors/EntityReferenceProcessor.php b/includes/CAPx/Drupal/Processors/EntityReferenceProcessor.php new file mode 100644 index 00000000..a19e0562 --- /dev/null +++ b/includes/CAPx/Drupal/Processors/EntityReferenceProcessor.php @@ -0,0 +1,67 @@ + + */ + +namespace CAPx\Drupal\Processors; + +use CAPx\Drupal\Mapper\EntityMapper; +use CAPx\Drupal\Util\CAPx; + +class EntityReferenceProcessor { + + protected $entity; + protected $importer; + protected $target; + + /** + * Creates an entityReferenceProcessor to handle entity reference fields. + * @param [type] $entity [description] + * @param [type] $importer [description] + * @param [type] $target [description] + */ + public function __construct($entity, $importer, $target) { + $this->entity = $entity; + $this->importer = $importer; + $this->fieldName = $fieldName; + $this->target = $target; + } + + /** + * Returns a list of possible matches. + * @return [type] [description] + */ + public function execute() { + + // Get the profile ID of this entity as the profile id will be the same + // for other importers and entity/bundle types. + $profile_id = CAPx::getProfileIdByEntity($this->entity); + + // Did not find one. It could be that is hasn't been created yet and may + // take another cycle or two to come up. + if (!$profile_id) { + return array(); + } + + $match = db_select("capx_profiles", 'capx') + ->fields('capx') + ->condition('profile_id', $profile_id) + ->condition('importer', $this->target) + ->orderBy('id', 'DESC') + ->execute() + ->fetchAssoc(); + + if (empty($match)) { + return array(); + } + + // try to load it. + $entity = entity_load_single($match['entity_type'], $match['entity_id']); + + // Return the result. + return empty($entity) ? array() : $entity; + } + + +} diff --git a/includes/CAPx/Drupal/Processors/FieldProcessors/EntityReferenceFieldProcessor.php b/includes/CAPx/Drupal/Processors/FieldProcessors/EntityReferenceFieldProcessor.php index 1fff429b..70a636e0 100644 --- a/includes/CAPx/Drupal/Processors/FieldProcessors/EntityReferenceFieldProcessor.php +++ b/includes/CAPx/Drupal/Processors/FieldProcessors/EntityReferenceFieldProcessor.php @@ -6,29 +6,39 @@ namespace CAPx\Drupal\Processors\FieldProcessors; -class EntityReferenceFieldProcessor extends FieldTypeProcessor { +class EntityReferenceFieldProcessor { - /** - * Default implementation of put. - * - * @see FieldProcessorAbstract::put() - */ - public function put($data) { - $data = $this->findProfileID($data); - parent::put($data); + protected $entity; + protected $fieldName; + + public function __construct($entity, $fieldName) { + $this->entity = $entity; + $this->fieldName = $fieldName; } /** - * Prepares CAP API data to feet to Drupal field. - * - * @param array $data - * CAP API field data. - * - * @return array - * Prepared data. + * Default implementation of put. */ - public function findProfileID($data) { - return 82; + public function put($relatedEntity) { + $id = $relatedEntity->getIdentifier(); + $entity = $this->entity; + $field = $entity->{$this->fieldName}; + + // Metadata wrapper is smarter than plain field info. + switch (get_class($field)) { + // Structure wrapper assumes we providing multiple columns. + case 'EntityStructureWrapper': + case 'EntityListWrapper': + $field->set(array($id)); + break; + + // Value wrapper assumes we providing single value. + case 'EntityDrupalWrapper': + case 'EntityValueWrapper': + $field->set($id); + break; + } } + } diff --git a/includes/CAPx/Drupal/Processors/FieldProcessors/LinkFieldProcessor.php b/includes/CAPx/Drupal/Processors/FieldProcessors/LinkFieldProcessor.php index e0c7d76a..8fb40373 100644 --- a/includes/CAPx/Drupal/Processors/FieldProcessors/LinkFieldProcessor.php +++ b/includes/CAPx/Drupal/Processors/FieldProcessors/LinkFieldProcessor.php @@ -30,12 +30,31 @@ public function put($data) { public function prepareData($data) { foreach ($data as $column => $values) { foreach ($values as $delta => $value) { + if (!is_string($value)) { $data[$column][$delta] = ''; // @todo: Do we really want to log this? // Keep in mind that log will be polluted. $this->logIssue(new \Exception(t('Received not allowed value, expecting string got %type.', array('%type' => gettype($value))))); } + + // If autotruncating is enabled lets do that here. This is to help + // avoid issues when trying to map data to the API. + if (variable_get("stanford_capx_autotruncate_textfields", FALSE)) { + + $maxlength = 2048; // Default for url. + + // Title has custom setting. + if ($column == "title") { + $info = field_info_field($this->fieldName); + $maxlength = $info['settings']['title_maxlength']; + } + + if (strlen($value) > $maxlength) { + $data[$column][$delta] = substr($value, 0, $maxlength); + } + } + } } From f32ebdb643f7ca5825fbf3472307c601aff408ef Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Tue, 17 May 2016 17:32:35 -0700 Subject: [PATCH 09/44] HSDO-325: New mapper settings multiple and subquery --- stanford_capx.forms.inc | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/stanford_capx.forms.inc b/stanford_capx.forms.inc index df8e606e..f5942c0c 100644 --- a/stanford_capx.forms.inc +++ b/stanford_capx.forms.inc @@ -314,6 +314,25 @@ function stanford_capx_mapper_form_get_entity_mapping_form($form, &$form_state, } + $form['entity-mapping']['multiple-entities'] = array( + '#type' => 'checkbox', + '#title' => t("Create multiple entities"), + '#description' => t('By default, a mapper will create one entity per person. If you wanted to import a persons publications you would need multiple entities per person. Check this box for multiple.'), + '#default_value' => isset($mapper->multiple) ? $mapper->multiple : FALSE, + ); + + $form['entity-mapping']['subquery'] = array( + '#type' => 'textfield', + '#title' => t("Multiple entity creation sub query"), + '#description' => t('Which part of the schema will you be using to create multiple entities. Ie: for publications enter: $.publications'), + '#default_value' => isset($mapper->subquery) ? $mapper->subquery : '', + '#states' => array( + 'visible' => array( + ':input[name="multiple-entities"]' => array('checked' => TRUE), + ), + ), + ); + return $form; } @@ -1018,6 +1037,8 @@ function stanford_capx_mapper_form_submit($form, $form_state) { $settings['entity_type'] = $entity; $settings['bundle_type'] = $bundle; + $settings['multiple'] = $values['multiple-entities']; + $settings['subquery'] = $values['subquery']; // If there are properties... $properties = @$values['field-mapping'][$entity . "-properties"]; From 92640b29ca8aa2303372e364a0e240c2ae36a568 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Wed, 18 May 2016 17:01:39 -0700 Subject: [PATCH 10/44] HSDO-325: Revised mapping process to maintain index association and a patched up entity relationship for new entities --- includes/CAPx/Drupal/Mapper/EntityMapper.php | 88 +++++++++++++++++ .../CAPx/Drupal/Mapper/MapperAbstract.php | 94 +++++++++++++++++++ .../Drupal/Processors/EntityProcessor.php | 57 +++++++++++ .../Processors/EntityReferenceProcessor.php | 5 +- .../PropertyProcessorAbstract.php | 9 ++ 5 files changed, 251 insertions(+), 2 deletions(-) diff --git a/includes/CAPx/Drupal/Mapper/EntityMapper.php b/includes/CAPx/Drupal/Mapper/EntityMapper.php index 028af659..75b2a9d2 100644 --- a/includes/CAPx/Drupal/Mapper/EntityMapper.php +++ b/includes/CAPx/Drupal/Mapper/EntityMapper.php @@ -17,6 +17,8 @@ class EntityMapper extends MapperAbstract { + protected $index; + /** * Execute starts the mapping process. * @@ -30,6 +32,11 @@ class EntityMapper extends MapperAbstract { */ public function execute($entity, $data) { + // Always attach the profileId to the entity + $raw = $entity->value(); + $raw->capx['profileId'] = $data['profileId']; + $entity->set($raw); + // Store this for later. $this->setEntity($entity); @@ -142,6 +149,12 @@ public function mapFields($data) { continue; } + // If we are running a multiple entity mapping we only want part of + // the result. + if ($this->isMultiple()) { + $info = $this->getMultipleIndexInfoResultField($info); + } + // Widgets can change the way the data needs to be parsed. Provide // that to the FieldProcessor. $widget = $fieldInfoInstance['widget']['type']; @@ -206,6 +219,12 @@ public function mapProperties($data) { continue; } + // If we are running a multiple entity mapping we only want part of + // the result. + if ($this->isMultiple()) { + $info = $this->getMultipleIndexInfoResultProperty($info); + } + // Let the property processor do its magic. try { $propertyProcessor = new PropertyProcessor($entity, $propertyName); @@ -524,4 +543,73 @@ public function getAffectedImporters() { return $affected; } + /** + * Boolean to whether or not this is a multiple import or not. + * @return boolean [description] + */ + protected function isMultiple() { + try { + return $this->getConfigSetting("multiple"); + } + catch (\Exception $e) { + return FALSE; + } + } + + /** + * [getIndex description] + * @return [type] [description] + */ + protected function getIndex() { + return $this->index; + } + + /** + * [setIndex description] + * @param [type] $i [description] + */ + public function setIndex($i) { + if (is_int($i) && $i >= 0) { + $this->index = $i; + } + else { + throw new \Exception("Could not set index. Invalid option."); + + } + } + + /** + * [getMultipleIndexInfoResult description] + * @param [type] $info [description] + * @return [type] [description] + */ + protected function getMultipleIndexInfoResultField($info) { + $index = $this->getIndex(); + $keys = array_keys($info); + $ret = array(); + + foreach ($keys as $key) { + if (isset($info[$key][$index])) { + $ret[$key] = array($info[$key][$index]); + } + } + + return !empty($ret) ? $ret : $info; + } + + /** + * @param $info + * @return array + */ + protected function getMultipleIndexInfoResultProperty($info) { + $index = $this->getIndex(); + + // If an index value exists: + if (isset($info[$index])) { + return array($info[$index]); + } + + return $info; + } + } diff --git a/includes/CAPx/Drupal/Mapper/MapperAbstract.php b/includes/CAPx/Drupal/Mapper/MapperAbstract.php index 97402f11..b10fa2de 100644 --- a/includes/CAPx/Drupal/Mapper/MapperAbstract.php +++ b/includes/CAPx/Drupal/Mapper/MapperAbstract.php @@ -52,10 +52,93 @@ public function getRemoteDataByJsonPath($data, $path) { throw new \Exception("Path cannot be empty", 1); } + // Wildcard paths need to have index association maintained. JSONPath does + // not provide much help in this and we need to do funny things. + if (strpos($path, "*")) { + $parsed = $this->getRemoteDataWithWildcard($data, $path); + } + // No wildcard, just get stuff. + else { + $parsed = $this->getRemoteDataRegular($data, $path); + } + + return $parsed; + + } + + /** + * Returns an array of data that matched the given path. + * + * @param $data + * @param $path + * @return array + */ + protected function getRemoteDataRegular($data, $path) { $jsonParser = new JsonParser($data); $parsed = $jsonParser->get($path); return $parsed; + } + + /** + * Returns an array of data that matched the given path with empty strings + * for missing leaf items. + * @param $data + * @param $path + */ + protected function getRemoteDataWithWildcard($data, $path) { + + // To maintain index association over wildcards we need to return an empty + // result on a missing leaf. JSONPATH does not provide a nice way of doing + // this out of the box at this point in time so we are going to have to + // break up the requests in to parts. + + $pathParts = explode("*", $path); + $lastPart = array_pop($pathParts); + $countQuery = implode("*", $pathParts); + $countQuery .= "*"; + + $jsonParser = new JsonParser($data); + $results = $jsonParser->get($path); + $wildResults = $jsonParser->get($countQuery); + // If the number of results matches the number of wildcards then we have no + // Missing items. Just return the results. + if (count($results) == count($wildResults)) { + return $results; + } + + // Darn, we have a mismatch in wildcards and leafs. Find out who is missing + // and add an empty value for them in the return array. + $parsed = array(); + $tmpData = $data; + + // Iterate over each of the wild card queries and evaluate the expression. + foreach ($pathParts as $i => $part) { + + // Tack on the wildcard character to the end of the first expression. + if ($i == 0) { + $part .= "*"; + } + else { + $part = "$.*" . $part . "*"; + } + + $jsonParser = new JsonParser($tmpData); + $tmpData = $jsonParser->get($part); + } + + foreach ($tmpData as $values) { + $jsonParser = new JsonParser($values); + $result = $jsonParser->get("$" . $lastPart); + if ($result) { + $parsed[] = array_pop($result); + } + else { + $parsed[] = ""; + } + } + + return $parsed; } /** @@ -109,6 +192,8 @@ public function addConfig($settings) { $mapper->settings['fields'] = $fields; $mapper->settings['properties'] = array(); $mapper->settings['collections'] = array(); + $mapper->settings['multiple'] = FALSE; + $mapper->settings['subquery'] = ''; $settings['fieldCollections'][$fieldName] = new FieldCollectionMapper($mapper); $settings['fieldCollections'][$fieldName]->addConfig($mapper->settings); @@ -220,5 +305,14 @@ public function getErrors() { return FALSE; } + /** + * [getMultipleEntityCount description] + * @return [type] [description] + */ + public function getMultipleEntityCountBySubquery($data) { + $result = $this->getRemoteDataByJsonPath($data, $this->getConfigSetting("subquery")); + return count($result); + } + } diff --git a/includes/CAPx/Drupal/Processors/EntityProcessor.php b/includes/CAPx/Drupal/Processors/EntityProcessor.php index 77fbc651..8509aa2e 100644 --- a/includes/CAPx/Drupal/Processors/EntityProcessor.php +++ b/includes/CAPx/Drupal/Processors/EntityProcessor.php @@ -31,8 +31,65 @@ class EntityProcessor extends ProcessorAbstract { * The new or updated wrapped entity. */ public function execute($force = FALSE) { + + try { + $multi = $this->getMapper()->getConfigSetting('multiple'); + } + catch (\Exception $e) { + $multi = FALSE; + } + + // Sometimes we need to create one, other times we need moar. + if (!empty($multi) && $multi == 1) { + $entity = $this->executeMultiple($force); + } + else { + $entity = $this->executeSingle($force); + } + + return $entity; + } + + /** + * Process the execution and creation of multiple entities per profile. + * @param [type] $force [description] + * @return [type] [description] + */ + protected function executeMultiple($force) { + + $mapper = $this->getMapper(); $data = $this->getData(); + $numEntities = $mapper->getMultipleEntityCountBySubquery($data); + + // @todo: Remove old ones if they exist. + if ($numEntities <= 0) { + return; + } + + $entityImporter = $this->getEntityImporter(); + $importerMachineName = $entityImporter->getMachineName(); + $entityType = $mapper->getEntityType(); + $bundleType = $mapper->getBundleType(); + + $i = 0; + while ($i < $numEntities) { + $mapper->setIndex($i); + $entity = $this->newEntity($entityType, $bundleType, $data, $mapper); + $this->setStatus(1, 'Created new entity.'); + $i++; + } + + } + + + /** + * Process the execution and creation of a single entity per profile response. + * @param [type] $force [description] + * @return [type] [description] + */ + protected function executeSingle($force) { $mapper = $this->getMapper(); + $data = $this->getData(); $entityImporter = $this->getEntityImporter(); $importerMachineName = $entityImporter->getMachineName(); diff --git a/includes/CAPx/Drupal/Processors/EntityReferenceProcessor.php b/includes/CAPx/Drupal/Processors/EntityReferenceProcessor.php index a19e0562..286097e1 100644 --- a/includes/CAPx/Drupal/Processors/EntityReferenceProcessor.php +++ b/includes/CAPx/Drupal/Processors/EntityReferenceProcessor.php @@ -36,12 +36,13 @@ public function execute() { // Get the profile ID of this entity as the profile id will be the same // for other importers and entity/bundle types. - $profile_id = CAPx::getProfileIdByEntity($this->entity); + + $profile_id = $this->entity->value()->capx['profileId']; // Did not find one. It could be that is hasn't been created yet and may // take another cycle or two to come up. if (!$profile_id) { - return array(); + throw new \Exception('Could not find profileId. Did something change in the API?'); } $match = db_select("capx_profiles", 'capx') diff --git a/includes/CAPx/Drupal/Processors/PropertyProcessors/PropertyProcessorAbstract.php b/includes/CAPx/Drupal/Processors/PropertyProcessors/PropertyProcessorAbstract.php index e2b92ee5..2fe5bd69 100644 --- a/includes/CAPx/Drupal/Processors/PropertyProcessors/PropertyProcessorAbstract.php +++ b/includes/CAPx/Drupal/Processors/PropertyProcessors/PropertyProcessorAbstract.php @@ -42,6 +42,15 @@ public function put($data) { return; } + // If autotruncating is enabled lets do that here. This is to help + // avoid issues when trying to map data to the API. + if (variable_get("stanford_capx_autotruncate_textfields", FALSE)) { + $maxlength = 255; // Default. + if (strlen($data) > $maxlength) { + $data = substr($data, 0, $maxlength); + } + } + try { $entity->{$propertyName}->set($data); } From d748bf16e189f1f38fbb2fe5542c625c436d73c8 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Thu, 19 May 2016 14:54:23 -0700 Subject: [PATCH 11/44] HSDO-325: Multiple entity update routine strategy wip commit --- .../CAPx/Drupal/Mapper/MapperAbstract.php | 1 + .../Drupal/Processors/EntityProcessor.php | 90 +++++++++++++++++-- stanford_capx.forms.inc | 13 +++ stanford_capx.install | 29 +++++- 4 files changed, 125 insertions(+), 8 deletions(-) diff --git a/includes/CAPx/Drupal/Mapper/MapperAbstract.php b/includes/CAPx/Drupal/Mapper/MapperAbstract.php index b10fa2de..6c30a0bc 100644 --- a/includes/CAPx/Drupal/Mapper/MapperAbstract.php +++ b/includes/CAPx/Drupal/Mapper/MapperAbstract.php @@ -194,6 +194,7 @@ public function addConfig($settings) { $mapper->settings['collections'] = array(); $mapper->settings['multiple'] = FALSE; $mapper->settings['subquery'] = ''; + $mapper->settings['guuidquery'] = ''; $settings['fieldCollections'][$fieldName] = new FieldCollectionMapper($mapper); $settings['fieldCollections'][$fieldName]->addConfig($mapper->settings); diff --git a/includes/CAPx/Drupal/Processors/EntityProcessor.php b/includes/CAPx/Drupal/Processors/EntityProcessor.php index 8509aa2e..7170872e 100644 --- a/includes/CAPx/Drupal/Processors/EntityProcessor.php +++ b/includes/CAPx/Drupal/Processors/EntityProcessor.php @@ -15,7 +15,7 @@ class EntityProcessor extends ProcessorAbstract { protected $entity; // Skip etag check. - protected $force = FALSE; + protected $force; /** * Process entity. @@ -32,6 +32,9 @@ class EntityProcessor extends ProcessorAbstract { */ public function execute($force = FALSE) { + // Set this force. + $this->force == $force; + try { $multi = $this->getMapper()->getConfigSetting('multiple'); } @@ -41,10 +44,10 @@ public function execute($force = FALSE) { // Sometimes we need to create one, other times we need moar. if (!empty($multi) && $multi == 1) { - $entity = $this->executeMultiple($force); + $entity = $this->executeMultiple(); } else { - $entity = $this->executeSingle($force); + $entity = $this->executeSingle(); } return $entity; @@ -55,32 +58,105 @@ public function execute($force = FALSE) { * @param [type] $force [description] * @return [type] [description] */ - protected function executeMultiple($force) { + protected function executeMultiple() { $mapper = $this->getMapper(); $data = $this->getData(); $numEntities = $mapper->getMultipleEntityCountBySubquery($data); - // @todo: Remove old ones if they exist. + // Let the Orphan cron runs take care of the clean up. We just need to stop. if ($numEntities <= 0) { return; } + // Time to loop through our results and either update or create entities. $entityImporter = $this->getEntityImporter(); $importerMachineName = $entityImporter->getMachineName(); $entityType = $mapper->getEntityType(); $bundleType = $mapper->getBundleType(); + // Check to see what entities we have for this profile. + $entities = CAPx::getProfiles($entityType, array('profile_id' => $data['profileId'], 'importer' => $importerMachineName)); + + // Looks like we have a whole new batch to create. + if (empty($entities)) { + $this->multipleCreateNewEntity($numEntities, $entityType, $data, $mapper); + return; + } + + // Check if we even need to update. A matching etag will allow us to + // avoid a costly update routine. + if (!$this->isETagDifferent() && !$this->skipEtagCheck()) { + // Nothing to do, same etag and no forceful update. + return; + } + + // At this point we have saved entities and the etag changed or we are + // forcing an update. + + // Strategy: If the user can provide a GUID (fingerprint) then we can try to + // update in place. If the user cannot provide a GUID then we go with the + // delete everything and replace strategy. + + try { + $guuid = $mapper->getConfigSetting("guuidquery"); + } + catch (\Exception $e) { + // An older mapper that has not yet been updated. + // @todo: Think of a way to update the older mappers with an update hook. + } + + // NO GUUID available. Delete all of the existing entities and replace with + // new ones. + if (empty($guuid)) { + $this->mulitpleDeleteEntities($entities); + $this->multipleCreateNewEntity($numEntities, $entityType, $data, $mapper); + } + + // GUUID available lets check to see if it matches the numEntities count. + $numEntityIDs = count($this->getRemoteDataByJsonPath($data, $guuid)); + + // If the total number of ids does not match the total number of entities + // default to the delete and replace method. + if ($numEntityIDs !== $numEntities) { + watchdog('EntityProcessor', "Entity count and GUUID mismatch on: %mapper. Defaulting to delete and replace update method.", array("%mapper" => $mapper->getMachineName())); + $this->mulitpleDeleteEntities($entities); + $this->multipleCreateNewEntity($numEntities, $entityType, $data, $mapper); + } + + // Ok, we are in good shape at this point. We have results, we have ids, and + // now we can update them in place! + + + + } + + /** + * [multipleCreateNewEntity description] + * @param [type] $numEntities [description] + * @param [type] $entityType [description] + * @param [type] $data [description] + * @param [type] $mapper [description] + * @return [type] [description] + */ + protected function multipleCreateNewEntity($numEntities, $entityType, $data, $mapper) { $i = 0; while ($i < $numEntities) { $mapper->setIndex($i); $entity = $this->newEntity($entityType, $bundleType, $data, $mapper); - $this->setStatus(1, 'Created new entity.'); $i++; } - + return TRUE; } + /** + * Deletes all entities passed to this function. + * @param [type] $entities [description] + * @return [type] [description] + */ + protected function multipleDeleteEntities($entityType, $entities) { + entity_delete_multiple($entityType, $entities); + } /** * Process the execution and creation of a single entity per profile response. diff --git a/stanford_capx.forms.inc b/stanford_capx.forms.inc index f5942c0c..cc39e33f 100644 --- a/stanford_capx.forms.inc +++ b/stanford_capx.forms.inc @@ -333,6 +333,18 @@ function stanford_capx_mapper_form_get_entity_mapping_form($form, &$form_state, ), ); + $form['entity-mapping']['guuid-query'] = array( + '#type' => 'textfield', + '#title' => t("Unique ID of entity"), + '#description' => t('Do the entities you are creating have a unique id? For example, publications have a publicationId at $.publications.*.publicationId. If this is provided then CAPx can update the entity in place. If there is no unique ID available then the CAPx module will delete and re-import the entities each time there is a change to the profile.'), + '#default_value' => isset($mapper->guuidquery) ? $mapper->guuidquery : '', + '#states' => array( + 'visible' => array( + ':input[name="multiple-entities"]' => array('checked' => TRUE), + ), + ), + ); + return $form; } @@ -1039,6 +1051,7 @@ function stanford_capx_mapper_form_submit($form, $form_state) { $settings['bundle_type'] = $bundle; $settings['multiple'] = $values['multiple-entities']; $settings['subquery'] = $values['subquery']; + $settings['guuidquery'] = $values['guuid-query']; // If there are properties... $properties = @$values['field-mapping'][$entity . "-properties"]; diff --git a/stanford_capx.install b/stanford_capx.install index f9ed1bc2..54aefb85 100644 --- a/stanford_capx.install +++ b/stanford_capx.install @@ -76,6 +76,12 @@ function stanford_capx_schema() { 'not null' => TRUE, 'default' => '', ), + 'fingerprint' => array( + 'description' => 'fingerprint.', + 'type' => 'varchar', + 'length' => 255, + 'not null' => TRUE, + ), 'sync' => array( 'description' => 'Does this profile needs to be synced?', 'type' => 'int', @@ -103,7 +109,7 @@ function stanford_capx_schema() { ), ); - // Configuration Entity + // Configuration Entity. $schema['capx_cfe'] = array( 'description' => 'The base table for configuration entities.', 'fields' => array( @@ -266,3 +272,24 @@ function stanford_capx_requirements($phase) { } return $requirements; } + +/** + * Add the fingerprint field to the capx_profiles table. + */ +function stanford_capx_update_7100() { + + $table = "capx_profiles"; + $field = "fingerprint"; + $spec = array( + 'description' => 'fingerprint.', + 'type' => 'varchar', + 'length' => 255, + 'not null' => TRUE, + 'default' => '', + ); + + db_add_field($table, $field, $spec); + + + +} From af7726ebdb36df924a2db169713b03937c61d0e7 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Fri, 20 May 2016 12:35:57 -0700 Subject: [PATCH 12/44] HSDO-325: First working version of create and update for multiple entities --- includes/CAPx/Drupal/Mapper/EntityMapper.php | 2 +- .../CAPx/Drupal/Mapper/MapperAbstract.php | 11 ++- .../Drupal/Processors/EntityProcessor.php | 81 ++++++++++++++----- includes/CAPx/Drupal/Util/CAPx.php | 25 +++++- stanford_capx.install | 13 ++- 5 files changed, 103 insertions(+), 29 deletions(-) diff --git a/includes/CAPx/Drupal/Mapper/EntityMapper.php b/includes/CAPx/Drupal/Mapper/EntityMapper.php index 75b2a9d2..0137f770 100644 --- a/includes/CAPx/Drupal/Mapper/EntityMapper.php +++ b/includes/CAPx/Drupal/Mapper/EntityMapper.php @@ -560,7 +560,7 @@ protected function isMultiple() { * [getIndex description] * @return [type] [description] */ - protected function getIndex() { + public function getIndex() { return $this->index; } diff --git a/includes/CAPx/Drupal/Mapper/MapperAbstract.php b/includes/CAPx/Drupal/Mapper/MapperAbstract.php index 6c30a0bc..6f1b2c97 100644 --- a/includes/CAPx/Drupal/Mapper/MapperAbstract.php +++ b/includes/CAPx/Drupal/Mapper/MapperAbstract.php @@ -26,7 +26,6 @@ abstract class MapperAbstract implements MapperInterface { // Error storage so they can be fetched after everything has run. protected $errors = array(); - /** * Merges default configuration options with the passed in set. * @@ -315,5 +314,15 @@ public function getMultipleEntityCountBySubquery($data) { return count($result); } + /** + * [getGUUID description] + * @param [type] $index [description] + * @return [type] [description] + */ + public function getGUUID($data, $index) { + $path = $this->getConfigSetting("guuidquery"); + $guuids = $this->getRemoteDataByJsonPath($data, $path); + return $guuids[$index]; + } } diff --git a/includes/CAPx/Drupal/Processors/EntityProcessor.php b/includes/CAPx/Drupal/Processors/EntityProcessor.php index 7170872e..8234a87f 100644 --- a/includes/CAPx/Drupal/Processors/EntityProcessor.php +++ b/includes/CAPx/Drupal/Processors/EntityProcessor.php @@ -80,7 +80,7 @@ protected function executeMultiple() { // Looks like we have a whole new batch to create. if (empty($entities)) { - $this->multipleCreateNewEntity($numEntities, $entityType, $data, $mapper); + $this->multipleCreateNewEntity($numEntities, $entityType, $bundleType, $data, $mapper); return; } @@ -99,7 +99,7 @@ protected function executeMultiple() { // delete everything and replace strategy. try { - $guuid = $mapper->getConfigSetting("guuidquery"); + $guuidPath = $mapper->getConfigSetting("guuidquery"); } catch (\Exception $e) { // An older mapper that has not yet been updated. @@ -108,26 +108,26 @@ protected function executeMultiple() { // NO GUUID available. Delete all of the existing entities and replace with // new ones. - if (empty($guuid)) { - $this->mulitpleDeleteEntities($entities); - $this->multipleCreateNewEntity($numEntities, $entityType, $data, $mapper); + if (empty($guuidPath)) { + $this->multipleDeleteEntities($entities); + $this->multipleCreateNewEntity($numEntities, $entityType, $bundleType, $data, $mapper); + return; } // GUUID available lets check to see if it matches the numEntities count. - $numEntityIDs = count($this->getRemoteDataByJsonPath($data, $guuid)); + $numGUUIDs = count($mapper->getRemoteDataByJsonPath($data, $guuidPath)); // If the total number of ids does not match the total number of entities // default to the delete and replace method. - if ($numEntityIDs !== $numEntities) { - watchdog('EntityProcessor', "Entity count and GUUID mismatch on: %mapper. Defaulting to delete and replace update method.", array("%mapper" => $mapper->getMachineName())); - $this->mulitpleDeleteEntities($entities); - $this->multipleCreateNewEntity($numEntities, $entityType, $data, $mapper); - } + // if ($numGUUIDs !== $numEntities) { + // watchdog('EntityProcessor', "Entity count and GUUID mismatch on: %mapper. Defaulting to delete and replace update method.", array("%mapper" => $mapper->getMachineName())); + // $this->multipleDeleteEntities($entities); + // $this->multipleCreateNewEntity($numEntities, $entityType, $bundleType, $data, $mapper); + // } // Ok, we are in good shape at this point. We have results, we have ids, and // now we can update them in place! - - + $this->multipleUpdateEntities($numEntities, $entityType, $bundleType, $data, $mapper); } @@ -139,11 +139,13 @@ protected function executeMultiple() { * @param [type] $mapper [description] * @return [type] [description] */ - protected function multipleCreateNewEntity($numEntities, $entityType, $data, $mapper) { + protected function multipleCreateNewEntity($numEntities, $entityType, $bundleType, $data, $mapper) { $i = 0; while ($i < $numEntities) { + // Setting the index tells the mapper which values to save. $mapper->setIndex($i); - $entity = $this->newEntity($entityType, $bundleType, $data, $mapper); + $guuid = $mapper->getGUUID($data, $i); + $entity = $this->newEntity($entityType, $bundleType, $data, $mapper, $guuid); $i++; } return TRUE; @@ -158,6 +160,39 @@ protected function multipleDeleteEntities($entityType, $entities) { entity_delete_multiple($entityType, $entities); } + /** + * Update function for when there are multiple entities being created per + * person. + * @param [type] $numEntities [description] + * @param [type] $entityType [description] + * @param [type] $bundleType [description] + * @param [type] $data [description] + * @param [type] $mapper [description] + * @return [type] [description] + */ + protected function multipleUpdateEntities($numEntities, $entityType, $bundleType, $data, $mapper) { + $i = 0; + while ($i < $numEntities) { + // Setting the index tells the mapper which values to save. + $mapper->setIndex($i); + $guuid = $mapper->getGUUID($data, $i); + $importerMachineName = $this->getEntityImporter()->getMachineName(); + $profileId = $data['profileId']; + + $entity = CAPx::getEntityIdByGUUID($importerMachineName, $profileId, $guuid); + if ($entity) { + $entity = entity_metadata_wrapper($entityType, $entity); + $entity = $this->updateEntity($entity, $data, $mapper); + } + else { + $entity = $this->newEntity($entityType, $bundleType, $data, $mapper, $guuid); + } + + $i++; + } + return TRUE; + } + /** * Process the execution and creation of a single entity per profile response. * @param [type] $force [description] @@ -259,7 +294,8 @@ public function updateEntity($entity, $data, $mapper) { $entityImporter = $this->getEntityImporter(); $importerMachineName = $entityImporter->getMachineName(); - CAPx::updateProfileRecord($entity, $data['profileId'], $data['meta']['etag'], $importerMachineName); + $guuid = $mapper->getGUUID($data, $mapper->getIndex()); + CAPx::updateProfileRecord($entity, $data['profileId'], $data['meta']['etag'], $importerMachineName, $guuid); drupal_alter('capx_post_update_entity', $entity); @@ -281,11 +317,13 @@ public function updateEntity($entity, $data, $mapper) { * The data to be mapped to the new entity * @param object $mapper * The EntityMapper instance + * @param mixed $guuid + * The * * @return object * The new entity after it has been saved. */ - public function newEntity($entityType, $bundleType, $data, $mapper) { + public function newEntity($entityType, $bundleType, $data, $mapper, $guuid) { $properties = array( 'type' => $bundleType, @@ -324,7 +362,7 @@ public function newEntity($entityType, $bundleType, $data, $mapper) { // Write a new record. $entityImporter = $this->getEntityImporter(); $importerMachineName = $entityImporter->getMachineName(); - CAPx::insertNewProfileRecord($entity, $data['profileId'], $data['meta']['etag'], $importerMachineName); + CAPx::insertNewProfileRecord($entity, $data['profileId'], $data['meta']['etag'], $importerMachineName, $guuid); return $entity; } @@ -370,6 +408,13 @@ public function skipEtagCheck($bool = NULL) { if (is_bool($bool)) { $this->force = $bool; } + + // Allow a debug force var. + $debug = variable_get("stanford_capx_debug_always_force_etag", -1); + if ($debug !== -1) { + return $debug; + } + return $this->force; } diff --git a/includes/CAPx/Drupal/Util/CAPx.php b/includes/CAPx/Drupal/Util/CAPx.php index 4d43e53b..957fa706 100644 --- a/includes/CAPx/Drupal/Util/CAPx.php +++ b/includes/CAPx/Drupal/Util/CAPx.php @@ -200,6 +200,24 @@ public static function getProfileIdByEntity($entity) { return isset($query['profile_id']) ? $query['profile_id'] : FALSE; } + /** + * Returns a loaded entity or false. + * + */ + public static function getEntityIdByGUUID($importerMachineName, $profileId, $guuid) { + + $query = db_select("capx_profiles", 'capx') + ->fields('capx', array('entity_type', 'entity_id')) + ->condition('importer', $importerMachineName) + ->condition('profile_id', $profileId) + ->condition('guuid', $guuid) + ->orderBy('id', 'DESC') + ->execute() + ->fetchAssoc(); + + return isset($query['entity_id']) ? array_pop(entity_load($query['entity_type'], array($query['entity_id']))) : FALSE; + } + /** * Create new profile record. * @@ -209,7 +227,7 @@ public static function getProfileIdByEntity($entity) { * @param Entity $entity * The entity that was just saved. */ - public static function insertNewProfileRecord($entity, $profileId, $etag, $importer) { + public static function insertNewProfileRecord($entity, $profileId, $etag, $importer, $guuid = '') { // BEAN is returning its delta when using this. // $entityId = $entity->getIdentifier(); @@ -229,6 +247,7 @@ public static function insertNewProfileRecord($entity, $profileId, $etag, $impor 'sync' => 1, 'last_sync' => $time, 'orphaned' => 0, + 'guuid' => $guuid, ); $yes = drupal_write_record('capx_profiles', $record); @@ -247,7 +266,7 @@ public static function insertNewProfileRecord($entity, $profileId, $etag, $impor * @param [type] $importer [description] * @return [type] [description] */ - public static function updateProfileRecord($entity, $profileId, $etag, $importer) { + public static function updateProfileRecord($entity, $profileId, $etag, $importer, $guuid = '') { $time = time(); // BEAN is returning its delta when using this. @@ -265,6 +284,7 @@ public static function updateProfileRecord($entity, $profileId, $etag, $importer 'profile_id' => $profileId, 'etag' => $etag, 'last_sync' => $time, + 'guuid' => $guuid, ); $keys = array( @@ -272,6 +292,7 @@ public static function updateProfileRecord($entity, $profileId, $etag, $importer 'entity_id', 'importer', 'profile_id', + 'guuid', ); $yes = drupal_write_record('capx_profiles', $record, $keys); diff --git a/stanford_capx.install b/stanford_capx.install index 54aefb85..e631a1b9 100644 --- a/stanford_capx.install +++ b/stanford_capx.install @@ -21,6 +21,7 @@ function stanford_capx_uninstall() { variable_del('stanford_capx_token'); variable_del("capx_last_orgs_sync"); variable_del("capx_last_schema_sync"); + variable_det("stanford_capx_debug_always_force_etag"); } /** @@ -76,8 +77,8 @@ function stanford_capx_schema() { 'not null' => TRUE, 'default' => '', ), - 'fingerprint' => array( - 'description' => 'fingerprint.', + 'guuid' => array( + 'description' => 'A guuid that is not the profile id. Used for multiple creation.', 'type' => 'varchar', 'length' => 255, 'not null' => TRUE, @@ -274,14 +275,14 @@ function stanford_capx_requirements($phase) { } /** - * Add the fingerprint field to the capx_profiles table. + * Add the guuid field to the capx_profiles table. */ function stanford_capx_update_7100() { $table = "capx_profiles"; - $field = "fingerprint"; + $field = "guuid"; $spec = array( - 'description' => 'fingerprint.', + 'description' => 'A guuid that is not the profile id. Used for multiple creation.', 'type' => 'varchar', 'length' => 255, 'not null' => TRUE, @@ -290,6 +291,4 @@ function stanford_capx_update_7100() { db_add_field($table, $field, $spec); - - } From 4f9d46f735f3afb252827568461a9cb799b432f4 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Fri, 20 May 2016 17:11:21 -0700 Subject: [PATCH 13/44] HSDO-325: Regular mapper fix --- includes/CAPx/Drupal/Mapper/MapperAbstract.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/includes/CAPx/Drupal/Mapper/MapperAbstract.php b/includes/CAPx/Drupal/Mapper/MapperAbstract.php index 6f1b2c97..8f1caeac 100644 --- a/includes/CAPx/Drupal/Mapper/MapperAbstract.php +++ b/includes/CAPx/Drupal/Mapper/MapperAbstract.php @@ -321,6 +321,9 @@ public function getMultipleEntityCountBySubquery($data) { */ public function getGUUID($data, $index) { $path = $this->getConfigSetting("guuidquery"); + if (empty($path)) { + return ''; + } $guuids = $this->getRemoteDataByJsonPath($data, $path); return $guuids[$index]; } From 2f54d82d2d3d6474c0c65cfe9733c78829726156 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Wed, 25 May 2016 10:12:56 -0700 Subject: [PATCH 14/44] HSDO-325: Multiple orphaned and adoption programs --- .../Orphans/EntityImporterOrphans.php | 227 ++++++++++++++---- .../Orphans/EntityImporterOrphansBatch.php | 2 +- .../Orphans/Lookups/LookupMissingFromAPI.php | 2 +- .../Lookups/LookupMissingMultipleByGUUID.php | 62 +++++ includes/CAPx/Drupal/Mapper/EntityMapper.php | 26 +- includes/CAPx/Drupal/Util/CAPxImporter.php | 53 ++-- stanford_capx.forms.inc | 2 +- 7 files changed, 310 insertions(+), 64 deletions(-) create mode 100644 includes/CAPx/Drupal/Importer/Orphans/Lookups/LookupMissingMultipleByGUUID.php diff --git a/includes/CAPx/Drupal/Importer/Orphans/EntityImporterOrphans.php b/includes/CAPx/Drupal/Importer/Orphans/EntityImporterOrphans.php index acad4340..8726e193 100644 --- a/includes/CAPx/Drupal/Importer/Orphans/EntityImporterOrphans.php +++ b/includes/CAPx/Drupal/Importer/Orphans/EntityImporterOrphans.php @@ -69,10 +69,12 @@ public function __construct(EntityImporter $importer, Array $profiles, Array $lo $this->setProfiles($profiles); $this->setLimit(count($profiles)); + // Store all of the lookups. if (is_array($lookups)) { $this->lookups = $lookups; } + // Store all of the comparisons. if (is_array($comparisons)) { $this->comparisons = $comparisons; } @@ -87,9 +89,6 @@ public function execute() { // If the action is set to do nothing to orphaned profiles, do nothing. $options = $this->getImporterOptions(); $action = $options['orphan_action']; - - $importer = $this->getImporter(); - $options = $this->getImporterOptions(); $profiles = $this->getProfiles(); $client = $this->getClient(); $limit = $this->getLimit(); @@ -120,35 +119,42 @@ public function execute() { // Now that we have identified all of the orphans lets compare them to each // way we can import a profile to identify if an orphan in one is not an // orphan in another. Only an orphan across all ways is a true orphan. - $comparisons = $this->getComparisons(); foreach ($comparisons as $k => $comparison) { $orphans = $comparison->execute($this); $this->setOrphans($orphans); } - // If we have no orphans after all of that just end. - if (empty($orphans)) { - return; + // Special processor for multiple items. + if (isset($orphans['multiple'])) { + $this->processMultipleImporterOrphans($orphans['multiple']); + unset($orphans['multiple']); } // We have looked at everything and now it is time to process the orphans. // In order to be an orphan the orphan id has to appear in all importer // Options. So we can just take one and run the process on that. - $orphaned = array_pop($orphans); + $orphaned = end($orphans); + + // Always want to do this in case the action has changed. + $this->processAdopted($profiles, $orphaned); + $this->processAdoptedMultiple($profiles); + + // If we have no orphans after all of that just end. + if (empty($orphans)) { + return; + } // Small patch up fix. if (isset($orphans["missing"])) { $orphaned = array_merge($orphaned, $orphans["missing"]); } + // If no action is required then just skip over this and go to the adopted. if ($action !== "nothing") { - $this->processOrphans($orphaned, $importer); + $this->processOrphans($orphaned); } - // Always want to do this in case the action has changed. - $this->processAdopted($profiles, $orphaned); - } /** @@ -331,6 +337,45 @@ public function getResults() { // /////////////////////////////////////////////////////////////////////////// + /** + * @param $entityIds + */ + protected function processMultipleImporterOrphans($entityIds) { + $importer = $this->getImporter(); + $entityType = $importer->getEntityType(); + $options = $this->getImporterOptions(); + $action = $options['orphan_action']; + + // Do nothing John Snow. + if ($action == "nothing") { + return; + } + + foreach ($entityIds as $entityId) { + $entity = entity_load($entityType, array($entityId)); + $profile = entity_metadata_wrapper($entityType, $entity[$entityId]); + + switch ($action) { + case 'delete': + $profile->delete(); + break; + + case 'block': + case 'unpublish': + $profile->status->set(0); + $profile->save(); + + // Log that this profile was orphaned. + $this->logOrphan($profile); + break; + + default: + // Do nothing. + } + } + + } + /** * Handles what to do when a profile has been orphaned. @@ -375,6 +420,62 @@ public function processOrphans($profileIds) { } } + + /** + * Logs that a profile was orphaned. + * + * @param object $profile + * The loaded and wrapped entity metadata. + */ + public function logOrphan($entity) { + + // Set the flag to 1 in the capx_profiles table. + + // BEAN is returning its delta when using this. + // $id = $entity->getIdentifier(); + + $entityType = $entity->type(); + $entityRaw = $entity->raw(); + list($id, $vid, $bundle) = entity_extract_ids($entityType, $entityRaw); + + $guuid = $entityRaw->capx['guuid']; + + $importer = $this->getImporter(); + $importerName = $importer->getMachineName(); + $entityType = $importer->getEntityType(); + $bundleType = $importer->getBundleType(); + + $record = array( + 'entity_type' => $entityType, + 'bundle_type' => $bundleType, + 'entity_id' => $id, + 'importer' => $importerName, + 'orphaned' => 1, + ); + + // For the multiple entity. + if (!empty($guuid)) { + $record['guuid'] = $guuid; + } + + $keys = array( + 'entity_type', + 'entity_id', + 'importer', + 'bundle_type', + ); + + // For multiple entities. + if (!empty($guuid)) { + $keys[] = 'guuid'; + } + + $yes = drupal_write_record('capx_profiles', $record, $keys); + + if ($yes) { + watchdog('EntityImporterOrphans', "%title was orphaned from the importer %importername.", array("%title" => $entity->label(), "%importername" => $importerName), WATCHDOG_NOTICE, ''); + } + } /** * Orphaned profiles that have returned to the importer. @@ -432,49 +533,87 @@ public function processAdopted($profiles, $orphans) { return $orphans; } + /** - * Logs that a profile was orphaned. + * Orphaned profiles that have returned to the importer. * - * @param object $profile - * The loaded and wrapped entity metadata. + * Find and process profiles that have been added back into an import by + * removing their orphan status. + * + * @param array $profiles + * An array of profile ids + * @param array $orphans + * An array of orphaned profile ides */ - public function logOrphan($entity) { - - // Set the flag to 1 in the capx_profiles table. - - // BEAN is returning its delta when using this. - // $id = $entity->getIdentifier(); - - $entityType = $entity->type(); - $entityRaw = $entity->raw(); - list($id, $vid, $bundle) = entity_extract_ids($entityType, $entityRaw); - + public function processAdoptedMultiple($profiles) { $importer = $this->getImporter(); - $importerName = $importer->getMachineName(); + $importerMachineName = $importer->getMachineName(); $entityType = $importer->getEntityType(); $bundleType = $importer->getBundleType(); + $options = $importer->getOptions(); + $orphanAction = $options['orphan_action']; - $record = array( - 'entity_type' => $entityType, - 'bundle_type' => $bundleType, - 'entity_id' => $id, - 'importer' => $importerName, - 'orphaned' => 1, - ); + if ($orphanAction !== "unpublish") { + // Nothing actionable to perform. + return; + } - $keys = array( - 'entity_type', - 'entity_id', - 'importer', - 'bundle_type', - ); + $results = $this->getResults(); + $mapper = $this->getImporter()->getMapper(); + $guuidquery = $mapper->getGUUIDQuery(); + $parts = explode(".", $guuidquery); + $subquery = "$.." . array_pop($parts); + $remoteGUUIDs = $mapper->getRemoteDataByJsonPath($results, $subquery); + + $or = db_or()->condition('profile_id', $profiles); + $localResults = db_select("capx_profiles", "cxp") + ->fields('cxp', array('entity_id', 'guuid')) + ->condition($or) + ->condition("importer", $importerMachineName) + ->condition("orphaned", 1) + ->execute() + ->fetchAllAssoc('guuid'); + + $localGUUIDs = array_keys($localResults); + $diff = array_intersect($localGUUIDs, $remoteGUUIDs); + + // Nothing to adopt. Yay. + if (empty($diff)) { + return array(); + } - $yes = drupal_write_record('capx_profiles', $record, $keys); + foreach ($diff as $guuid) { + $entityID = $localResults[$guuid]->entity_id; + $ids = array($entityID); + $entity = array_pop(entity_load($entityType, $ids)); + $profile = entity_metadata_wrapper($entityType, $entity); + $profile->status->set(1); + $profile->save(); + + // Profile is not an orphan. Lets enable it again. + $record = array( + 'entity_type' => $entityType, + 'bundle_type' => $bundleType, + 'entity_id' => $entityID, + 'importer' => $importerMachineName, + 'orphaned' => 0, + 'guuid' => $guuid, + ); - if ($yes) { - watchdog('EntityImporterOrphans', "%title was orphaned from the importer %importername.", array("%title" => $entity->label(), "%importername" => $importerName), WATCHDOG_NOTICE, ''); + $keys = array( + 'entity_type', + 'entity_id', + 'importer', + 'bundle_type', + 'guuid', + ); + + drupal_write_record('capx_profiles', $record, $keys); } - } + } + + + } diff --git a/includes/CAPx/Drupal/Importer/Orphans/EntityImporterOrphansBatch.php b/includes/CAPx/Drupal/Importer/Orphans/EntityImporterOrphansBatch.php index bfd39d06..cbf28f65 100644 --- a/includes/CAPx/Drupal/Importer/Orphans/EntityImporterOrphansBatch.php +++ b/includes/CAPx/Drupal/Importer/Orphans/EntityImporterOrphansBatch.php @@ -13,7 +13,7 @@ class EntityImporterOrphansBatch { /** * Callback for batch import functionality. - + * * @param string $importerMachineName * The machine name of the importer configuration entity. * @param int $profiles diff --git a/includes/CAPx/Drupal/Importer/Orphans/Lookups/LookupMissingFromAPI.php b/includes/CAPx/Drupal/Importer/Orphans/Lookups/LookupMissingFromAPI.php index e24c8449..e73b8c3d 100644 --- a/includes/CAPx/Drupal/Importer/Orphans/Lookups/LookupMissingFromAPI.php +++ b/includes/CAPx/Drupal/Importer/Orphans/Lookups/LookupMissingFromAPI.php @@ -9,7 +9,7 @@ class LookupMissingFromApi implements LookupInterface { /** - * Checks the API server to see if the SUNET item still exists. + * Checks the API server to see if the item still exists. * * @param EntityImporterOrphans $orphaner * The orphan processor object. diff --git a/includes/CAPx/Drupal/Importer/Orphans/Lookups/LookupMissingMultipleByGUUID.php b/includes/CAPx/Drupal/Importer/Orphans/Lookups/LookupMissingMultipleByGUUID.php new file mode 100644 index 00000000..4cb02883 --- /dev/null +++ b/includes/CAPx/Drupal/Importer/Orphans/Lookups/LookupMissingMultipleByGUUID.php @@ -0,0 +1,62 @@ + + */ + +namespace CAPx\Drupal\Importer\Orphans\Lookups; + +class LookupMissingMultipleByGUUID implements LookupInterface { + + /** + * Checks the API server to see if the item still exists. + * + * @param EntityImporterOrphans $orphaner + * The orphan processor object. + * + * @return array + * The remaining orphans + */ + public function execute($orphaner) { + $results = $orphaner->getResults(); + $profiles = $orphaner->getProfiles(); + $importer = $orphaner->getImporter()->getMachineName(); + $mapper = $orphaner->getImporter()->getMapper(); + $guuidquery = $mapper->getGUUIDQuery(); + $parts = explode(".", $guuidquery); + $subquery = "$.." . array_pop($parts); + $remoteGUUIDs = $mapper->getRemoteDataByJsonPath($results, $subquery); + $localInfo = $this->getLocalGUUIDs($profiles, $importer); + $localGUUIDs = array_keys($localInfo); + $diff = array_diff($localGUUIDs, $remoteGUUIDs); + + // No orphans. Yay. + if (empty($diff)) { + return array(); + } + + // New column for the comparisons. + $orphans = array('multiple' => array()); + foreach ($diff as $key => $guuid) { + $orphans['multiple'][] = $localInfo[$guuid]->entity_id; + } + + return $orphans; + } + + /** + * @param $profileIDs + * @param $importerMachineName + */ + protected function getLocalGUUIds($profileIDs, $importerMachineName) { + $or = db_or()->condition('profile_id', $profileIDs); + $results = db_select("capx_profiles", "cxp") + ->fields('cxp', array('entity_id', 'guuid')) + ->condition($or) + ->condition("importer", $importerMachineName) + ->execute() + ->fetchAllAssoc('guuid'); + return $results; + } + +} diff --git a/includes/CAPx/Drupal/Mapper/EntityMapper.php b/includes/CAPx/Drupal/Mapper/EntityMapper.php index 0137f770..f152db85 100644 --- a/includes/CAPx/Drupal/Mapper/EntityMapper.php +++ b/includes/CAPx/Drupal/Mapper/EntityMapper.php @@ -547,7 +547,7 @@ public function getAffectedImporters() { * Boolean to whether or not this is a multiple import or not. * @return boolean [description] */ - protected function isMultiple() { + public function isMultiple() { try { return $this->getConfigSetting("multiple"); } @@ -556,6 +556,30 @@ protected function isMultiple() { } } + /** + * get Subquery wrapper + */ + public function getSubquery() { + try { + return $this->getConfigSetting("subquery"); + } + catch (\Exception $e) { + return FALSE; + } + } + + /** + * get guuid wrapper + */ + public function getGUUIDQuery() { + try { + return $this->getConfigSetting("guuidquery"); + } + catch (\Exception $e) { + return FALSE; + } + } + /** * [getIndex description] * @return [type] [description] diff --git a/includes/CAPx/Drupal/Util/CAPxImporter.php b/includes/CAPx/Drupal/Util/CAPxImporter.php index 4f1419dd..3af744d6 100644 --- a/includes/CAPx/Drupal/Util/CAPxImporter.php +++ b/includes/CAPx/Drupal/Util/CAPxImporter.php @@ -26,6 +26,7 @@ use CAPx\Drupal\Importer\Orphans\Lookups\LookupMissingFromSunetList; use CAPx\Drupal\Importer\Orphans\Lookups\LookupOrgOrphans; use CAPx\Drupal\Importer\Orphans\Lookups\LookupWorkgroupOrphans; +use CAPx\Drupal\Importer\Orphans\Lookups\LookupMissingMultipleByGUUID; class CAPxImporter { @@ -155,39 +156,59 @@ public static function getCronOptions() { * @return [type] [description] */ public static function getEntityOrphanator($importerName, $profiles = array()) { - $orphanator = NULL; + $orphanator = NULL; $importer = CAPxImporter::loadEntityImporter($importerName); - if ($importer) { - $lookups = array(); - $comparisons = array(); + // If we couldn't load the importer we need to log and error out. + if (!$importer) { + $vars = array( + '%name' => $importerName, + '!log' => l(t('log messages'), 'admin/reports/dblog'), + ); + drupal_set_message(t('There was an issue loading the importer with %name machine name. Check !log.', $vars), 'error'); + return; + } + + // Track wether this is a multiple entity importer or not. The checks differ + // for these types. + $mapper = $importer->getMapper(); + $multiple = $mapper->isMultiple(); + $lookups = array(); + $comparisons = array(); - // Load all the lookups... - $lookups[] = new LookupMissingFromAPI(); + // Load the lookups... + $lookups[] = new LookupMissingFromAPI(); + + // Only want these if not a multiple import. + if (!$multiple) { $lookups[] = new LookupMissingFromSunetList(); $lookups[] = new LookupOrgOrphans(); $lookups[] = new LookupWorkgroupOrphans(); + } - // Load all the comparisons... - $comparisons[] = new CompareMissingFromAPI(); + // Load the comparisons... + $comparisons[] = new CompareMissingFromAPI(); + + // Only want these if not a multiple import. + if (!$multiple) { $comparisons[] = new CompareOrgCodesSunet(); $comparisons[] = new CompareOrgCodesWorkgroups(); $comparisons[] = new CompareSunetOrgCodes(); $comparisons[] = new CompareSunetWorkgroups(); $comparisons[] = new CompareWorkgroupsOrgCodes(); $comparisons[] = new CompareWorkgroupsSunet(); - - $orphanator = new EntityImporterOrphans($importer, $profiles, $lookups, $comparisons); } - else { - $vars = array( - '%name' => $importerName, - '!log' => l(t('log messages'), 'admin/reports/dblog'), - ); - drupal_set_message(t('There was an issue loading the importer with %name machine name. Check !log.', $vars), 'error'); + + // For multiple entity importers we need to check not only if the profile + // exists but the part of the profile that is being imported still exists. + if ($multiple) { + $lookups[] = new LookupMissingMultipleByGUUID(); } + // Load it up and send it along the way. + $orphanator = new EntityImporterOrphans($importer, $profiles, $lookups, $comparisons); + return $orphanator; } } diff --git a/stanford_capx.forms.inc b/stanford_capx.forms.inc index cc39e33f..11c05ecd 100644 --- a/stanford_capx.forms.inc +++ b/stanford_capx.forms.inc @@ -336,7 +336,7 @@ function stanford_capx_mapper_form_get_entity_mapping_form($form, &$form_state, $form['entity-mapping']['guuid-query'] = array( '#type' => 'textfield', '#title' => t("Unique ID of entity"), - '#description' => t('Do the entities you are creating have a unique id? For example, publications have a publicationId at $.publications.*.publicationId. If this is provided then CAPx can update the entity in place. If there is no unique ID available then the CAPx module will delete and re-import the entities each time there is a change to the profile.'), + '#description' => t('Do the entities you are creating have a unique id? For example, publications have a publicationId and $.publications.*.publicationId would be the value to add to this field. If this is provided then CAPx can update the entity in place. If there is no unique ID available then the CAPx module will delete and re-import the entities each time there is a change to the profile.'), '#default_value' => isset($mapper->guuidquery) ? $mapper->guuidquery : '', '#states' => array( 'visible' => array( From 157c732267430f3aa1eb8fc1414a348e76b765ce Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Wed, 25 May 2016 12:14:49 -0700 Subject: [PATCH 15/44] HSDO-325: Fixed message issues --- includes/CAPx/Drupal/Importer/EntityImporterBatch.php | 5 +++++ .../Drupal/Importer/Orphans/EntityImporterOrphans.php | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/includes/CAPx/Drupal/Importer/EntityImporterBatch.php b/includes/CAPx/Drupal/Importer/EntityImporterBatch.php index 17fd76f4..246f63b4 100644 --- a/includes/CAPx/Drupal/Importer/EntityImporterBatch.php +++ b/includes/CAPx/Drupal/Importer/EntityImporterBatch.php @@ -165,6 +165,11 @@ public static function processResults($results, $importer) { $message = $processor->getStatusMessage(); + // Nada. + if (empty($message)) { + continue; + } + // Log some information. // @todo This needs to be better. watchdog('stanford_capx', $message . " | " . $info['displayName'], array(), WATCHDOG_DEBUG); diff --git a/includes/CAPx/Drupal/Importer/Orphans/EntityImporterOrphans.php b/includes/CAPx/Drupal/Importer/Orphans/EntityImporterOrphans.php index 8726e193..d33fca72 100644 --- a/includes/CAPx/Drupal/Importer/Orphans/EntityImporterOrphans.php +++ b/includes/CAPx/Drupal/Importer/Orphans/EntityImporterOrphans.php @@ -420,7 +420,7 @@ public function processOrphans($profileIds) { } } - + /** * Logs that a profile was orphaned. * @@ -613,7 +613,7 @@ public function processAdoptedMultiple($profiles) { } - - - + + + } From 9143c6adf230a1736b67c66b17b75a5fdf77d2de Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Wed, 25 May 2016 17:50:00 -0700 Subject: [PATCH 16/44] CAPX-213: Merge the connect and settings forms in to one. --- stanford_capx.forms.inc | 255 +++++++++++++++++++++------------------- stanford_capx.module | 23 ++-- stanford_capx.pages.inc | 51 +++----- 3 files changed, 161 insertions(+), 168 deletions(-) diff --git a/stanford_capx.forms.inc b/stanford_capx.forms.inc index ea5dde06..d2bcb4d7 100644 --- a/stanford_capx.forms.inc +++ b/stanford_capx.forms.inc @@ -13,83 +13,6 @@ use CAPx\Drupal\Util\CAPxMapper; use CAPx\Drupal\Util\CAPxImporter; use \Peekmo\JsonPath\JsonStore as JsonParser; -// ///////////////////////////////////////////////////////////////////////////// -// CONNECT FORM -// ///////////////////////////////////////////////////////////////////////////// - - -/** - * Connection details form builder. - */ -function stanford_capx_forms_connect_form($form, &$form_state) { - - $username = CAPx::getAuthUsername(); - $connection = CAPxConnection::testConnection(); - - $form['status'] = array( - '#type' => 'fieldset', - '#title' => t('Connection Status'), - '#collapsible' => TRUE, - ); - - $form['status']['status'] = array( - '#markup' => stanford_capx_connection_status_block(), - ); - - $form['auth'] = array( - '#type' => 'fieldset', - '#title' => t('Authorization'), - '#collapsible' => TRUE, - '#collapsed' => $connection->status, - ); - - $form['auth']['description'] = array( - '#markup' => t('Please enter your authentication information for the CAP API.'), - ); - - $form['auth']['stanford_capx_username'] = array( - '#type' => 'textfield', - '#title' => t('Client ID:'), - '#default_value' => $username, - '#required' => TRUE, - ); - - $form['auth']['stanford_capx_password'] = array( - '#type' => 'password', - '#title' => t('Password:'), - ); - - $form['advanced'] = array( - '#type' => 'fieldset', - '#title' => t('Advanced'), - '#collapsible' => TRUE, - '#collapsed' => TRUE, - '#description' => t('Advanced setting for CAP API and authentication URIs'), - ); - - $form['advanced']['stanford_capx_api_base_url'] = array( - '#type' => 'textfield', - '#title' => t('Endpoint'), - '#description' => t('CAP API endpoint URI, only useful when switching between development/production environment.'), - '#default_value' => CAPx::getAPIEndpoint(), - '#required' => TRUE, - ); - - $form['advanced']['stanford_capx_api_auth_uri'] = array( - '#type' => 'textfield', - '#title' => t('Authentication URI'), - '#description' => t('CAP API authentication URI.'), - '#default_value' => CAPx::getAuthEndpoint(), - '#required' => TRUE, - ); - - $form['submit'] = array( - '#type' => 'submit', - '#value' => t('Save connection settings'), - ); - - return $form; -} /** * [stanford_capx_forms_connect_form_validate description] @@ -104,46 +27,17 @@ function stanford_capx_forms_connect_form_validate($form, &$form_state) { $endpoint = $form_state['values']['stanford_capx_api_base_url']; $authpoint = $form_state['values']['stanford_capx_api_auth_uri']; - // If password changed validate auth. - if (!empty($password)) { - $auth_test = CAPxConnection::testAuthConnection($username, $password, $authpoint); - if (!$auth_test->status) { - form_set_error('stanford_capx_username', t('Could not authenticate with server. Invalid credentials. Please check your username and password.')); - form_set_error('stanford_capx_password'); - } - } - - // In case the end points change we need to get pass from db. - $password = !empty($password) ? $password : CAPx::getAuthPassword(); - - // If authpoint changed. - if ($authpoint !== "https://authz.stanford.edu/oauth/token") { - - if (!isset($auth_test)) { - $auth_test = CAPxConnection::testAuthConnection($username, $password, $authpoint); - } - - if (!$auth_test->status) { - form_set_error("stanford_capx_api_auth_uri", t("Please check your authorization url.")); - } - + // Do not process a change on an empty password. Re-apply the password from + // the database because the system_form will save an empty value otherwise. + if (empty($password)) { + return; } - // If endpoint changed. - if ($endpoint !== "https://api.stanford.edu") { - - if (!isset($auth_test)) { - $auth_test = CAPxConnection::testAuthConnection($username, $password, $authpoint); - } - - if (isset($auth_test->token)) { - $token = $auth_test->token; - $end_test = CAPxConnection::testApiConnection($token, $endpoint); - if (!$end_test->status) { - form_set_error("stanford_capx_api_base_url", t("Could not connect to API endpoint.")); - } - } - + // If password changed validate auth. + $auth_test = CAPxConnection::testAuthConnection($username, $password, $authpoint); + if (!$auth_test->status) { + form_set_error('stanford_capx_username', t('Could not authenticate with server. Invalid credentials. Please check your username and password.')); + form_set_error('stanford_capx_password'); } } @@ -156,7 +50,7 @@ function stanford_capx_forms_connect_form_validate($form, &$form_state) { */ function stanford_capx_forms_connect_form_submit($form, &$form_state) { - drupal_set_message(t('Configuration options were saved.')); + drupal_set_message(t("Configuration options saved."), "status"); // Username and Pass. if (!empty($form_state['values']['stanford_capx_username']) @@ -167,16 +61,16 @@ function stanford_capx_forms_connect_form_submit($form, &$form_state) { variable_set('stanford_capx_username', $username); variable_set('stanford_capx_password', $password); + // In order to get here we have to have a valid connection. + // Lets get the org codes and schema. + stanford_capx_schema_refresh(); + // stanford_capx_organizations_sync(); } + // Endpoints. variable_set('stanford_capx_api_base_url', $form_state['values']['stanford_capx_api_base_url']); variable_set('stanford_capx_api_auth_uri', $form_state['values']['stanford_capx_api_auth_uri']); - - // In order to get here we have to have a valid connection. Lets get the org codes - // and schema. - stanford_capx_schema_refresh(); - stanford_capx_organizations_sync(); } // ///////////////////////////////////////////////////////////////////////////// @@ -1373,6 +1267,43 @@ function stanford_capx_importer_form($form, &$form_state, $importer = NULL) { return $form; } +/** + * [stanford_capx_config_settings_data_sync_form description] + * @param [type] $form [description] + * @param [type] $form_state [description] + * @return [type] [description] + */ +function stanford_capx_config_settings_data_sync_form($form, $form_state) { + $form['orggroup'] = array( + '#type' => "fieldset", + '#title' => t("Organizations & Schema"), + '#description' => t("The CAPx module needs some information from the CAP API in order to function properly. Below are actions that require communication with the CAP API server and you must be connected before you can run these tasks."), + '#collapsible' => TRUE, + '#collapsed' => FALSE, + ); + + // Organizations. + $form['orggroup']['orgcodeinfo']['#markup'] = "

Organizations

"; + $form['orggroup']['orgcodeinfo']['#markup'] .= "

" . t("Organization codes come from the CAP API and are saved to a taxonomy. This taxonomy powers the organization code autocomplete in the import section.") . "

"; + $form['orggroup']['orgcodeinfo']['#markup'] .= "

" . t("Last sync: @date", array("@date" => variable_get('capx_last_orgs_sync', 'never'))) . "

"; + $form['orggroup']['orgcodeinfo']['#markup'] .= "

" . t("View all organization codes: !link", array("!link" => l(t('Organization taxonomy'), 'admin/structure/taxonomy/capx_organizations'))) . "

"; + $form['orggroup']['orgcodeinfo']['#markup'] .= "

" . l(t('Get organization data'), 'admin/config/capx/organizations/sync', array("attributes" => array('class' => array('btn button')), 'query' => array('destination' => current_path()))) . "

"; + + // Schema. + $form['orggroup']['schemainfo']['#markup'] = "

Schema Information

"; + $form['orggroup']['schemainfo']['#markup'] .= "

" . t("The CAP API provides a schema of the information available. The CAPx module needs this in order to power the data browser that is found on a mapping page.") . "

"; + $form['orggroup']['schemainfo']['#markup'] .= "

" . t("Last sync: @date", array("@date" => variable_get('capx_last_schema_sync', 'never'))) . "

"; + $form['orggroup']['submit'] = array( + '#type' => 'button', + '#value' => 'Get schema information', + '#op' => 'submit', + // '#element_validate' => array("stanford_capx_schema_refresh"), // Yes, I know this is abuse. + ); + + // $form['#submit'][] = "stanford_capx_schema_refresh"; + return $form; +} + /** * AJAX callback. */ @@ -1546,7 +1477,7 @@ function stanford_capx_admin_config_importer_delete($form, &$form_state, $machin $form['profile_action'] = array( '#type' => 'select', '#title' => t('What would you like to do with the items that are associated with this importer?'), -// '#description' => t('Perform an action on all of the items that have been imported by this importer.'), + // '#description' => t('Perform an action on all of the items that have been imported by this importer.'), '#options' => array( 'nothing' => t('Do nothing and leave the items as they are'), 'delete' => t('Delete all of the items'), @@ -1924,6 +1855,70 @@ function stanford_capx_schema_format_validation($schema = NULL) { return $formatted_schema; } +/** + * Auth settings form. Was previously the connect page. + * @param [type] $form [description] + * @param [type] &$form_state [description] + * @return [type] [description] + */ +function stanford_capx_config_auth_settings_form($form, &$form_state) { + $username = CAPx::getAuthUsername(); + $connection = CAPxConnection::testConnection(); + + $form['auth'] = array( + '#type' => 'fieldset', + '#title' => t('Authorization'), + '#collapsible' => TRUE, + '#collapsed' => $connection->status, + '#weight' => -10, + ); + + $form['auth']['description'] = array( + '#markup' => t('Please enter your authentication information for the CAP API. If you don\'t have these credentials yet you can !helpsu.', array('!helpsu' => l("File a HelpSU request", "https://helpsu.stanford.edu/helpsu/3.0/auth/helpsu-form?pcat=CAP_API&dtemplate=CAP-OAuth-Info"))), + ); + + $form['auth']['stanford_capx_username'] = array( + '#type' => 'textfield', + '#title' => t('Client ID:'), + '#default_value' => $username, + '#required' => TRUE, + ); + + $form['auth']['stanford_capx_password'] = array( + '#type' => 'password', + '#title' => t('Password:'), + ); + + $form['advanced'] = array( + '#type' => 'fieldset', + '#title' => t('Advanced'), + '#collapsible' => TRUE, + '#collapsed' => TRUE, + '#description' => t('Advanced setting for CAP API and authentication URIs'), + '#access' => user_access("capx advanced administrator"), + ); + + $form['advanced']['stanford_capx_api_base_url'] = array( + '#type' => 'textfield', + '#title' => t('Endpoint'), + '#description' => t('CAP API endpoint URI, only useful when switching between development/production environment.'), + '#default_value' => CAPx::getAPIEndpoint(), + '#required' => TRUE, + ); + + $form['advanced']['stanford_capx_api_auth_uri'] = array( + '#type' => 'textfield', + '#title' => t('Authentication URI'), + '#description' => t('CAP API authentication URI.'), + '#default_value' => CAPx::getAuthEndpoint(), + '#required' => TRUE, + ); + + $form['#validate'][] = "stanford_capx_forms_connect_form_validate"; + $form['#submit'][] = "stanford_capx_forms_connect_form_submit"; + + return $form; +} /** * The global configuration settings for CAPx form builder. @@ -1959,11 +1954,23 @@ function stanford_capx_config_settings_form($form, &$form_state) { '#default_value' => variable_get('stanford_capx_default_field_format', 'filtered_html'), ); - $form = system_settings_form($form); - $form['actions']['submit']['#value'] = t("Save settings"); + $form["#submit"][] = "stanford_capx_config_settings_form_submit"; + return $form; } +/** + * Submit function for stanford_capx_config_settings_form + * @param [type] $form [description] + * @param [type] &$form_state [description] + * @return [type] [description] + */ +function stanford_capx_config_settings_form_submit($form, &$form_state) { + $values = $form_state["values"]; + variable_set("stanford_capx_batch_limit", check_plain($values['stanford_capx_batch_limit'])); + variable_set("stanford_capx_default_field_format", check_plain($values['stanford_capx_default_field_format'])); +} + /** * Validates wether or not a path is acceptible. * diff --git a/stanford_capx.module b/stanford_capx.module index 07823b82..e74dba39 100644 --- a/stanford_capx.module +++ b/stanford_capx.module @@ -69,7 +69,19 @@ function stanford_capx_permission() { return array( 'administer capx' => array( 'title' => t('Administer CAPx'), - 'description' => t('Administer and Configure CAPx settings.'), + 'description' => t('Administer and configure CAPx settings.'), + ), + 'capx administer importers' => array( + 'title' => t('Administer CAPx Importers'), + 'description' => t('Administer and configure CAPx importers.'), + ), + 'capx administer mappers' => array( + 'title' => t('Administer CAPx Mappers'), + 'description' => t('Administer and configure CAPx mappers.'), + ), + 'capx advanced administrator' => array( + 'title' => t('Administer Advanced CAPx settings'), + 'description' => t('Administer and Ccnfigure CAPx advanced settings.'), ), ); } @@ -109,15 +121,6 @@ function stanford_capx_menu() { 'type' => MENU_NORMAL_ITEM, ); - // Connect page. - $items['admin/config/capx/connect'] = array( - 'title' => 'Connect', - 'page callback' => 'stanford_capx_admin_config_connect', - 'access arguments' => array('administer capx'), - 'type' => MENU_NORMAL_ITEM, - 'weight' => -10, - ); - // Settings Page. $items['admin/config/capx/settings'] = array( 'title' => 'Settings', diff --git a/stanford_capx.pages.inc b/stanford_capx.pages.inc index da96f8aa..474aa1e0 100644 --- a/stanford_capx.pages.inc +++ b/stanford_capx.pages.inc @@ -83,53 +83,36 @@ function stanford_capx_admin_config_settings() { drupal_set_message(t('The organization codes are not available. You should sync with the API now.'), 'warning', FALSE); } - // Organizations - // --------------------------------------------------------------------------- + $form = drupal_get_form("stanford_capx_admin_config_settings_get_all_the_forms_form"); - $form = drupal_get_form("stanford_capx_config_settings_data_sync_form"); - $output['content']["orgsform"] = $form; - - // Settings form - // --------------------------------------------------------------------------- - - $form = drupal_get_form('stanford_capx_config_settings_form'); - $output["content"]["settingsform"] = $form; + $output['content']['submitbuttons'] = $form; return $output; } /** - * [stanford_capx_config_settings_data_sync_form description] + * Returns all the forms that make up the settings page. * @param [type] $form [description] * @param [type] $form_state [description] * @return [type] [description] */ -function stanford_capx_config_settings_data_sync_form($form, $form_state) { - $form['orggroup'] = array( - '#type' => "fieldset", - '#title' => t("Organizations & Schema"), - '#description' => t("The CAPx module needs some information from the CAP API in order to function properly. Below are actions that require communication with the CAP API server and you must be connected before you can run these tasks."), - '#collapsible' => TRUE, - '#collapsed' => FALSE, - ); +function stanford_capx_admin_config_settings_get_all_the_forms_form($form, &$form_state) { + + // $form = system_settings_form($form); + $form = stanford_capx_config_auth_settings_form($form, $form_state); - // Organizations. - $form['orggroup']['orgcodeinfo']['#markup'] = "

Organizations

"; - $form['orggroup']['orgcodeinfo']['#markup'] .= "

" . t("Organization codes come from the CAP API and are saved to a taxonomy. This taxonomy powers the organization code autocomplete in the import section.") . "

"; - $form['orggroup']['orgcodeinfo']['#markup'] .= "

" . t("Last sync: @date", array("@date" => variable_get('capx_last_orgs_sync', 'never'))) . "

"; - $form['orggroup']['orgcodeinfo']['#markup'] .= "

" . t("View all organization codes: !link", array("!link" => l(t('Organization taxonomy'), 'admin/structure/taxonomy/capx_organizations'))) . "

"; - $form['orggroup']['orgcodeinfo']['#markup'] .= "

" . l(t('Get organization data'), 'admin/config/capx/organizations/sync', array("attributes" => array('class' => array('btn button')), 'query' => array('destination' => current_path()))) . "

"; - - // Schema. - $form['orggroup']['schemainfo']['#markup'] = "

Schema Information

"; - $form['orggroup']['schemainfo']['#markup'] .= "

" . t("The CAP API provides a schema of the information available. The CAPx module needs this in order to power the data browser that is found on a mapping page.") . "

"; - $form['orggroup']['schemainfo']['#markup'] .= "

" . t("Last sync: @date", array("@date" => variable_get('capx_last_schema_sync', 'never'))) . "

"; - $form['orggroup']['submit'] = array( + $connection = CAPxConnection::testConnection(); + if ($connection->status) { + $form = stanford_capx_config_settings_form($form, $form_state); + $form = stanford_capx_config_settings_data_sync_form($form, $form_state); + } + + + $form['actions']['submit'] = array( + '#value' => t("Save settings"), '#type' => 'submit', - '#value' => 'Get schema information', - '#op' => 'submit', ); - $form['#submit'][] = "stanford_capx_schema_refresh"; + return $form; } From 8548f38c226552f348d3d43e4c008abeb58ef902 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Wed, 25 May 2016 17:52:30 -0700 Subject: [PATCH 17/44] CAPX-213: Sync orgs on connect save --- stanford_capx.forms.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stanford_capx.forms.inc b/stanford_capx.forms.inc index d2bcb4d7..73e12954 100644 --- a/stanford_capx.forms.inc +++ b/stanford_capx.forms.inc @@ -64,7 +64,7 @@ function stanford_capx_forms_connect_form_submit($form, &$form_state) { // In order to get here we have to have a valid connection. // Lets get the org codes and schema. stanford_capx_schema_refresh(); - // stanford_capx_organizations_sync(); + stanford_capx_organizations_sync(); } From a314d85d083579dc6d1c62dc75af986745b58964 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Thu, 26 May 2016 09:26:10 -0700 Subject: [PATCH 18/44] stub --- stanford_capx.forms.inc | 1 - 1 file changed, 1 deletion(-) diff --git a/stanford_capx.forms.inc b/stanford_capx.forms.inc index 73e12954..c9b18c85 100644 --- a/stanford_capx.forms.inc +++ b/stanford_capx.forms.inc @@ -67,7 +67,6 @@ function stanford_capx_forms_connect_form_submit($form, &$form_state) { stanford_capx_organizations_sync(); } - // Endpoints. variable_set('stanford_capx_api_base_url', $form_state['values']['stanford_capx_api_base_url']); variable_set('stanford_capx_api_auth_uri', $form_state['values']['stanford_capx_api_auth_uri']); From dfc9ac799c49121af84c36641fa8bd1bfba29156 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Thu, 26 May 2016 09:28:45 -0700 Subject: [PATCH 19/44] HSDO-325: Remove required text on left side of field mapping and update help on subquery --- stanford_capx.forms.inc | 2 +- stanford_capx.theme.inc | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/stanford_capx.forms.inc b/stanford_capx.forms.inc index 11c05ecd..8a9d812d 100644 --- a/stanford_capx.forms.inc +++ b/stanford_capx.forms.inc @@ -324,7 +324,7 @@ function stanford_capx_mapper_form_get_entity_mapping_form($form, &$form_state, $form['entity-mapping']['subquery'] = array( '#type' => 'textfield', '#title' => t("Multiple entity creation sub query"), - '#description' => t('Which part of the schema will you be using to create multiple entities. Ie: for publications enter: $.publications'), + '#description' => t('Which part of the schema will you be using to create multiple entities. Ie: for publications enter: $.publications.*'), '#default_value' => isset($mapper->subquery) ? $mapper->subquery : '', '#states' => array( 'visible' => array( diff --git a/stanford_capx.theme.inc b/stanford_capx.theme.inc index 3c99cd0e..c21eeb18 100644 --- a/stanford_capx.theme.inc +++ b/stanford_capx.theme.inc @@ -53,7 +53,6 @@ function theme_stanford_capx_columns_to_the_right($elements) { $req = "" . t("TRUE") . ""; $element['#children'] = preg_replace("/\<\/label>/", " *$0", $element['#children']); } - $info .= "

Required: " . $req . "

"; // Set up the rows even though there will only be one. $rows = array(); From b8a472b239c590d8ff73a872d4f9a918313fbc93 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Thu, 26 May 2016 11:12:50 -0700 Subject: [PATCH 20/44] Ham. Hands. --- stanford_capx.install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stanford_capx.install b/stanford_capx.install index e631a1b9..95d54c73 100644 --- a/stanford_capx.install +++ b/stanford_capx.install @@ -21,7 +21,7 @@ function stanford_capx_uninstall() { variable_del('stanford_capx_token'); variable_del("capx_last_orgs_sync"); variable_del("capx_last_schema_sync"); - variable_det("stanford_capx_debug_always_force_etag"); + variable_del("stanford_capx_debug_always_force_etag"); } /** From 0ba36d7d9c585384bddeca35890710fe549ebdcb Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Thu, 26 May 2016 11:45:41 -0700 Subject: [PATCH 21/44] HSDO-325: PHP notice error fix on mapper form --- stanford_capx.forms.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stanford_capx.forms.inc b/stanford_capx.forms.inc index 8a9d812d..8aedab6a 100644 --- a/stanford_capx.forms.inc +++ b/stanford_capx.forms.inc @@ -673,7 +673,7 @@ function stanford_capx_mapper_form_get_fields_fields(&$form_tree, &$form_state, '#options' => $options, '#title' => t("Referencing Importer"), '#description' => t("Select the importer with the reference you would like to attach this item to."), - '#default_value' => $mapper->references[$field_machine_name][0], + '#default_value' => isset($mapper->references[$field_machine_name]) ? $mapper->references[$field_machine_name][0] : '', ); return; From 7437108146c9dac0a57cf5d988cdab65561c2e3a Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Thu, 26 May 2016 12:27:32 -0700 Subject: [PATCH 22/44] FCS: Fix up default value for field collection items in mapper form --- stanford_capx.forms.inc | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/stanford_capx.forms.inc b/stanford_capx.forms.inc index 8aedab6a..11cb0465 100644 --- a/stanford_capx.forms.inc +++ b/stanford_capx.forms.inc @@ -533,7 +533,6 @@ function stanford_capx_mapper_form_get_field_mapping_form($form, &$form_state, $ foreach ($field_collections as $entity_machine_name => $bundles) { foreach ($bundles as $bundle_machine_name => $fields) { foreach ($fields as $field_machine_name => $field_info) { - $instances = field_info_instances('field_collection_item', $field_machine_name); $collection_container = $entity_machine_name . "-" . $bundle_machine_name . "-collections"; @@ -555,7 +554,6 @@ function stanford_capx_mapper_form_get_field_mapping_form($form, &$form_state, $ foreach ($instances as $fc_field_name => $instance_info) { $fc_field_info = field_info_field($fc_field_name); - $form['field-mapping'][$collection_container][$field_machine_name][$fc_field_name] = array( '#type' => 'container', '#title' => t("@var", array("@var" => $instance_info['label'])), @@ -654,7 +652,7 @@ function stanford_capx_mapper_form_get_fields_fields(&$form_tree, &$form_state, '#default_value' => @$mapper->fields[$field_machine_name][0], ); - if ($field_instance_info['required']) { + if ($field_instance_info['required'] && empty($field_instance_info['default_value'])) { $form_tree['#attributes']['class'][] = "required"; } return; @@ -685,13 +683,25 @@ function stanford_capx_mapper_form_get_fields_fields(&$form_tree, &$form_state, foreach ($field_info_field['columns'] as $column_key => $column_info) { if (!in_array($column_key, $CAP_RESTRICTED_FIELD_PROPERTIES)) { + + // Get the default value for the field. If the field was in a field + // collection then we need to adjust. + $default_value = isset($mapper->fields[$field_machine_name][$column_key]) ? $mapper->fields[$field_machine_name][$column_key] : ""; + if (isset($field_info_field['bundles']['field_collection_item']) && !empty($field_info_field['bundles']['field_collection_item'])) { + if (!empty($mapper->collections[$field_machine_name][$field_info_field['field_name']][$column_key])) { + $default_value = $mapper->collections[$field_machine_name][$field_info_field['field_name']][$column_key]; + } + } + $form_tree[$column_key] = array( '#type' => 'textfield', '#title' => t("@var", array("@var" => $column_key)), '#description' => !empty($column_info['description']) ? t("@var", array("@var" => $column_info['description'])) : "", - '#default_value' => isset($mapper->fields[$field_machine_name][$column_key]) ? $mapper->fields[$field_machine_name][$column_key] : "", + '#default_value' => $default_value, ); - if ($field_instance_info['required']) { + + // Only require a field if it is required and there is no default value. + if ($field_instance_info['required'] && empty($field_instance_info['default_value'])) { $form_tree['#attributes']['class'][] = "required"; } } @@ -896,7 +906,6 @@ function stanford_capx_mapper_form_validate_required_fields(&$form, &$form_state // Get all the properties of the entity type. $entity_info = entity_get_property_info($entity); $properties = $entity_info['properties']; - $fields = isset($entity_info['bundles']) ? $entity_info['bundles'][$bundle]['properties'] : array(); // Properties validation. @@ -910,17 +919,31 @@ function stanford_capx_mapper_form_validate_required_fields(&$form, &$form_state // Field validation. foreach ($fields as $field_name => $value) { + // Check if the value is required. if (isset($value['required']) && $value['required']) { + // Check if the value exists. if (!isset($form_state['values']['field-mapping'][$entity . "-" . $bundle . "-fields"][$field_name])) { continue; } + // Check if there is a required class on the field as we add this to + // bypass the multiple hide/show options and required fields. + if (!isset($form['field-mapping'][$entity . "-" . $bundle . "-fields"]['#attributes']['class'])) { + continue; + } + + // No css class, no need to validate. + if (!in_array("required", $form['field-mapping'][$entity . "-" . $bundle . "-fields"]['#attributes']['class'])) { + continue; + } + + $columns = $form_state['values']['field-mapping'][$entity . "-" . $bundle . "-fields"][$field_name]; // Loop through the columns and check for values. foreach ($columns as $column_name => $value) { - if (empty($value) && $column_name != 'summary') { + if (empty($value) && $column_name !== 'summary') { $element = "field-mapping][" . $entity . "-" . $bundle . "-" . "fields][" . $field_name . "][" . $column_name; form_set_error($element, t("@name field is required", array("@name" => $field_name))); break; From 4bac731fdb08099b9ec8d749c4ae6269c95c67b2 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Thu, 26 May 2016 18:27:06 -0700 Subject: [PATCH 23/44] FCS: Proof of concept loop --- .../Processors/FieldCollectionProcessor.php | 52 ++++++++++++++----- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/includes/CAPx/Drupal/Processors/FieldCollectionProcessor.php b/includes/CAPx/Drupal/Processors/FieldCollectionProcessor.php index 9727201c..e426ebe4 100644 --- a/includes/CAPx/Drupal/Processors/FieldCollectionProcessor.php +++ b/includes/CAPx/Drupal/Processors/FieldCollectionProcessor.php @@ -12,7 +12,7 @@ class FieldCollectionProcessor extends EntityProcessor { // The field collection entity - protected $fieldCollectionEntity = null; + protected $fieldCollectionEntity = array(); // The parent entity protected $parentEntity = null; @@ -30,8 +30,22 @@ public function execute() { $entityType = $mapper->getEntityType(); $bundleType = $mapper->getBundleType(); - $entity = $this->newEntity($entityType, $bundleType, $data, $mapper); - return $entity; + // Use this funciton to find out how many items we really need to create. + $dataPile = $this->fetchDataPile($data, $mapper); + + // Loop through an create new field collections based on the number of + // results for each field. + foreach ($dataPile as $fcData) { + $entity = $this->newEntity($entityType, $bundleType, $fcdData, $mapper); + drupal_alter('capx_new_fc', $entity); + $entity = $mapper->execute($entity, $data); + $entity->save(); + // Storage for something that may need it. + $this->addFieldCollectionEntity($entity); + } + + // Return all the things we just created. + return $this->getEntity(); } @@ -59,24 +73,38 @@ public function newEntity($entityType, $bundleType, $data, $mapper) { // Wrap it up baby! $entity = entity_metadata_wrapper($entityType, $entity); - $entity = $mapper->execute($entity, $data); - $entity->save(); + return $entity; + } - drupal_alter('capx_new_fc', $entity); + /** + * Return a multidimensional array of result data + * + * @param $data + * @param $mapper + * @return array + */ + protected function fetchDataPile($data, $mapper) { + $pile = array(); + $pile = array_pad($pile, 10, $data); + return $pile; + } - // Storage for something that may need it. - $this->setFieldCollectionEntity($entity); - return $entity; + /** + * Setter function + * @param Array $entities an array of field collection items + */ + protected function addFieldCollectionEntity($entity) { + $this->fieldCollectionEntity[] = $entity; } /** * Setter function - * @param FieldCollectionItem $entity the field collection item to be acted on + * @param Array $entities an array of field collection items */ - public function setFieldCollectionEntity($entity) { - $this->fieldCollectionEntity = $entity; + public function setFieldCollectionEntity($entities) { + $this->fieldCollectionEntity = $entities; } /** From 8a6d86f62fa9c22827444d082f4a21cd21a2e20a Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Fri, 27 May 2016 10:28:27 -0700 Subject: [PATCH 24/44] stash --- .../Processors/FieldCollectionProcessor.php | 19 +++++++++---------- .../Drupal/Processors/ProcessorAbstract.php | 6 ++++-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/includes/CAPx/Drupal/Processors/FieldCollectionProcessor.php b/includes/CAPx/Drupal/Processors/FieldCollectionProcessor.php index e426ebe4..39c0c051 100644 --- a/includes/CAPx/Drupal/Processors/FieldCollectionProcessor.php +++ b/includes/CAPx/Drupal/Processors/FieldCollectionProcessor.php @@ -12,7 +12,7 @@ class FieldCollectionProcessor extends EntityProcessor { // The field collection entity - protected $fieldCollectionEntity = array(); + protected $fieldCollectionEntities = array(); // The parent entity protected $parentEntity = null; @@ -27,7 +27,7 @@ class FieldCollectionProcessor extends EntityProcessor { public function execute() { $data = $this->getData(); $mapper = $this->getMapper(); - $entityType = $mapper->getEntityType(); + $entityType = $mapper->getEntitiesType(); $bundleType = $mapper->getBundleType(); // Use this funciton to find out how many items we really need to create. @@ -45,26 +45,25 @@ public function execute() { } // Return all the things we just created. - return $this->getEntity(); + return $this->getEntities(); } - /** * New entity override as FieldCollections have some different defualts. - * @see parent:newEntity(); + * @see parent:newEntity(); */ public function newEntity($entityType, $bundleType, $data, $mapper) { $properties = array( 'type' => $bundleType, - 'uid' => 1, // @TODO - set this to something else - 'status' => 1, // @TODO - allow this to change - 'comment' => 0, // Any reason to set otherwise? + 'uid' => 1, // @TODO - set this to something else. + 'status' => 1, // @TODO - allow this to change. + 'comment' => 0, // Any reason to set otherwise?. 'promote' => 0, // Fogetaboutit. 'field_name' => $bundleType, ); - // Create an empty entity + // Create an empty entity. $entity = entity_create($entityType, $properties); $hostEntity = $this->getParentEntity(); @@ -77,7 +76,7 @@ public function newEntity($entityType, $bundleType, $data, $mapper) { } /** - * Return a multidimensional array of result data + * Return a multidimensional array of result data. * * @param $data * @param $mapper diff --git a/includes/CAPx/Drupal/Processors/ProcessorAbstract.php b/includes/CAPx/Drupal/Processors/ProcessorAbstract.php index ed24a2bf..526f6918 100644 --- a/includes/CAPx/Drupal/Processors/ProcessorAbstract.php +++ b/includes/CAPx/Drupal/Processors/ProcessorAbstract.php @@ -17,8 +17,10 @@ abstract class ProcessorAbstract implements ProcessorInterface { /** * construction method - * @param EntityMapper $mapper EntityMapper - * @param Array $capData an array of cap data + * @param $mapper EntityMapper + * The mapper. + * @param $data array + * An array of cap data */ public function __construct($mapper, $data) { $this->setMapper($mapper); From 11c52940f3339252aa8e80aac4c879873722283f Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Fri, 27 May 2016 12:46:41 -0700 Subject: [PATCH 25/44] Working field collection items --- includes/CAPx/Drupal/Mapper/EntityMapper.php | 9 +++ .../Processors/FieldCollectionProcessor.php | 76 ++++++++++++------- 2 files changed, 58 insertions(+), 27 deletions(-) diff --git a/includes/CAPx/Drupal/Mapper/EntityMapper.php b/includes/CAPx/Drupal/Mapper/EntityMapper.php index f152db85..cbd7d300 100644 --- a/includes/CAPx/Drupal/Mapper/EntityMapper.php +++ b/includes/CAPx/Drupal/Mapper/EntityMapper.php @@ -556,6 +556,15 @@ public function isMultiple() { } } + /** + * Sets the multiple config value to passed in. + * @param $val bool + * Boolean for multiple or not + */ + public function setIsMultiple($val = TRUE) { + $this->config['multiple'] = $val; + } + /** * get Subquery wrapper */ diff --git a/includes/CAPx/Drupal/Processors/FieldCollectionProcessor.php b/includes/CAPx/Drupal/Processors/FieldCollectionProcessor.php index 39c0c051..9e60f64d 100644 --- a/includes/CAPx/Drupal/Processors/FieldCollectionProcessor.php +++ b/includes/CAPx/Drupal/Processors/FieldCollectionProcessor.php @@ -27,25 +27,61 @@ class FieldCollectionProcessor extends EntityProcessor { public function execute() { $data = $this->getData(); $mapper = $this->getMapper(); - $entityType = $mapper->getEntitiesType(); + $entityType = $mapper->getEntityType(); $bundleType = $mapper->getBundleType(); - - // Use this funciton to find out how many items we really need to create. - $dataPile = $this->fetchDataPile($data, $mapper); + $parent = $this->getParentEntity(); // Loop through an create new field collections based on the number of - // results for each field. - foreach ($dataPile as $fcData) { - $entity = $this->newEntity($entityType, $bundleType, $fcdData, $mapper); + // results for each field. Keep looping through the index of data until + // there is no more data to add to the entity. We can accomplish this by + // checking to see if the last FC that was created is exactly the same as + // the one just created. If they are identical then no new data is available + // and the loop should be stopped. + + $mapper->setIsMultiple(true); + $lastEntity = NULL; + + $i = 0; + $hasValues = TRUE; + while($hasValues) { + $mapper->setIndex($i); + $entity = $this->newEntity($entityType, $bundleType, $data, $mapper); drupal_alter('capx_new_fc', $entity); $entity = $mapper->execute($entity, $data); + + // Here we check to see if anything came out the other end. + $hash = md5(serialize($entity)); + if ($hash == $lastEntity) { + $hasValues = FALSE; + break; + } + + // Not the same. Store the current entity as the last entity before saving + // or we will pollute the object with ids. + $lastEntity = $hash; + + // Save. $entity->save(); + // Storage for something that may need it. $this->addFieldCollectionEntity($entity); + + // Add to index for next one. + $i++; } + // Remove the last two items from the entities array and from the parent + // entity as they are full of duplicate or not complete data. + $fieldName = $mapper->getBundleType(); + array_pop($this->fieldCollectionEntities); + array_pop($this->fieldCollectionEntities); + $rawParent = $parent->raw(); + array_pop($rawParent->{$fieldName}[LANGUAGE_NONE]); + array_pop($rawParent->{$fieldName}[LANGUAGE_NONE]); + $parent->set($rawParent); + // Return all the things we just created. - return $this->getEntities(); + return $this->getFieldCollectionEntities(); } /** @@ -75,26 +111,12 @@ public function newEntity($entityType, $bundleType, $data, $mapper) { return $entity; } - /** - * Return a multidimensional array of result data. - * - * @param $data - * @param $mapper - * @return array - */ - protected function fetchDataPile($data, $mapper) { - $pile = array(); - $pile = array_pad($pile, 10, $data); - return $pile; - } - - /** * Setter function * @param Array $entities an array of field collection items */ protected function addFieldCollectionEntity($entity) { - $this->fieldCollectionEntity[] = $entity; + $this->fieldCollectionEntities[] = $entity; } @@ -102,16 +124,16 @@ protected function addFieldCollectionEntity($entity) { * Setter function * @param Array $entities an array of field collection items */ - public function setFieldCollectionEntity($entities) { - $this->fieldCollectionEntity = $entities; + public function setFieldCollectionEntities($entities) { + $this->fieldCollectionEntities = $entities; } /** * Getter function * @return FieldCollectionItem the field collection item. */ - public function getFieldCollectionEntity() { - return $this->fieldCollectionEntity; + public function getFieldCollectionEntities() { + return $this->fieldCollectionEntities; } /** From ed439de2f80fcc016f1cd41d227b3bffe67ac9b1 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Fri, 27 May 2016 12:53:14 -0700 Subject: [PATCH 26/44] CAPX-2.x start --- CHANGELOG.txt | 3 +++ README.md | 2 +- stanford_capx.info | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index fa784526..0ce3af27 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,9 @@ Stanford CAPx x.x-x.x, xxxx-xx-xx --------------------------------- +Stanford CAPx 7.x-2.0, 2016-05-27 +--------------------------------- + Stanford CAPx 7.x-1.3, 2016-02-05 --------------------------------- diff --git a/README.md b/README.md index 57b72257..229b76d6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Stanford CAPx -#### Version 1.3 +#### Version 2.x Stanford CAP Extensible module builds on some great work. This module provides an interface for administrators to pull information directly from the CAP API into Drupal. This allows profile owners to continue to manage their profile information on the CAP web service and have that information automatically reflected into a Drupal website. diff --git a/stanford_capx.info b/stanford_capx.info index bab90fea..a85a3c23 100644 --- a/stanford_capx.info +++ b/stanford_capx.info @@ -3,7 +3,7 @@ description = Provides the ability to import profiles into existing entity types core = 7.x package = Stanford CAPx project = stanford_capx -version = 7.x-1.3 +version = 7.x-2.0-dev dependencies[] = entity dependencies[] = block From 4d3c1178eccbd09d143fd3182f440a880d6a08bc Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Fri, 27 May 2016 12:58:11 -0700 Subject: [PATCH 27/44] CAPX-2.x changelog.txt --- CHANGELOG.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 0ce3af27..3b44f2e1 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -3,6 +3,14 @@ Stanford CAPx x.x-x.x, xxxx-xx-xx Stanford CAPx 7.x-2.0, 2016-05-27 --------------------------------- +- Entity relationships! +- Multiple entity creation (IE: Publications). +- Multiple field collections. +- Merged connect and settings form. +- Maintains index association on multiple item creation and wildcard json queries. +- Bug fix for field collection item mapping values that went missing after save. +- + Stanford CAPx 7.x-1.3, 2016-02-05 --------------------------------- From 6aeaa689dc461aa713d4069eb57f6abf4f7c6a50 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Fri, 27 May 2016 15:20:53 -0700 Subject: [PATCH 28/44] CAPX-213: Removed importer and mapper permissions for now --- CHANGELOG.txt | 2 +- stanford_capx.module | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 3b44f2e1..6f17fb03 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -9,7 +9,7 @@ Stanford CAPx 7.x-2.0, 2016-05-27 - Merged connect and settings form. - Maintains index association on multiple item creation and wildcard json queries. - Bug fix for field collection item mapping values that went missing after save. -- +- Added several new permissions for more granular control over the UI Stanford CAPx 7.x-1.3, 2016-02-05 diff --git a/stanford_capx.module b/stanford_capx.module index e74dba39..77df8993 100644 --- a/stanford_capx.module +++ b/stanford_capx.module @@ -71,14 +71,14 @@ function stanford_capx_permission() { 'title' => t('Administer CAPx'), 'description' => t('Administer and configure CAPx settings.'), ), - 'capx administer importers' => array( - 'title' => t('Administer CAPx Importers'), - 'description' => t('Administer and configure CAPx importers.'), - ), - 'capx administer mappers' => array( - 'title' => t('Administer CAPx Mappers'), - 'description' => t('Administer and configure CAPx mappers.'), - ), + // 'capx administer importers' => array( + // 'title' => t('Administer CAPx Importers'), + // 'description' => t('Administer and configure CAPx importers.'), + // ), + // 'capx administer mappers' => array( + // 'title' => t('Administer CAPx Mappers'), + // 'description' => t('Administer and configure CAPx mappers.'), + // ), 'capx advanced administrator' => array( 'title' => t('Administer Advanced CAPx settings'), 'description' => t('Administer and Ccnfigure CAPx advanced settings.'), From a8768a473f4b3edec496b6df2994420fbaadf2b7 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Fri, 27 May 2016 15:38:01 -0700 Subject: [PATCH 29/44] Updated changelog.txt --- CHANGELOG.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 6f17fb03..9dc37b02 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -9,8 +9,9 @@ Stanford CAPx 7.x-2.0, 2016-05-27 - Merged connect and settings form. - Maintains index association on multiple item creation and wildcard json queries. - Bug fix for field collection item mapping values that went missing after save. -- Added several new permissions for more granular control over the UI - +- Added several new permissions for more granular control over the +- Removed superfluous required text on field mapping +- Required fields with a default value are no longer required in the mapping. Stanford CAPx 7.x-1.3, 2016-02-05 --------------------------------- From 4d7e01671cb9e2b92abe5a0c5824daa6b326927d Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Mon, 30 May 2016 08:14:51 -0700 Subject: [PATCH 30/44] Set the stanford_capx_autotruncate_textfields value default to TRUE --- CHANGELOG.txt | 3 +-- .../Drupal/Processors/FieldProcessors/LinkFieldProcessor.php | 2 +- .../PropertyProcessors/PropertyProcessorAbstract.php | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 9dc37b02..423a7e85 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -12,10 +12,10 @@ Stanford CAPx 7.x-2.0, 2016-05-27 - Added several new permissions for more granular control over the - Removed superfluous required text on field mapping - Required fields with a default value are no longer required in the mapping. +- Added auto truncating of text fields in order to avoid the PDO exception error of value too long. Stanford CAPx 7.x-1.3, 2016-02-05 --------------------------------- - - Bugfix: Removed hook_install views save and install time check for private files dir. - Bugfix: Reduced duplicate warning messages. - Bugfix: Fix UI for field collection mapping where it was only showing one. @@ -33,7 +33,6 @@ Stanford CAPx 7.x-1.3, 2016-02-05 Stanford CAPx 7.x-1.2, 2015-09-10 --------------------------------- - - Upgraded HTTP Client from Guzzle 3.7.4 -> 6.0.2 - User entities when created from CAP now get a random password assigned to them. - Fixed permission issues with the profiles list view diff --git a/includes/CAPx/Drupal/Processors/FieldProcessors/LinkFieldProcessor.php b/includes/CAPx/Drupal/Processors/FieldProcessors/LinkFieldProcessor.php index 8fb40373..6f3c353a 100644 --- a/includes/CAPx/Drupal/Processors/FieldProcessors/LinkFieldProcessor.php +++ b/includes/CAPx/Drupal/Processors/FieldProcessors/LinkFieldProcessor.php @@ -40,7 +40,7 @@ public function prepareData($data) { // If autotruncating is enabled lets do that here. This is to help // avoid issues when trying to map data to the API. - if (variable_get("stanford_capx_autotruncate_textfields", FALSE)) { + if (variable_get("stanford_capx_autotruncate_textfields", TRUE)) { $maxlength = 2048; // Default for url. diff --git a/includes/CAPx/Drupal/Processors/PropertyProcessors/PropertyProcessorAbstract.php b/includes/CAPx/Drupal/Processors/PropertyProcessors/PropertyProcessorAbstract.php index 2fe5bd69..4531deab 100644 --- a/includes/CAPx/Drupal/Processors/PropertyProcessors/PropertyProcessorAbstract.php +++ b/includes/CAPx/Drupal/Processors/PropertyProcessors/PropertyProcessorAbstract.php @@ -44,7 +44,7 @@ public function put($data) { // If autotruncating is enabled lets do that here. This is to help // avoid issues when trying to map data to the API. - if (variable_get("stanford_capx_autotruncate_textfields", FALSE)) { + if (variable_get("stanford_capx_autotruncate_textfields", TRUE)) { $maxlength = 255; // Default. if (strlen($data) > $maxlength) { $data = substr($data, 0, $maxlength); From c531cfa57cbda4e739f01ef0f0d87d8de7b70f31 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Mon, 30 May 2016 08:15:29 -0700 Subject: [PATCH 31/44] added variable_del for stanford_capx_autotruncate_textfields to .install uninstall --- stanford_capx.install | 1 + 1 file changed, 1 insertion(+) diff --git a/stanford_capx.install b/stanford_capx.install index 95d54c73..4fef21b5 100644 --- a/stanford_capx.install +++ b/stanford_capx.install @@ -22,6 +22,7 @@ function stanford_capx_uninstall() { variable_del("capx_last_orgs_sync"); variable_del("capx_last_schema_sync"); variable_del("stanford_capx_debug_always_force_etag"); + variable_del("stanford_capx_autotruncate_textfields"); } /** From dced0291d13c5131ed61b0f61dd634e2d995faa1 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Mon, 30 May 2016 08:24:27 -0700 Subject: [PATCH 32/44] Added a few code comments --- .../CAPx/Drupal/Importer/Orphans/EntityImporterOrphans.php | 4 ++-- includes/CAPx/Drupal/Processors/EntityProcessor.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/includes/CAPx/Drupal/Importer/Orphans/EntityImporterOrphans.php b/includes/CAPx/Drupal/Importer/Orphans/EntityImporterOrphans.php index d33fca72..f453e7a2 100644 --- a/includes/CAPx/Drupal/Importer/Orphans/EntityImporterOrphans.php +++ b/includes/CAPx/Drupal/Importer/Orphans/EntityImporterOrphans.php @@ -356,16 +356,16 @@ protected function processMultipleImporterOrphans($entityIds) { $profile = entity_metadata_wrapper($entityType, $entity[$entityId]); switch ($action) { + // Any entity can be deleted. case 'delete': $profile->delete(); break; + // Users and nodes can have their status set to 0. case 'block': case 'unpublish': $profile->status->set(0); $profile->save(); - - // Log that this profile was orphaned. $this->logOrphan($profile); break; diff --git a/includes/CAPx/Drupal/Processors/EntityProcessor.php b/includes/CAPx/Drupal/Processors/EntityProcessor.php index 8234a87f..f40d64cc 100644 --- a/includes/CAPx/Drupal/Processors/EntityProcessor.php +++ b/includes/CAPx/Drupal/Processors/EntityProcessor.php @@ -318,7 +318,7 @@ public function updateEntity($entity, $data, $mapper) { * @param object $mapper * The EntityMapper instance * @param mixed $guuid - * The + * The genuine unique id for this entity of other than profileId. * * @return object * The new entity after it has been saved. From 76467b8b2a1b77ce256035191caabe598392d654 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Tue, 31 May 2016 09:15:37 -0700 Subject: [PATCH 33/44] CAPX-215: Change the label for the multiple entities checkbox --- stanford_capx.forms.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stanford_capx.forms.inc b/stanford_capx.forms.inc index 7f8ab043..dd450c6a 100644 --- a/stanford_capx.forms.inc +++ b/stanford_capx.forms.inc @@ -209,7 +209,7 @@ function stanford_capx_mapper_form_get_entity_mapping_form($form, &$form_state, $form['entity-mapping']['multiple-entities'] = array( '#type' => 'checkbox', - '#title' => t("Create multiple entities"), + '#title' => t("Would you like to create multiple entities per imported bundle for the type of content you're importing?"), '#description' => t('By default, a mapper will create one entity per person. If you wanted to import a persons publications you would need multiple entities per person. Check this box for multiple.'), '#default_value' => isset($mapper->multiple) ? $mapper->multiple : FALSE, ); From bd945d0e3b57c1ee831da1dff0fe0e482066da62 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Fri, 17 Jun 2016 13:00:48 -0700 Subject: [PATCH 34/44] Fixing up some php errors --- .../Drupal/Processors/EntityProcessor.php | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/includes/CAPx/Drupal/Processors/EntityProcessor.php b/includes/CAPx/Drupal/Processors/EntityProcessor.php index f40d64cc..eaa1f33b 100644 --- a/includes/CAPx/Drupal/Processors/EntityProcessor.php +++ b/includes/CAPx/Drupal/Processors/EntityProcessor.php @@ -44,10 +44,10 @@ public function execute($force = FALSE) { // Sometimes we need to create one, other times we need moar. if (!empty($multi) && $multi == 1) { - $entity = $this->executeMultiple(); + $entity = $this->executeMultiple($force); } else { - $entity = $this->executeSingle(); + $entity = $this->executeSingle($force); } return $entity; @@ -58,7 +58,7 @@ public function execute($force = FALSE) { * @param [type] $force [description] * @return [type] [description] */ - protected function executeMultiple() { + protected function executeMultiple($force = FALSE) { $mapper = $this->getMapper(); $data = $this->getData(); @@ -117,14 +117,6 @@ protected function executeMultiple() { // GUUID available lets check to see if it matches the numEntities count. $numGUUIDs = count($mapper->getRemoteDataByJsonPath($data, $guuidPath)); - // If the total number of ids does not match the total number of entities - // default to the delete and replace method. - // if ($numGUUIDs !== $numEntities) { - // watchdog('EntityProcessor', "Entity count and GUUID mismatch on: %mapper. Defaulting to delete and replace update method.", array("%mapper" => $mapper->getMachineName())); - // $this->multipleDeleteEntities($entities); - // $this->multipleCreateNewEntity($numEntities, $entityType, $bundleType, $data, $mapper); - // } - // Ok, we are in good shape at this point. We have results, we have ids, and // now we can update them in place! $this->multipleUpdateEntities($numEntities, $entityType, $bundleType, $data, $mapper); @@ -132,7 +124,7 @@ protected function executeMultiple() { } /** - * [multipleCreateNewEntity description] + * Create a bunch of new entities! * @param [type] $numEntities [description] * @param [type] $entityType [description] * @param [type] $data [description] @@ -198,7 +190,7 @@ protected function multipleUpdateEntities($numEntities, $entityType, $bundleType * @param [type] $force [description] * @return [type] [description] */ - protected function executeSingle($force) { + protected function executeSingle($force = FALSE) { $mapper = $this->getMapper(); $data = $this->getData(); $entityImporter = $this->getEntityImporter(); @@ -323,7 +315,7 @@ public function updateEntity($entity, $data, $mapper) { * @return object * The new entity after it has been saved. */ - public function newEntity($entityType, $bundleType, $data, $mapper, $guuid) { + public function newEntity($entityType, $bundleType, $data, $mapper, $guuid = NULL) { $properties = array( 'type' => $bundleType, From ea447e6e27b7e64368a8ab74f52e8099147fb2f2 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Fri, 17 Jun 2016 13:05:45 -0700 Subject: [PATCH 35/44] Moar PHP error fixes --- .../Processors/EntityReferenceProcessor.php | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/includes/CAPx/Drupal/Processors/EntityReferenceProcessor.php b/includes/CAPx/Drupal/Processors/EntityReferenceProcessor.php index 286097e1..0bfe17b1 100644 --- a/includes/CAPx/Drupal/Processors/EntityReferenceProcessor.php +++ b/includes/CAPx/Drupal/Processors/EntityReferenceProcessor.php @@ -9,6 +9,9 @@ use CAPx\Drupal\Mapper\EntityMapper; use CAPx\Drupal\Util\CAPx; +/** + * Entity references are a bit different than normal. + */ class EntityReferenceProcessor { protected $entity; @@ -17,20 +20,25 @@ class EntityReferenceProcessor { /** * Creates an entityReferenceProcessor to handle entity reference fields. - * @param [type] $entity [description] - * @param [type] $importer [description] - * @param [type] $target [description] + * + * @param object $entity + * The entity we are currently working on to save. + * @param object $importer + * The importer object and all its glory. + * @param string $target + * The target importer where the relative entity lives. */ public function __construct($entity, $importer, $target) { $this->entity = $entity; $this->importer = $importer; - $this->fieldName = $fieldName; $this->target = $target; } /** * Returns a list of possible matches. - * @return [type] [description] + * + * @return mixed + * An empty array if no matches or a fully loaded entity. */ public function execute() { @@ -57,7 +65,7 @@ public function execute() { return array(); } - // try to load it. + // Try to load it. $entity = entity_load_single($match['entity_type'], $match['entity_id']); // Return the result. From a0a9869dc5a010a43192fa6bead5a56afc67b891 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Fri, 17 Jun 2016 16:55:13 -0700 Subject: [PATCH 36/44] Set CAPx to 2.x -dev branch --- .../Processors/FieldProcessors/FieldProcessorAbstract.php | 8 +++++++- includes/CAPx/Drupal/Util/CAPx.php | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/includes/CAPx/Drupal/Processors/FieldProcessors/FieldProcessorAbstract.php b/includes/CAPx/Drupal/Processors/FieldProcessors/FieldProcessorAbstract.php index 1199d470..e965ca9d 100644 --- a/includes/CAPx/Drupal/Processors/FieldProcessors/FieldProcessorAbstract.php +++ b/includes/CAPx/Drupal/Processors/FieldProcessors/FieldProcessorAbstract.php @@ -86,7 +86,13 @@ public function put($data) { // Value wrapper assumes we providing single value. case 'EntityDrupalWrapper': case 'EntityValueWrapper': - $field->set(array_shift($data)); + + // Only Shift if it is an array. + if (is_array($data)) { + $data = array_shift($data); + } + + $field->set($data); break; } } diff --git a/includes/CAPx/Drupal/Util/CAPx.php b/includes/CAPx/Drupal/Util/CAPx.php index 957fa706..13c8b07f 100644 --- a/includes/CAPx/Drupal/Util/CAPx.php +++ b/includes/CAPx/Drupal/Util/CAPx.php @@ -215,7 +215,13 @@ public static function getEntityIdByGUUID($importerMachineName, $profileId, $guu ->execute() ->fetchAssoc(); - return isset($query['entity_id']) ? array_pop(entity_load($query['entity_type'], array($query['entity_id']))) : FALSE; + // If we gots one send it back. + if (isset($query['entity_id'])) { + $entities = entity_load($query['entity_type'], array($query['entity_id'])); + return array_pop($entities); + } + + return FALSE; } /** From cee6e93ecbbdf4a42f303f898215d4e2e5aa12b8 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Wed, 22 Jun 2016 16:20:03 -0700 Subject: [PATCH 37/44] Changed shcema file paths to play better with S3 --- stanford_capx.docs.inc | 2 +- stanford_capx.module | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/stanford_capx.docs.inc b/stanford_capx.docs.inc index 292783e5..60854670 100644 --- a/stanford_capx.docs.inc +++ b/stanford_capx.docs.inc @@ -90,7 +90,7 @@ function stanford_capx_admin_config_data_browser_refresh_submit($form, &$form_st */ function stanford_capx_admin_data_browser_schema_flush() { $files = array( - 'schema_file' => STANFORD_CAPX_FILE_PATH . STANFORD_CAPX_SCHEMA_FILE, + 'schema_file' => STANFORD_CAPX_FILE_PATH . "/" . STANFORD_CAPX_SCHEMA_FILE, 'sample_data_file' => STANFORD_CAPX_FILE_PATH . STANFORD_CAPX_SAMPLE_PROFILE, ); diff --git a/stanford_capx.module b/stanford_capx.module index 77df8993..2dc87315 100644 --- a/stanford_capx.module +++ b/stanford_capx.module @@ -49,7 +49,7 @@ $CAP_RESTRICTED_FIELD_PROPERTIES = array( 'attributes', ); -define('STANFORD_CAPX_FILE_PATH', 'private://stanford_capx/'); +define('STANFORD_CAPX_FILE_PATH', 'private://stanford_capx'); define('STANFORD_CAPX_SCHEMA_FILE', 'schema.json'); /** @@ -512,7 +512,7 @@ function stanford_capx_schema_load($format = 'json', $reset = FALSE) { return FALSE; } - $file = STANFORD_CAPX_FILE_PATH . STANFORD_CAPX_SCHEMA_FILE; + $file = STANFORD_CAPX_FILE_PATH . "/" . STANFORD_CAPX_SCHEMA_FILE; // @TODO: Refresh automatically under some condition? Weekly? // If forcing a refesh or the schema hasn't been downloaded, download it! @@ -543,7 +543,7 @@ function stanford_capx_schema_load($format = 'json', $reset = FALSE) { * FALSE if failed or a json string if success. */ function stanford_capx_schema_refresh() { - $file = STANFORD_CAPX_FILE_PATH . STANFORD_CAPX_SCHEMA_FILE; + $file = STANFORD_CAPX_FILE_PATH . "/" . STANFORD_CAPX_SCHEMA_FILE; // Ensure the modules' private files directory exists and is writable. if (stanford_capx_files_dir_exists()) { From b8897005e9599b2a742c1e6db5413d35179df980 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Wed, 22 Jun 2016 16:48:32 -0700 Subject: [PATCH 38/44] Updated path for sample data schema --- stanford_capx.docs.inc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stanford_capx.docs.inc b/stanford_capx.docs.inc index 60854670..7e59ea6a 100644 --- a/stanford_capx.docs.inc +++ b/stanford_capx.docs.inc @@ -91,7 +91,7 @@ function stanford_capx_admin_config_data_browser_refresh_submit($form, &$form_st function stanford_capx_admin_data_browser_schema_flush() { $files = array( 'schema_file' => STANFORD_CAPX_FILE_PATH . "/" . STANFORD_CAPX_SCHEMA_FILE, - 'sample_data_file' => STANFORD_CAPX_FILE_PATH . STANFORD_CAPX_SAMPLE_PROFILE, + 'sample_data_file' => STANFORD_CAPX_FILE_PATH . "/" . STANFORD_CAPX_SAMPLE_PROFILE, ); // Delete the cached files, which will be re-fetched on next page reload. @@ -252,13 +252,13 @@ function stanford_capx_sample_data_render($sample_data) { function stanford_capx_sample_data($selector) { static $sample_profile; static $json_parser; - $file = STANFORD_CAPX_FILE_PATH . STANFORD_CAPX_SAMPLE_PROFILE; + $file = STANFORD_CAPX_FILE_PATH . "/" . STANFORD_CAPX_SAMPLE_PROFILE; // Load and static cache the sample data file. if (!isset($sample_profile)) { if (file_exists($file)) { // Load sample data from the file cache. - $json = file_get_contents(STANFORD_CAPX_FILE_PATH . STANFORD_CAPX_SAMPLE_PROFILE); + $json = file_get_contents(STANFORD_CAPX_FILE_PATH . "/" . STANFORD_CAPX_SAMPLE_PROFILE); } else { // Fetch the profile from CAP and save to file cache. From 0ab01c7e4b7e1db305d7565159e8c7c588b82305 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Thu, 23 Jun 2016 11:21:06 -0700 Subject: [PATCH 39/44] Added a quick timeout to API check --- includes/CAPx/Drupal/Util/CAPxConnection.php | 4 ++++ stanford_capx.blocks.inc | 2 +- stanford_capx.pages.inc | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/includes/CAPx/Drupal/Util/CAPxConnection.php b/includes/CAPx/Drupal/Util/CAPxConnection.php index f4685c3f..406c6ab7 100644 --- a/includes/CAPx/Drupal/Util/CAPxConnection.php +++ b/includes/CAPx/Drupal/Util/CAPxConnection.php @@ -116,6 +116,10 @@ public static function testApiConnection($token = null, $endpoint = null) { $client = new HTTPClient(); $client->setEndpoint($endpoint); $client->setApiToken($token); + $opts = $client->getHttpOptions(); + $opts['connect_timeout'] = 2.00; + $opts['timeout'] = 5.00; + $client->setHttpOptions($opts); try { $results = $client->api('orgs')->getOrg('BSWS'); diff --git a/stanford_capx.blocks.inc b/stanford_capx.blocks.inc index baa87a24..b758dc5e 100644 --- a/stanford_capx.blocks.inc +++ b/stanford_capx.blocks.inc @@ -153,7 +153,7 @@ function stanford_capx_connection_status_block() { $content .= '

' . t('Not Connected') . '

'; if ($connection->code == 0) { - $content .= t("Please go to the !connt page to connect to the CAP API.", array("!connt" => l(t("CAPx connection"), "admin/config/capx/connect"))); + $content .= t("Please go to the !connt page to connect to the CAP API.", array("!connt" => l(t("CAPx settings"), "admin/config/capx/settings"))); } else { $vars = array('@code' => $connection->code, '@message' => $connection->message); diff --git a/stanford_capx.pages.inc b/stanford_capx.pages.inc index 474aa1e0..6f0a14af 100644 --- a/stanford_capx.pages.inc +++ b/stanford_capx.pages.inc @@ -150,7 +150,7 @@ function stanford_capx_admin_config_help() { $content = "

" . t("Getting started") . "

"; $content .= "

" . t("Importing content from CAP can be completed in 3 steps:") . "

"; $content .= "
  1. "; - $content .= t("!connect - Connect to the CAP API using your authentication credentials.", array("!connect" => l(t("Connect"), "admin/config/capx/connect"))); + $content .= t("!connect - Connect to the CAP API using your authentication credentials.", array("!connect" => l(t("Settings"), "admin/config/capx/settings"))); $content .= "
  2. "; $content .= t("!mapping - Create a mapping that links CAP fields to your own fields.", array("!mapping" => l(t("Mapping"), "admin/config/capx/mapper"))); $content .= "
  3. "; From c97b31db7416f87a04c62edb380c4ba7401d4cab Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Tue, 12 Jul 2016 11:50:07 -0700 Subject: [PATCH 40/44] Added composer to branch --- .gitignore | 1 + composer.json | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 .gitignore create mode 100644 composer.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..57872d0f --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/vendor/ diff --git a/composer.json b/composer.json new file mode 100644 index 00000000..c6e1e17e --- /dev/null +++ b/composer.json @@ -0,0 +1,17 @@ +{ + "name": "su-sws/stanford_capx", + "description": "Stanford Career Academic Profiles API integration Module", + "type": "stanford-module", + "license": "GPL-2.0+", + "authors": [ + { + "name": "Shea McKinney", + "email": "sheamck@stanford.edu" + } + ], + "require": {} + { + "homepage": "https://github.com/SU-SWS/stanford_capx", +} + +} From e32f2e33673ba231e8b30b8ab58ffdc0ff932896 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Tue, 12 Jul 2016 11:51:11 -0700 Subject: [PATCH 41/44] Fixed up composer.json --- composer.json | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/composer.json b/composer.json index c6e1e17e..a9c8e0ef 100644 --- a/composer.json +++ b/composer.json @@ -9,9 +9,6 @@ "email": "sheamck@stanford.edu" } ], - "require": {} - { - "homepage": "https://github.com/SU-SWS/stanford_capx", -} - + "require": {}, + "homepage": "https://github.com/SU-SWS/stanford_capx" } From 674d55963edb520fec542a80faab7f6998548b18 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Tue, 12 Jul 2016 13:14:40 -0700 Subject: [PATCH 42/44] Trying custom module path --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index a9c8e0ef..d6628055 100644 --- a/composer.json +++ b/composer.json @@ -1,7 +1,7 @@ { "name": "su-sws/stanford_capx", "description": "Stanford Career Academic Profiles API integration Module", - "type": "stanford-module", + "type": "custom-module", "license": "GPL-2.0+", "authors": [ { From e03afacf1839aaca5e3d053db21f312748b1c2bd Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Tue, 12 Jul 2016 13:18:13 -0700 Subject: [PATCH 43/44] Trying drupal-module --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index d6628055..d9b9b64b 100644 --- a/composer.json +++ b/composer.json @@ -1,7 +1,7 @@ { "name": "su-sws/stanford_capx", "description": "Stanford Career Academic Profiles API integration Module", - "type": "custom-module", + "type": "drupal-module", "license": "GPL-2.0+", "authors": [ { From 4d6d7326761b98aeafd322b76c870b9169e34055 Mon Sep 17 00:00:00 2001 From: Shea McKinney Date: Tue, 12 Jul 2016 13:41:48 -0700 Subject: [PATCH 44/44] Back to stanford-module --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index d9b9b64b..a9c8e0ef 100644 --- a/composer.json +++ b/composer.json @@ -1,7 +1,7 @@ { "name": "su-sws/stanford_capx", "description": "Stanford Career Academic Profiles API integration Module", - "type": "drupal-module", + "type": "stanford-module", "license": "GPL-2.0+", "authors": [ {