Skip to content

Commit

Permalink
9.3.21
Browse files Browse the repository at this point in the history 
- Remove sparq from provisioning
- HSD8-1321 Remove and uninstall react paragraphs module (#1136)
- HSD8-1318 Delete sparkbox-sandbox site
- Updated dependencies Jun 15 2022
- Fixup codeclimate issues
- HSD8-1315 HSD8-1314: provision suac & sparq
- HSD8-1312 Adjust database copy tasks to use guzzle instead of 3rd party acquia API
  • Loading branch information
@pookmish
pookmish authored Jun 20, 2022
2 parents 79e2edf + 41af4d0 commit 9a665a1
Show file tree
Hide file tree
Showing 51 changed files with 827 additions and 889 deletions.
2 changes: 1 addition & 1 deletion blt/blt.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,11 +164,11 @@ multisites:
- siw
- sociology
- southasia
- sparkbox_sandbox
- stanfordsciencefellows
- starlab
- statistics
- sts
- suac
- swshumsci_sandbox
- symsys
- tessier_lavigne_lab
Expand Down
164 changes: 100 additions & 64 deletions blt/src/Blt/Plugin/Commands/HsAcquiaApiCommands.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,20 @@ class HsAcquiaApiCommands extends BltTasks {
*/
protected $failedDatabases = [];

/**
* Keyed array of environment UUIDs.
*
* @var array
*/
protected $enivronments = [];

/**
* Keyed array with access token data and expiration.
*
* @var array
*/
protected $accessToken = [];

/**
* Get the environment UUID for the application from the machine name.
*
Expand All @@ -41,9 +55,13 @@ class HsAcquiaApiCommands extends BltTasks {
* @throws \Exception
*/
protected function getEnvironmentUuid(string $name) {
if (isset($this->enivronments[$name])) {
return $this->enivronments[$name];
}
/** @var \AcquiaCloudApi\Response\EnvironmentResponse $env */
foreach ($this->acquiaEnvironments->getAll($this->appId) as $env) {
if ($env->name == $name) {
$this->enivronments[$name] = $env->uuid;
return $env->uuid;
}
}
Expand Down Expand Up @@ -147,44 +165,46 @@ public function syncStaging(array $options = [
'env' => 'test',
'no-notify' => FALSE,
]) {
$task_started = time() - (60 * 60 * 24);
$this->connectAcquiaApi();
$from_uuid = $this->getEnvironmentUuid('prod');
$to_uuid = $this->getEnvironmentUuid($options['env']);

$sites = $this->getSitesToSync($task_started, $options);
$this->taskStartedTime = time() - (60 * 60 * 24);

$sites = $this->getSitesToSync($options);
if (empty($options['no-interaction']) && !$this->confirm(sprintf('Are you sure you wish to stage the following sites: <comment>%s</comment>', implode(', ', $sites)))) {
return;
}
$count = count($sites);
$copy_sites = array_splice($sites, 0, 4);

foreach ($copy_sites as $site) {
$this->say("Copying $site database to staging.");
$this->acquiaDatabases->copy($this->getEnvironmentUuid('prod'), $site, $this->getEnvironmentUuid($options['env']));
}

$concurrent_copies = 5;
$in_progress = [];
while (!empty($sites)) {
echo '.';
sleep(10);
$finished_databases = $this->getCompletedDatabaseCopies($task_started);

if ($finished = array_intersect($copy_sites, $finished_databases)) {
echo PHP_EOL;
foreach ($finished as $copied_db) {
$db_position = array_search($copied_db, $copy_sites);
$new_site = array_splice($sites, 0, 1);
$new_site = reset($new_site);
$copy_sites[$db_position] = $new_site;
$this->say("Copying $new_site database to staging.");
$this->connectAcquiaApi();
$this->say($this->acquiaDatabases->copy($this->getEnvironmentUuid('prod'), $new_site, $this->getEnvironmentUuid($options['env']))->message);
if (count($in_progress) >= $concurrent_copies) {
// Check for completion.
foreach ($in_progress as $key => $database_name) {
if ($this->databaseCopyFinished($database_name)) {
unset($in_progress[$key]);
}
}
}
}
$this->yell("$count database have been copied to staging.");

if (array_unique($this->failedDatabases)) {
$this->yell('Databases failed: ' . implode(', ', array_unique($this->failedDatabases)), 40, 'red');
$copy_these = array_splice($sites, 0, $concurrent_copies - count($in_progress));
foreach ($copy_these as $database_name) {
$in_progress[] = $database_name;
$this->say(sprintf('Copying database %s', $database_name));
$access_token = $this->getAccessToken();
$client = new Client();
$response = $client->post("https://cloud.acquia.com/api/environments/$to_uuid/databases", [
'headers' => ['Authorization' => "Bearer $access_token"],
'json' => ['name' => $database_name, 'source' => $from_uuid],
]);
$message = json_decode((string) $response->getBody(), TRUE, 512, JSON_THROW_ON_ERROR);
$this->say($message['message']);
}
echo '.';
sleep(30);
}
$this->yell("$count database have been copied to staging.");

$root = $this->getConfigValue('repo.root');
if (file_exists("$root/keys/secrets.settings.php")) {
Expand All @@ -196,7 +216,7 @@ public function syncStaging(array $options = [
'form_params' => [
'payload' => json_encode([
'username' => 'Acquia Cloud',
'text' => sprintf('All Databases have been copied to %s environment.', $options['env']),
'text' => sprintf('%s Databases have been copied to %s environment.', $count, $options['env']),
'icon_emoji' => 'information_source',
]),
],
Expand All @@ -205,33 +225,53 @@ public function syncStaging(array $options = [
}

/**
* Get a list of all databases that have finished copying after a time.
* Call the API and usig the notifications, find out if it's done copying.
*
* @param int $time_comparison
* Time to compare the completed task.
* @param string $database_name
* Acquia database name.
*
* @return array
* Array of database names.
* @return bool
* If the database has been copied in the past 12 hours.
*/
protected function getCompletedDatabaseCopies($time_comparison) {
$databases = [];
$this->connectAcquiaApi();
/** @var \AcquiaCloudApi\Response\NotificationResponse $notification */
foreach ($this->acquiaNotifications->getAll($this->appId) as $notification) {
if (
isset($notification->event) &&
$notification->event == 'DatabaseCopied' &&
strtotime($notification->created_at) >= $time_comparison
) {
if ($notification->status == 'completed') {
$databases = array_merge($databases, $notification->context->database->names);
}
elseif ($notification->status != 'in-progress') {
$this->failedDatabases = array_merge($this->failedDatabases, $notification->context->database->names);
}
}
protected function databaseCopyFinished(string $database_name): bool {
$access_token = $this->getAccessToken();
$client = new Client();
$created_since = date('c', time() - (60 * 60 * 12));
$response = $client->get("https://cloud.acquia.com/api/applications/{$this->appId}/notifications", [
'headers' => ['Authorization' => "Bearer $access_token"],
'query' => [
'filter' => "event=DatabaseCopied;description=@*$database_name*;status!=in-progress;created_at>=$created_since",
],
]);
$message = json_decode((string) $response->getBody(), TRUE, 512, JSON_THROW_ON_ERROR);
return $message['total'] > 0;
}

/**
* Call the API and fetch the OAuth token.
*
* @return string
* Access bearer token.
*/
protected function getAccessToken(): string {
if (isset($this->accessToken['expires']) && time() <= $this->accessToken['expires']) {
return $this->accessToken['token'];
}
return array_values(array_unique($databases));

$client = new Client();
$response = $client->post('https://accounts.acquia.com/api/auth/oauth/token', [
'form_params' => [
'client_id' => getenv('ACQUIA_KEY'),
'client_secret' => getenv('ACQUIA_SECRET'),
'grant_type' => 'client_credentials',
],
]);
$response_body = json_decode((string) $response->getBody(), TRUE, 512, JSON_THROW_ON_ERROR);
$this->accessToken = [
'token' => $response_body['access_token'],
'expires' => time() + $response_body['expires_in'] - 60,
];
return $this->accessToken['token'];
}

/**
Expand Down Expand Up @@ -285,38 +325,34 @@ protected function connectAcquiaApi() {
/**
* Get an overall list of database names to sync.
*
* @param int $task_started
* Time to compare the completed task.
* @param array $options
* Array of keyed command options.
*
* @return array
* Array of database names to sync.
*/
protected function getSitesToSync($task_started, array $options) {
$finished_databases = $this->getCompletedDatabaseCopies($task_started);

protected function getSitesToSync(array $options) {
$sites = $this->getConfigValue('multisites');
foreach ($sites as $key => &$db_name) {
$db_name = $db_name == 'default' ? 'humscigryphon' : $db_name;

if (strpos($db_name, 'sandbox') !== FALSE) {
unset($sites[$key]);
continue;
}

$this->say(sprintf('Checking if %s has recently been copied', $db_name));
if ($this->databaseCopyFinished($db_name)) {
unset($sites[$key]);
}
}
asort($sites);
$sites = array_values($sites);
if (!empty($options['exclude'])) {
$exclude = explode(',', $options['exclude']);
$sites = array_diff($sites, $exclude);
}

if ($options['resume']) {
asort($finished_databases);
$last_database = end($finished_databases);
$last_db_position = array_search($last_database, $sites);
$sites = array_slice($sites, $last_db_position);
}
return array_diff($sites, $finished_databases);
return array_values($sites);
}

/**
Expand Down
Loading

0 comments on commit 9a665a1

Please sign in to comment.