diff --git a/.circleci/config.yml b/.circleci/config.yml index 2dcee84a..11670737 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,7 +49,7 @@ jobs: echo "phpmd for tasks" vendor/bin/phpmd src text defaults/standard/phpmd.xml --suffixes php,inc,module,theme,profile,install,test echo "phpstan for tasks" - vendor/bin/phpstan analyse src --level=2 + vendor/bin/phpstan analyse -c phpstan.neon # Install a drupal test project. - run: diff --git a/composer.json b/composer.json index bd235da9..3f025514 100644 --- a/composer.json +++ b/composer.json @@ -34,7 +34,8 @@ "sort-packages": true, "allow-plugins": { "cweagans/composer-patches": true, - "dealerdirect/phpcodesniffer-composer-installer": true + "dealerdirect/phpcodesniffer-composer-installer": true, + "phing/phing-composer-configurator": true } }, "extra": { diff --git a/defaults.yml b/defaults.yml index 2baf9021..b3eafb9a 100644 --- a/defaults.yml +++ b/defaults.yml @@ -15,6 +15,10 @@ build: # The destination host, either 'acquia', 'pantheon', 'platformsh', or 'other'. This is # currently only used when setting up the settings.php file for a Drupal site. host: acquia + # Adding specific packages for specific host. One per line. + host_packages: + acquia: + - "typhonius/acquia-php-sdk-v2:^2.0" # Drupal configuration used by targets/drupal.xml drupal: @@ -134,9 +138,9 @@ drupal: # $> gunzip -c FILENAME.sql.gz | drush sqlc # # Command to extract text contents of the backup file. - contents_command: gunzip -c + contents_command: gzip -dc # Command to load database contents into Drupal. - mysql_command: drush sqlc + mysql_command: mysql --host=${drupal.site.database.host} --user=${drupal.site.database.username} --password=${drupal.site.database.password} --database=${drupal.site.database.database} # Load a specific file rather than one matching the `export_pattern`. This can be used # if your build relies on a seed database that is checked in to the repository. @@ -191,14 +195,6 @@ acquia: # Directory for storing downloaded database backups. backups: artifacts/backups - # Max age of the downloaded backup database, in hours. - backup_age_hours: 24 - - # The Acquia Cloud hosting "realm" where the site is running. - # - Acquia Cloud Enterprise: 'prod' - # - Acquia Cloud Professional: 'devcloud' - realm: "" - # Acquia site/application name. site: "" @@ -209,12 +205,6 @@ acquia: # Acquia environment to download backups from. env: "prod" - # Acquia Cloud API credentials file, downloaded from your Acquia account. Do not check - # this file into your codebase. - cloud: - conf: "${env.HOME}/.acquia/cloudapi.conf" - - # Configuration to use the PHP interpreter's built in linter to check for syntax errors # and deprecated code. This property is used by the task in the # defaults/build.xml template. diff --git a/defaults/install/.env.example b/defaults/install/.env.example new file mode 100644 index 00000000..164630ff --- /dev/null +++ b/defaults/install/.env.example @@ -0,0 +1,9 @@ +# ddev docker env file. +# See: https://ddev.readthedocs.io/en/stable/users/extend/customization-extendibility/#providing-custom-environment-variables-to-a-container + +# Copy this file into .env (ie. remove .example) + +# Set Acquia Cloud API keys. +# See DevOps secure note in 1Password. +export ACQUIA_CLOUD_API_KEY='REPLACE_ME' +export ACQUIA_CLOUD_API_SECRET='REPLACE_ME' diff --git a/defaults/install/the-build/build.yml b/defaults/install/the-build/build.yml index b8aeda31..8cb420d6 100644 --- a/defaults/install/the-build/build.yml +++ b/defaults/install/the-build/build.yml @@ -35,7 +35,7 @@ drupal: # OPTIONAL: The Drupal database name defaults to the site's "dir" value. database: - database: "drupal" + database: "db" # Multisites created by `phing drupal-add-multisite` will be automatically added here. # @multisite_placeholder@ diff --git a/docs/tasks.md b/docs/tasks.md index 6daa6afd..95cb6e8c 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -103,13 +103,9 @@ Download a recent backup from Acquia Cloud. | Name | Type | Description | Default | Required | |---|---|---|---|---| | dir | directory path | Local backups directory. | | Yes | -| realm | string | Acquia hosting realm, either "devcloud" or "prod". | | Yes | | site | string | Acquia site name. | | Yes | | env | string | Acquia environment, generally "dev", "test", or "prod". | | Yes | | database | string | Acquia database name. | The site name. | No | -| maxAge | int | Maximum age of the backup, in hours. | 24 | No | -| propertyName | string | Name of a property to set to the backup file. | | No | -| credentialsFile | file path | Path to your Acquia Cloud API credentials. (Do not check this file in to your repository) | `~/.acquia/cloudapi.conf` | No | ### Example @@ -118,8 +114,6 @@ Download a recent backup from Acquia Cloud. - + - - ``` diff --git a/phpstan.neon b/phpstan.neon new file mode 100644 index 00000000..c119e483 --- /dev/null +++ b/phpstan.neon @@ -0,0 +1,7 @@ +parameters: + level: 2 + paths: + - src + excludePaths: + analyse: + - src/TheBuild/Acquia diff --git a/src/TheBuild/Acquia/AcquiaTask.php b/src/TheBuild/Acquia/AcquiaTask.php deleted file mode 100644 index dc53c1c0..00000000 --- a/src/TheBuild/Acquia/AcquiaTask.php +++ /dev/null @@ -1,126 +0,0 @@ -mail) || empty($this->key)) { - if (empty($this->credentialsFile)) { - $this->credentialsFile = new \PhingFile($_SERVER['HOME'] . '/.acquia/cloudapi.conf'); - } - - if (!file_exists($this->credentialsFile) || !is_readable($this->credentialsFile)) { - throw new \BuildException("Acquia Cloud credentials file '{$this->credentialsFile}' is not available."); - } - - $contents = file_get_contents($this->credentialsFile); - $creds = json_decode($contents, TRUE); - - $this->mail = $creds['mail']; - $this->key = $creds['key']; - } - - if (empty($this->mail) || empty($this->key)) { - throw new \BuildException('Missing Acquia Cloud API credentials.'); - } - } - - /** - * Build an HTTP request object against the Acquia Cloud API. - * - * @param string $path - * Acquia Cloud API path. - * - * @return \HTTP_Request2 - * Request object. - */ - protected function createRequest(string $path) : \HTTP_Request2 { - $this->loadCredentials(); - - $uri = $this->endpoint . '/' . ltrim($path, '/'); - - $request = new \HTTP_Request2($uri); - $request->setConfig('follow_redirects', TRUE); - $request->setAuth($this->mail, $this->key); - - return $request; - } - - /** - * Example of how to query the Acquia Cloud API. - * - * @param string $path - * Acquia Cloud API path. - * - * @return string - * API response. - */ - protected function getApiResponseBody(string $path) : string { - $request = $this->createRequest($path); - - $this->log('GET ' . $request->getUrl()); - $response = $request->send(); - return $response->getBody(); - } - - /** - * Set the Acquia credentials file. - * - * @param \PhingFile $file - * Acquia credentials file. - * - * @throws \IOException - * @throws \NullPointerException - */ - public function setCredentialsFile(\PhingFile $file) { - $this->credentialsFile = new \PhingFile($file); - } - -} diff --git a/src/TheBuild/Acquia/GetLatestBackupTask.php b/src/TheBuild/Acquia/GetLatestBackupTask.php index 3f11cf13..03b2589f 100644 --- a/src/TheBuild/Acquia/GetLatestBackupTask.php +++ b/src/TheBuild/Acquia/GetLatestBackupTask.php @@ -2,10 +2,16 @@ namespace TheBuild\Acquia; +use AcquiaCloudApi\Connector\Client; +use AcquiaCloudApi\Connector\Connector; +use AcquiaCloudApi\Endpoints\Applications; +use AcquiaCloudApi\Endpoints\DatabaseBackups; +use AcquiaCloudApi\Endpoints\Environments; + /** * Fetch a recent backup from Acquia. */ -class GetLatestBackupTask extends AcquiaTask { +class GetLatestBackupTask extends \Task { /** * Required. Directory for storing downloaded database backups. @@ -14,18 +20,6 @@ class GetLatestBackupTask extends AcquiaTask { */ protected $dir; - /** - * Required. The Acquia Cloud hosting realm where the site is running. - * - * This also appears in a site's server names, as - * 'sitename.REALM.hosting.acquia.com'. - * - Acquia Cloud Enterprise: 'prod' - * - Acquia Cloud Professional: 'devcloud' - * - * @var string - */ - protected $realm; - /** * Required. The Acquia Cloud site account name. * @@ -53,35 +47,6 @@ class GetLatestBackupTask extends AcquiaTask { */ protected $database; - /** - * Optional. Maximum age of the database backup in hours. - * - * If there is no backup matching this age in the current backups.json, the - * backups.json will be refreshed and the newest backup will be downloaded. - * - * @var int - */ - protected $maxAge = 24; - - /** - * Name of a property to populate with the path to the latest database backup. - * - * Optional parameter. - * - * @var string - */ - protected $propertyName; - - /** - * Where to store the JSON list of database backups. - * - * This info is downloaded from the Acquia Cloud API. The file is set to - * 'backups.json' in the directory specified by $dir. - * - * @var \PhingFile - */ - protected $backupsFile; - /** * {@inheritdoc} * @@ -90,207 +55,97 @@ class GetLatestBackupTask extends AcquiaTask { */ public function main() { $this->validate(); - - // Store the Acquia Cloud API JSON database backup records in our backups - // directory.. - $this->backupsFile = new \PhingFile($this->dir, "backups-{$this->site}-{$this->database}-{$this->env}.json"); - - // Check the database backup records for entries within our time window. - $backups = $this->getCurrentBackupRecords(); - - // Have we already downloaded any of the entries in our time window? - $downloaded_backups = []; - foreach ($backups as $backup) { - $filename = basename($backup['path']); - $file = new \PhingFile($this->dir, $filename); - - if ($file->exists()) { - $downloaded_backups[] = $backup; - } - } - - // Pick out the newest current backup record, preferring already downloaded - // backups. - $newest_backup = FALSE; - if (!empty($downloaded_backups)) { - $newest_backup = end($downloaded_backups); - $this->log("Using previously downloaded backup from " . $this->formatBackupTime($newest_backup) . " ({$newest_backup['id']})"); - } - elseif (!empty($backups)) { - $newest_backup = end($backups); - $this->log("Using backup from " . $this->formatBackupTime($newest_backup) . " ({$newest_backup['id']})"); - } - - // If we don't have a current enough backup record, check the API directly. - if (!$newest_backup) { - $this->downloadBackupRecords($this->backupsFile); - // Always return something, regardless of the time window. - $backups = $this->getBackupRecords($this->backupsFile); - $newest_backup = end($backups); - - $this->log("Using backup from " . $this->formatBackupTime($newest_backup) . " ({$newest_backup['id']})"); - } - - // This means that we didn't have a current record in our backups json, and - // the Acquia Cloud API returned empty or malformed JSON. - if (empty($newest_backup)) { - throw new \BuildException('Failed to find a backup record.'); - } - - // Download the backup if it does not yet exist on the filesystem. - $filename = basename($newest_backup['path']); - $file = new \PhingFile($this->dir, $filename); - if (!$file->exists()) { - $this->log("Downloading the backup to " . $file->getAbsolutePath()); - $this->downloadBackup($newest_backup, $file); - } - else { - $this->log("Existing backup found at " . $file->getAbsolutePath()); - } - - // Set the property value if a propertyName was provided. - if ($this->propertyName) { - $project = $this->getProject(); - $project->setNewProperty($this->propertyName, $file->getAbsolutePath()); - } + $credentials = $this->getAcquiaCloudCredentials(); + $client = $this->connectAcquiaCloud($credentials); + $application_uuid = $this->getApplicationUuid($client); + $env_uuid = $this->getEnvironmentsUuid($client, $application_uuid); + $this->getLatestBackup($client, $env_uuid); } /** - * Download a backup from Acquia Cloud. - * - * @param array $backup - * Acquia backup info array. - * @param \PhingFile $destination - * Destination file for the downloaded backup. - */ - protected function downloadBackup(array $backup, \PhingFile $destination) { - $stream = fopen($destination->getAbsolutePath(), 'wb'); - if (!$stream) { - throw new \BuildException('Can not write to ' . $destination->getAbsolutePath()); - } - - // Use an HTTP_Request2 with the Observer pattern in order to download large - // backups. - // @see HTTP/Request2/Observer/UncompressingDownload.php - // @see https://cloudapi.acquia.com/#GET__sites__site_envs__env_dbs__db_backups-instance_route - $request = $this->createRequest("/sites/{$this->realm}:{$this->site}/envs/{$this->env}/dbs/{$this->database}/backups/{$backup['id']}/download.json"); - $request->setConfig('store_body', FALSE); - - $observer = new \HTTP_Request2_Observer_UncompressingDownload($stream, 5000000000); - $request->attach($observer); - - $response = $request->send(); - fclose($stream); - - $this->log("Downloaded " . intval($response->getHeader('content-length')) / 1000000 . "MB to " . $destination->getAbsolutePath()); - } - - /** - * Get backup records that are within the desired time window. + * Get acquia cloud credentials stored in the environments variables. * * @return array - * Array of available backups within the specified timeframe. + * The array structure require to instantiate the cloud api client. */ - protected function getCurrentBackupRecords() { - try { - $backups = $this->getBackupRecords($this->backupsFile); - } - catch (\BuildException $e) { - $backups = []; + private function getAcquiaCloudCredentials() { + if (!$api_key = getenv('ACQUIA_CLOUD_API_KEY')) { + $this->log("Couldn't find ACQUIA_CLOUD_API_KEY env variable."); } - $current_backups = []; - - $threshold_time = new \DateTime("-{$this->maxAge} hours"); - $backup_time = new \DateTime(); - - foreach ($backups as $backup) { - $backup_time->setTimestamp($backup['started']); - if ($backup_time > $threshold_time) { - $current_backups[] = $backup; - } + if (!$api_secret = getenv('ACQUIA_CLOUD_API_SECRET')) { + $this->log("Couldn't find ACQUIA_CLOUD_API_SECRET env variable."); + } + if (!$api_key || !$api_secret) { + throw new \BuildException("Credentials are required."); } - return $current_backups; + return [ + 'key' => $api_key, + 'secret' => $api_secret, + ]; } /** - * Get the array of backup records from the Acquia Cloud API JSON output. - * - * Sorts records from oldest to newest. - * - * @param \PhingFile $file - * Temp file containing the Acquia Cloud API response. - * - * @return array - * Acquia backup info array. - * - * @throws \BuildException - * - * @SuppressWarnings(PHPMD.ShortVariable) + * Set Connection to Acquia Cloud using env variables. */ - protected function getBackupRecords(\PhingFile $file) { - if ($file->exists()) { - $backups = json_decode($file->contents(), TRUE); - - // If the backup records have loaded as an array, and the first record - // has the property that we're using, then it is *probably* valid data. - if (isset($backups[0]['started'])) { - - // Sort the backups by start time so that the newest is always last. - usort($backups, function ($a, $b) { - if ($a['started'] == $b['started']) { - return 0; - } - return ($a['started'] < $b['started']) ? -1 : 1; - }); - - return $backups; - } - elseif (count($backups) === 0) { - // The site might not have been backed up yet. - throw new \BuildException('No Acquia Cloud backups found: ' . $file->getCanonicalPath()); - } - } - throw new \BuildException('Acquia Cloud backup records could not be loaded from JSON: ' . $file->getCanonicalPath()); + private function connectAcquiaCloud($credentials) { + $connector = new Connector($credentials); + return Client::factory($connector); } /** - * Download the latest list of backup records from the Acquia Cloud API. - * - * @param \PhingFile $backups_file - * The file where the downloaded backup should be stored. - */ - protected function downloadBackupRecords(\PhingFile $backups_file) { - $json = $this->getApiResponseBody("/sites/{$this->realm}:{$this->site}/envs/{$this->env}/dbs/{$this->database}/backups.json"); - - $writer = new \FileWriter($backups_file); - $writer->write($json); + * Get latest backup from specified environment. + */ + protected function getLatestBackup($client, $environment_uuid) { + $backup = new DatabaseBackups($client); + $backups = $backup->getAll($environment_uuid, $this->database); + $filepath = $this->dir . '/' . $this->env . '_' . $this->database . '.sql.gz'; + if ($backups) { + // file_put_contents loads the response into memory. + // This is okay for small things like Drush aliases. + // But not for database backups. + // Use curl.options to stream data to disk and minimize memory usage. + $client->addOption('sink', $filepath); + $client->addOption('curl.options', [ + 'CURLOPT_RETURNTRANSFER' => TRUE, + 'CURLOPT_FILE' => $filepath, + ]); + // Get latest backup. + $backupId = $backups[0]->id; + $this->log("Downloading backup id $backupId of database $this->database from $this->env environment $this->env"); + // Downloading the latest backup. + if ($backup->download($environment_uuid, $this->database, $backupId)) { + $this->log("Database was downloaded successfully in $filepath"); + return TRUE; + } + } } /** - * Format the backup time to display in log messages. - * - * @param array $backup - * Acquia backup info array. - * - * @return string - * A human-readable date. + * Get all apps and return the one that belong to the specific project. */ - protected function formatBackupTime(array $backup) { - $time = new \DateTime('now'); - $time->setTimestamp($backup['started']); - return $time->format(DATE_RFC850); + protected function getApplicationUuid($client) { + $apps = new Applications($client); + $applications = $apps->getAll(); + foreach ($applications as $application) { + if ($application->name == $this->site) { + return $application->uuid; + } + } } /** - * Set the Acquia realm. - * - * @param string $value - * Acquia realm. + * Get all environments uuids from the project. */ - public function setRealm(string $value) { - $this->realm = $value; + protected function getEnvironmentsUuid($client, $appUuid) { + $environment = new Environments($client); + $environments = $environment->getAll($appUuid); + foreach ($environments as $env) { + if ($env->name == $this->env) { + return $env->uuid; + } + } } /** @@ -333,26 +188,6 @@ public function setDir(string $value) { $this->dir = new \PhingFile($value); } - /** - * Set the max age. - * - * @param int|string $value - * Max age in hours. - */ - public function setMaxAge(int|string $value) { - $this->maxAge = (int) $value; - } - - /** - * Set the property name. - * - * @param string $value - * Property name to use for the result. - */ - public function setPropertyName(string $value) { - $this->propertyName = $value; - } - /** * Verify that the required parameters are available. */ @@ -361,10 +196,11 @@ protected function validate() { if (empty($this->database)) { $this->database = $this->site; } + // Check the build attributes. - foreach (['dir', 'realm', 'site', 'env'] as $attribute) { + foreach (['dir', 'site', 'env'] as $attribute) { if (empty($this->$attribute)) { - throw new \BuildException("$attribute attribute is required.", $this->location); + throw new \BuildException("$attribute attribute is required."); } } } diff --git a/targets/acquia.xml b/targets/acquia.xml index 6cd52d95..7ea64316 100644 --- a/targets/acquia.xml +++ b/targets/acquia.xml @@ -25,8 +25,9 @@ + - + diff --git a/targets/install.xml b/targets/install.xml index 76c4901f..649e919d 100644 --- a/targets/install.xml +++ b/targets/install.xml @@ -306,11 +306,10 @@ - + - @@ -324,15 +323,35 @@ - - - - + + + + + + + + + + + + + + + + + + + + + + + +