Skip to content

Commit

Permalink
Merge pull request #139 from tractorcow/pulls/update-task
Browse files Browse the repository at this point in the history
API Rewrite source file update task
  • Loading branch information
camfindlay authored Jul 4, 2016
2 parents f6c297c + 2fdaa20 commit a96421e
Show file tree
Hide file tree
Showing 9 changed files with 266 additions and 72 deletions.
8 changes: 7 additions & 1 deletion app/_config.php
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
global $databaseConfig;

$parts = parse_url($_ENV['CLEARDB_DATABASE_URL']);

$databaseConfig['type'] = 'MySQLDatabase';
$databaseConfig['server'] = $parts['host'];
$databaseConfig['username'] = $parts['user'];
Expand Down Expand Up @@ -52,6 +52,12 @@
'Tags' => 'silverstripe sapphire php framework cms content management system'
));

// SS Platform logging
if(defined('AWS_SYSLOG_LEVEL')) {
$sysLogWriter = new SS_SysLogWriter('silverstripe', LOG_PID | LOG_CONS);
SS_Log::add_writer($sysLogWriter, (int)AWS_SYSLOG_LEVEL, '<=');
}

// Changelogs have heaps of phrases, but are rarely relevant for content searches
Config::inst()->update('DocumentationSearch', 'boost_by_path', array(
'/^changelog/' => 0.05
Expand Down
6 changes: 5 additions & 1 deletion app/_config/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,8 @@ Controller:
- ControllerExtension
QuickFeedbackExtension:
redirect_field: true
rate_limit: 1
rate_limit: 1
Injector:
MarkdownUpdater:
class: GitMarkdownUpdater

3 changes: 1 addition & 2 deletions app/_config/docsviewer.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,13 @@ DocumentationManifest:
Title: "Framework"
Version: "3.4"
Branch: "3.4"
Stable: false
Stable: true
DefaultEntity: true
-
Path: "assets/src/framework_3.3/docs/"
Title: "Framework"
Version: "3.3"
Branch: "3.3"
Stable: true
DefaultEntity: true
-
Path: "assets/src/framework_3.2/docs/"
Expand Down
148 changes: 98 additions & 50 deletions app/code/RefreshMarkdownTask.php
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ class RefreshMarkdownTask extends BuildTask
/**
* @var string
*/
protected $description = "Downloads a fresh version of markdown documentation files from source";
protected $description = "Downloads a fresh version of markdown documentation files from source. Options are force=1, addonly=1, and branch=<branch>";

/**
* @var bool
Expand All @@ -35,82 +35,130 @@ class RefreshMarkdownTask extends BuildTask
*/
public function run($request)
{
$this->request = $request;
$repositories = $this->getRepositories();
foreach ($repositories as $repository) {
$this->cloneRepository($repository);
$this->request = $request;
$repositories = $this->getRepositories();
$baseDir = $this->getSourceRoot();
$updater = $this->getUpdater();

// Ensure root directory exists
if(!$this->ensureRootDirectory($baseDir)) {
return;
}

// Update each repo
foreach ($repositories as list($repo, $folder, $branch)) {
$path = "{$baseDir}/{$folder}_{$branch}";

// Pass in ?addonly=1 to only update new branches
if($this->request && $this->request->getVar('addonly') && file_exists($path)) {
$this->printLine("Skipping update of {$branch}: Already exists at {$path};");
continue;
}

// Check if we want to update only a specific branch
if($this->request && ($onlyBranch = $this->request->getVar('branch')) && $onlyBranch != $branch) {
$this->printLine("Skipping update of {$branch}: doesn't match provided filter");
continue;
}

// Pass in ?force=1 to force delete existing repos, rather than trying to update them
if($this->request && $this->request->getVar('force') && file_exists($path)) {
$this->printLine("Removing {$path}");
Filesystem::removeFolder($path);
}

// Update this repo
$this->printLine("Beginning update of {$branch}");
$errors = $updater->update($repo, $path, $branch);

// Handle result
if(empty($errors)) {
$this->printLine("Successful update of {$branch}");
} else {
foreach($errors as $error) {
$this->error($error);
}
}
}
$this->printLine(" ");
$this->printLine("To re-index the freshly downloaded documentation files either:");
$this->printLine("(1) run the command 'sake dev/tasks/RebuildLuceneDocsIndex flush=1' in a shell or");
$this->printLine("(2) point your browser at the url 'http://localhost/path/to/ssdocs/dev/tasks/RebuildLuceneDocsIndex?flush=1'");
}

/**
* Gets the service that will pull down remote markdown files
* @return MarkdownUpdater
*/
protected function getUpdater() {
return Injector::inst()->get('MarkdownUpdater');
}

/**
* Get sources root directory
*
* @return string
*
* @todo document this new configuration parameter
*/
private function getPath()
private function getSourceRoot()
{
return ASSETS_PATH;
return ASSETS_PATH . '/src';
}

/**
* @param string $message
*/
private function printLine($message)
{
$this->eol = Director::is_cli() ? PHP_EOL : "<br>";
print $message . $this->eol;
flush();
$eol = Director::is_cli() ? PHP_EOL : "<br>";
echo $message . $eol;
if(ob_get_level() > 0) {
ob_flush();
}
flush();


}

/**
* Log an error
*
* @param string $message
*/
protected function error($message)
{
$this->printLine($message);
SS_Log::log($message, SS_Log::ERR);
}

/**
* Returns the array of repos to source markdown docs from
*
* @return array
*/
private function getRepositories()
protected function getRepositories()
{
if($repos = $this->config()->documentation_repositories)
{
return $repos;
} else {
user_error("You need to set 'RefreshMarkdownTask:documentation_repositories' array in a yaml configuration file", E_USER_WARNING);
return null;
}
$repositories = $this->config()->documentation_repositories;
if(empty($repositories)) {
$this->error(
"You need to set 'RefreshMarkdownTask:documentation_repositories' array in a yaml configuration file"
);
}
return $repositories;
}

/**
* Clone $repository which contains the most current documentation source markdown files
*
* @param array $repository
*/
private function cloneRepository(array $repository)
{

list($remote, $folder, $branch) = $repository;

$path = $this->getPath();

exec("mkdir -p {$path}/src");
exec("chmod -R 775 {$path}/src" );
exec("rm -rf {$path}/src/{$folder}_{$branch}");
chdir("{$path}/src");

// If the dev=1 flag is used when RefreshMarkdownTask is run, a full git clone of the framework repository is kept
// to enable local development of framework and doc.silverstripe.org from within doc.silverstripe.org. Otherwise,
// only the documentation source files are downloaded, only allowing the viewing of documentation files.

if ($this->request instanceof SS_HTTPRequest && $this->request->getVar('dev')) {
$this->printLine("Cloning repository {$remote}/{$branch} into assets/src/{$folder}_{$branch}");
exec("git clone --quiet https://github.com/{$remote}.git {$folder}_{$branch} --branch {$branch}");
} else {
$this->printLine("Downloading the latest documentation files from repository {$remote}/{$branch} to assets/src/{$folder}_{$branch}");
exec("svn export --quiet https://github.com/{$remote}.git/branches/{$branch}/docs {$folder}_{$branch}/docs");
}

}
/**
* Ensure root directory exists
*
* @param string $path
* @return bool True if the directory exists and is writable
*/
protected function ensureRootDirectory($path) {
Filesystem::makeFolder($path);
if(is_dir($path) && is_writable($path)) {
return true;
}
$this->error("Could not create {$path}");
return false;
}

}
86 changes: 86 additions & 0 deletions app/code/updaters/GitMarkdownUpdater.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
<?php

class GitMarkdownUpdater implements MarkdownUpdater
{

public function update($repo, $path, $branch)
{
// Ensure git is available
$errors = array();
if (!$this->runCommand('git --version', $errors)) {
return $errors;
}

// Check if we should do a git update or re-clone
if (is_dir($path . '/.git')) {
return $this->doFetch($path, $branch);
} else {
return $this->doClone($repo, $path, $branch);
}
}

/**
* Update existing git checkouts
*
* @param string $path
* @param string $branch
* @return array List of errors
*/
protected function doFetch($path, $branch) {
$errors = array();
$fetchCommand = sprintf(
'cd %s && git fetch origin %s',
escapeshellarg($path),
escapeshellarg($branch)
);
$resetCommand = sprintf(
'cd %s && git reset --hard origin/%s',
escapeshellarg($path),
escapeshellarg($branch)
);

// Run
if($this->runCommand($fetchCommand, $errors)) {
$this->runCommand($resetCommand, $errors);
}
return $errors;
}

/**
* Clone a new git repo
* @param string $repo Repo slug
* @param string $path Destination path
* @param string $branch Branch name
* @return array
*/
protected function doClone($repo, $path, $branch) {
$errors = array();
$remote = "https://github.com/{$repo}.git";
$this->runCommand(sprintf(
"git clone --quiet %s %s --branch %s",
escapeshellarg($remote),
escapeshellarg($path),
escapeshellarg($branch)
), $errors);
return $errors;
}

/**
* Run this command
*
* @param string $cmd
* @param array $errors
* @return bool Flag if the command was successful
*/
protected function runCommand($cmd, &$errors = array()) {
exec("{$cmd} 2>&1 >/dev/null", $output, $result);
if($result) {
$errors[] = "Error running command {$cmd}:";
foreach($output as $error) {
$errors[] = ' * ' . $error;
}
return false;
}
return true;
}
}
12 changes: 12 additions & 0 deletions app/code/updaters/MarkdownUpdater.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
<?php

interface MarkdownUpdater {

/**
* @param string $repo
* @param string $path
* @param string $branch
* @return array List of any errors that occurred
*/
public function update($repo, $path, $branch);
}
33 changes: 33 additions & 0 deletions app/code/updaters/SVNMarkdownUpdater.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
<?php

class SVNMarkdownUpdater implements MarkdownUpdater
{
public function update($repo, $path, $branch)
{
$errors = array();
$svnPath = "https://github.com/{$repo}.git/branches/{$branch}/docs";
$svnDest = "{$path}/docs";
$this->runCommand(sprintf(
"svn export %s %s",
escapeshellarg($svnPath),
escapeshellarg($svnDest)
), $errors);
return $errors;
}

/**
* Run this command
*
* @param string $cmd
* @param array $errors
* @return bool Flag if the command was successful
*/
protected function runCommand($cmd, &$errors = array()) {
exec($cmd, $output, $result);
if($result) {
$errors[] = "Error running command {$cmd}";
return false;
}
return true;
}
}
2 changes: 1 addition & 1 deletion composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"ext-gd": "*",
"ext-mbstring": "*",
"silverstripe/docsviewer": "^2.0",
"silverstripe/framework": "3.2.1",
"silverstripe/framework": "^3.2",
"silverstripe/toolbar": "^4.0",
"silverstripe/dynamodb": "^1.1",
"mandrew/silverstripe-quickfeedback": "^0.2"
Expand Down
Loading

0 comments on commit a96421e

Please sign in to comment.