From bb99b20d1fcc47b272f864ed388a2640558d792d Mon Sep 17 00:00:00 2001 From: Jordan Mussi Date: Sun, 7 Jun 2020 16:42:52 +0100 Subject: [PATCH 1/2] Fetch stats for issues closed for version-data --- build.xml | 4 ++ input/build.properties | 3 + php/version-data.php | 145 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 151 insertions(+), 1 deletion(-) diff --git a/build.xml b/build.xml index 4ba366c..5f25aef 100644 --- a/build.xml +++ b/build.xml @@ -25,6 +25,8 @@ ]]> + + @@ -450,6 +452,8 @@ --updateSetName '${updateSetName}' --targetVersion '${targetVersion}' --targetVersionCode '${targetVersionCode}' + --githubRepository '${githubRepository}' + --githubMilestoneNumber '${githubMilestoneNumber}' --distVersionDataFile '${distVersionDataFile}'" /> diff --git a/input/build.properties b/input/build.properties index 822050c..4fa1fc2 100644 --- a/input/build.properties +++ b/input/build.properties @@ -15,3 +15,6 @@ includeInstallInUpdateSet=true # The date applied to all files (YYYY-MM-DD HH:MM) packageDate=2018-05-22 00:00 + +# The milestone number to fetch issues resolved from - comment out to search for milestone with targetVersion as its title +#githubMilestoneNumber= diff --git a/php/version-data.php b/php/version-data.php index 878f622..0a38367 100644 --- a/php/version-data.php +++ b/php/version-data.php @@ -11,9 +11,19 @@ 'updateSetName:', 'targetVersion:', 'targetVersionCode:', + 'githubRepository:', + 'githubMilestoneNumber:', 'distVersionDataFile:', ]); +define('MY_USERAGENT', 'mybb/mybb-build'); +define('DEFAULT_CURLOPTS', [ + CURLOPT_USERAGENT => MY_USERAGENT, + CURLOPT_TIMEOUT => 10, + CURLOPT_RETURNTRANSFER => 1, + CURLOPT_SSL_VERIFYPEER => 1, +]); + function arrayToYml($array, $level = 1) { $output = null; @@ -60,6 +70,94 @@ function directoryStructureSort($a, $b) { } } +function fetchJson($ch, $path) { + $curlopt = [ + CURLOPT_URL => $path, + ] + DEFAULT_CURLOPTS; + + curl_setopt_array($ch, $curlopt); + + $response = curl_exec($ch); + + $data = json_decode($response, true); + + if ($data === null) { + return null; + } + + return $data; +} + +function getIssueStats($ch, $githubApiPath, $milestoneNumber) { + $issueAges = []; + $page = 1; + + $curlopt = [ + CURLOPT_HEADER => 1, + ] + DEFAULT_CURLOPTS; + + while ($page !== null) { + $curlopt[CURLOPT_URL] = $githubApiPath . 'issues?milestone=' . $milestoneNumber . '&state=closed&per_page=100&page=' . $page; + + curl_setopt_array($ch, $curlopt); + + $response = curl_exec($ch); + + $header_size = curl_getinfo($ch, CURLINFO_HEADER_SIZE); + $header = substr($response, 0, $header_size); + $body = substr($response, $header_size); + + $issues = json_decode($body, true); + + if ($issues === null) { + break; + } + + foreach ($issues as $issue) { + if (empty($issue['closed_at']) || !empty($issue['pull_request'])) { + continue; + } + $created = DateTime::createFromFormat(DateTime::ISO8601, $issue['created_at']); + $closed = DateTime::createFromFormat(DateTime::ISO8601, $issue['closed_at']); + + if ($created === false || $closed === false) { + continue; + } + + $interval = $created->diff($closed); + $issueAges[] = (int) $interval->format('%a'); + } + + preg_match('/^Link:(.*?)&page=([0-9]+)>; rel="next"/im', $header, $matches); + + if ($matches) { + $page = $matches[2]; + } else { + $page = null; + } + } + + sort($issueAges); + + $numIssues = count($issueAges); + + if ($numIssues == 0) { + return null; + } + + if ($numIssues % 2 === 0) { + $median = ($issueAges[($numIssues/2)-1] + $issueAges[$numIssues/2]) / 2; + } else { + $median = $issueAges[intdiv($numIssues, 2)]; + } + + return [ + 'medianAge' => $median, + 'meanAge' => round(array_sum($issueAges) / $numIssues, 1), + 'numIssues' => $numIssues, + ]; +} + // changed files YML $file = $args['distChangedFilesFile']; @@ -175,6 +273,51 @@ function directoryStructureSort($a, $b) { } } +// resolved issue YML +$resolvedIssuesYml = null; + +$ch = curl_init(); +$githubApiPath = 'https://api.github.com/repos/' . $args['githubRepository'] . '/'; + +if (empty($args['githubMilestoneNumber']) || ($milestone = fetchJson($ch, $githubApiPath . 'milestones/' . $args['githubMilestoneNumber'])) === null) { + // Attempt to determine milestone number ourselves + $milestones = fetchJson($ch, $githubApiPath . 'milestones?state=all'); + if ($milestones !== null) { + foreach ($milestones as $m) { + if (!empty($m['title']) && $m['title'] === $args['targetVersion']) { + $milestone = $m; + break; + } + } + } +} + +if ($milestone !== null) { + // We have a correct milestone + $resolvedIssuesYml = ''; + + $issueStats = getIssueStats($ch, $githubApiPath, $milestone['number']); + + if ($issueStats !== null) { + $resolvedIssuesYml .= << Date: Tue, 9 Jun 2020 14:25:36 +0100 Subject: [PATCH 2/2] Apply suggestions from code review --- build.xml | 10 ++-- input/build.properties | 3 - php/version-data.php | 126 +++++++++++++++++++++++++---------------- 3 files changed, 84 insertions(+), 55 deletions(-) diff --git a/build.xml b/build.xml index 5f25aef..ffd852c 100644 --- a/build.xml +++ b/build.xml @@ -25,8 +25,9 @@ ]]> - - + + + @@ -452,8 +453,9 @@ --updateSetName '${updateSetName}' --targetVersion '${targetVersion}' --targetVersionCode '${targetVersionCode}' - --githubRepository '${githubRepository}' - --githubMilestoneNumber '${githubMilestoneNumber}' + --issuesRepository '${issuesRepository}' + --resolvedIssuesMilestone '${resolvedIssuesMilestone}' + --resolvedIssuesLink '${resolvedIssuesLink}' --distVersionDataFile '${distVersionDataFile}'" /> diff --git a/input/build.properties b/input/build.properties index 4fa1fc2..822050c 100644 --- a/input/build.properties +++ b/input/build.properties @@ -15,6 +15,3 @@ includeInstallInUpdateSet=true # The date applied to all files (YYYY-MM-DD HH:MM) packageDate=2018-05-22 00:00 - -# The milestone number to fetch issues resolved from - comment out to search for milestone with targetVersion as its title -#githubMilestoneNumber= diff --git a/php/version-data.php b/php/version-data.php index 0a38367..3bd54b7 100644 --- a/php/version-data.php +++ b/php/version-data.php @@ -11,8 +11,9 @@ 'updateSetName:', 'targetVersion:', 'targetVersionCode:', - 'githubRepository:', - 'githubMilestoneNumber:', + 'issuesRepository:', + 'resolvedIssuesMilestone:', + 'resolvedIssuesLink:', 'distVersionDataFile:', ]); @@ -88,8 +89,8 @@ function fetchJson($ch, $path) { return $data; } -function getIssueStats($ch, $githubApiPath, $milestoneNumber) { - $issueAges = []; +function fetchJsonPaged($ch, $path, $sep, $callback, $perPage=100) { + $results = []; $page = 1; $curlopt = [ @@ -97,7 +98,7 @@ function getIssueStats($ch, $githubApiPath, $milestoneNumber) { ] + DEFAULT_CURLOPTS; while ($page !== null) { - $curlopt[CURLOPT_URL] = $githubApiPath . 'issues?milestone=' . $milestoneNumber . '&state=closed&per_page=100&page=' . $page; + $curlopt[CURLOPT_URL] = $path . $sep . 'per_page=' . $perPage . '&page=' . $page; curl_setopt_array($ch, $curlopt); @@ -107,25 +108,19 @@ function getIssueStats($ch, $githubApiPath, $milestoneNumber) { $header = substr($response, 0, $header_size); $body = substr($response, $header_size); - $issues = json_decode($body, true); + $data = json_decode($body, true); - if ($issues === null) { + if ($data === null) { break; } - foreach ($issues as $issue) { - if (empty($issue['closed_at']) || !empty($issue['pull_request'])) { + foreach ($data as $item) { + $result = $callback($item); + if ($result === null) { continue; } - $created = DateTime::createFromFormat(DateTime::ISO8601, $issue['created_at']); - $closed = DateTime::createFromFormat(DateTime::ISO8601, $issue['closed_at']); - if ($created === false || $closed === false) { - continue; - } - - $interval = $created->diff($closed); - $issueAges[] = (int) $interval->format('%a'); + $results[] = $result; } preg_match('/^Link:(.*?)&page=([0-9]+)>; rel="next"/im', $header, $matches); @@ -137,6 +132,62 @@ function getIssueStats($ch, $githubApiPath, $milestoneNumber) { } } + return $results; +} + +function getGithubIssueAges($ch, $repo, $resolvedIssuesMilestone) { + $githubApiPath = 'https://api.github.com/repos/' . $repo . '/'; + + $milestones = fetchJsonPaged($ch, $githubApiPath . 'milestones?state=all', '&', function($item) use ($resolvedIssuesMilestone) { + if ($item['title'] === $resolvedIssuesMilestone) { + return $item; + } + + return null; + }); + + if (empty($milestones[0])) { + return null; + } + + $path = $githubApiPath . 'issues?milestone=' . $milestones[0]['number'] . '&state=closed'; + + $issueAges = fetchJsonPaged($ch, $path, '&', function($item) { + if (empty($item['closed_at']) || !empty($item['pull_request'])) { + return null; + } + + $created = DateTime::createFromFormat(DateTime::ISO8601, $item['created_at']); + $closed = DateTime::createFromFormat(DateTime::ISO8601, $item['closed_at']); + + if ($created === false || $closed === false) { + return null; + } + + $interval = $created->diff($closed); + return (int) $interval->format('%a'); + }); + + return $issueAges; +} + +function getIssuesData($ch, $issuesRepository, $resolvedIssuesMilestone) { + $parsed = parse_url($issuesRepository); + + if ($parsed === false) { + return null; + } + + $repo = trim(preg_replace('/\.git$/', '', $parsed['path']), '/'); + + switch ($parsed['host']) { + case 'github.com': + $issueAges = getGithubIssueAges($ch, $repo, $resolvedIssuesMilestone); + break; + default: + return null; + } + sort($issueAges); $numIssues = count($issueAges); @@ -274,47 +325,26 @@ function getIssueStats($ch, $githubApiPath, $milestoneNumber) { } // resolved issue YML -$resolvedIssuesYml = null; +$resolvedIssuesYml = ''; $ch = curl_init(); -$githubApiPath = 'https://api.github.com/repos/' . $args['githubRepository'] . '/'; - -if (empty($args['githubMilestoneNumber']) || ($milestone = fetchJson($ch, $githubApiPath . 'milestones/' . $args['githubMilestoneNumber'])) === null) { - // Attempt to determine milestone number ourselves - $milestones = fetchJson($ch, $githubApiPath . 'milestones?state=all'); - if ($milestones !== null) { - foreach ($milestones as $m) { - if (!empty($m['title']) && $m['title'] === $args['targetVersion']) { - $milestone = $m; - break; - } - } - } -} -if ($milestone !== null) { - // We have a correct milestone - $resolvedIssuesYml = ''; +$issuesData = getIssuesData($ch, $args['issuesRepository'], $args['resolvedIssuesMilestone']); - $issueStats = getIssueStats($ch, $githubApiPath, $milestone['number']); - - if ($issueStats !== null) { - $resolvedIssuesYml .= <<