diff --git a/.kokoro/docs/publish.sh b/.kokoro/docs/publish.sh index 0f500c7ba52a..31a0f7e751d3 100755 --- a/.kokoro/docs/publish.sh +++ b/.kokoro/docs/publish.sh @@ -37,6 +37,7 @@ do --out $DIR/out \ --metadata-version $VERSION \ --staging-bucket $STAGING_BUCKET \ + --with-cache \ $VERBOSITY else # dry run @@ -44,6 +45,7 @@ do --component $COMPONENT \ --out $DIR/out \ --metadata-version $VERSION \ + --with-cache \ $VERBOSITY fi done diff --git a/composer.json b/composer.json index 2560944b0e4b..5a44b57cf05c 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "psr/http-message": "^1.0|^2.0", "ramsey/uuid": "^4.0", "google/gax": "^1.34.0", - "google/auth": "^1.34" + "google/auth": "^1.42" }, "require-dev": { "phpunit/phpunit": "^9.6", diff --git a/dev/src/Command/ComponentInfoCommand.php b/dev/src/Command/ComponentInfoCommand.php index a3984163cdb8..647fcd744f70 100644 --- a/dev/src/Command/ComponentInfoCommand.php +++ b/dev/src/Command/ComponentInfoCommand.php @@ -44,6 +44,8 @@ class ComponentInfoCommand extends Command 'php_namespaces' => 'Php Namespace', 'github_repo' => 'Github Repo', 'proto_path' => 'Proto Path', + 'proto_packages' => 'Proto Packages', + 'proto_namespaces' => 'Proto Namespaces', 'service_address' => 'Service Address', 'api_shortname' => 'API Shortname', 'description' => 'Description', @@ -216,7 +218,7 @@ private function getComponentDetailRow( 'migration_mode' => $package ? $package->getMigrationStatus() : implode(",", $component->getMigrationStatuses()), 'php_namespaces' => implode(",", array_keys($component->getNamespaces())), 'github_repo' => $component->getRepoName(), - 'proto_path' => $package ? $package->getProtoPackage() : implode(",", $component->getProtoPackages()), + 'proto_path' => $package ? $package->getProtoPath() : implode(",", $component->getProtoPaths()), 'service_address' => $package ? $package->getServiceAddress() : implode(",", $component->getServiceAddresses()), 'api_shortname' => $package ? $package->getApiShortname() : implode(",", array_filter($component->getApiShortnames())), 'description' => $component->getDescription(), @@ -239,6 +241,22 @@ private function getComponentDetailRow( if (array_key_exists('downloads', $requestedFields)) { $row['downloads'] = number_format($this->packagist->getDownloads($component->getPackageName())); } + if ( + array_key_exists('proto_namespaces', $requestedFields) + || array_key_exists('proto_packages', $requestedFields) + ) { + $protoNamespaces = $component->getProtoNamespaces(); + if (array_key_exists('proto_packages', $requestedFields)) { + $row['proto_packages'] = implode(",", array_keys($protoNamespaces)); + } + if (array_key_exists('proto_namespaces', $requestedFields)) { + $row['proto_namespaces'] = implode("\n", array_map( + fn ($key, $value) => $key . ' => ' . $value, + array_keys($protoNamespaces), + array_values($protoNamespaces) + )); + } + } // call again in case the filters were on the slow fields if ($this->filterRow($row, $filters)) { return null; @@ -248,7 +266,7 @@ private function getComponentDetailRow( private function getAvailableApiVersions(Component $component): string { - $protos = $component->getProtoPackages(); + $protos = $component->getProtoPaths(); $proto = array_shift($protos); // Proto packages should be in a version directory $versionPath = dirname($proto); diff --git a/dev/src/Command/DocFxCommand.php b/dev/src/Command/DocFxCommand.php index 74de50b014fc..bf3c18c87955 100644 --- a/dev/src/Command/DocFxCommand.php +++ b/dev/src/Command/DocFxCommand.php @@ -25,6 +25,7 @@ use Symfony\Component\Process\Process; use Symfony\Component\Yaml\Yaml; use RuntimeException; +use Google\Auth\Cache\FileSystemCacheItemPool; use Google\Cloud\Dev\Component; use Google\Cloud\Dev\DocFx\Node\ClassNode; use Google\Cloud\Dev\DocFx\Page\PageTree; @@ -64,6 +65,7 @@ protected function configure() InputOption::VALUE_OPTIONAL, 'Specify the path of the desired component. Please note, this option is only intended for testing purposes.' ) + ->addOption('--with-cache', '', InputOption::VALUE_NONE, 'Cache expensive proto namespace lookups to a file') ; } @@ -108,12 +110,14 @@ protected function execute(InputInterface $input, OutputInterface $output) $tocItems = []; $packageDescription = $component->getDescription(); $isBeta = 'stable' !== $component->getReleaseLevel(); + $packageNamespaces = $this->getProtoPackageToNamespaceMap($input->getOption('with-cache')); foreach ($component->getNamespaces() as $namespace => $dir) { $pageTree = new PageTree( $xml, $namespace, $packageDescription, - $component->getPath() + $component->getPath(), + $packageNamespaces ); foreach ($pageTree->getPages() as $page) { @@ -174,15 +178,20 @@ protected function execute(InputInterface $input, OutputInterface $output) if ($metadataVersion = $input->getOption('metadata-version')) { $output->write(sprintf('Writing docs.metadata with version %s... ', $metadataVersion)); + $xrefs = array_merge(...array_map( + fn ($c) => ['--xrefs', sprintf('devsite://php/%s', $c->getId())], + $component->getComponentDependencies(), + )); $process = new Process([ 'docuploader', 'create-metadata', - '--name', str_replace('google/', '', $component->getPackageName()), + '--name', $component->getId(), '--version', $metadataVersion, '--language', 'php', '--distribution-name', $component->getPackageName(), '--product-page', $component->getProductDocumentation(), '--github-repository', $component->getRepoName(), '--issue-tracker', $component->getIssueTracker(), + ...$xrefs, $outDir . '/docs.metadata' ]); $process->mustRun(); @@ -262,4 +271,21 @@ private function validate(ClassNode $class, OutputInterface $output): bool } return $valid; } + + private function getProtoPackageToNamespaceMap(bool $useFileCache): array + { + if (!$useFileCache) { + return Component::getProtoPackageToNamespaceMap(); + } + + $cache = new FileSystemCacheItemPool('.cache'); + $item = $cache->getItem('phpdoc_proto_package_to_namespace_map'); + + if (!$item->isHit()) { + $item->set(Component::getProtoPackageToNamespaceMap()); + $cache->save($item); + } + + return $item->get(); + } } diff --git a/dev/src/Component.php b/dev/src/Component.php index bc9e7514a8ee..aa04008d7c04 100644 --- a/dev/src/Component.php +++ b/dev/src/Component.php @@ -37,6 +37,8 @@ class Component private string $clientDocumentation; private string $description; private array $namespaces; + /** @var array */ + private array $componentDependencies; public function __construct(private string $name, string $path = null) { @@ -211,8 +213,9 @@ private function validateComponentFiles(): void $this->clientDocumentation = $repoMetadataJson['client_documentation']; $this->productDocumentation = $repoMetadataJson['product_documentation'] ?? ''; + $namespaces = []; foreach ($composerJson['autoload']['psr-4'] as $namespace => $dir) { - if (0 === strpos($dir, 'src')) { + if (str_starts_with($dir, 'src')) { $namespaces[rtrim($namespace, '\\')] = $dir; } } @@ -220,6 +223,22 @@ private function validateComponentFiles(): void throw new RuntimeException('composer autoload.psr-4 does not contain a namespace'); } $this->namespaces = $namespaces; + + // find dependencies which are google/cloud components + $this->componentDependencies = []; + foreach ($composerJson['require'] ?? [] as $name => $version) { + if ($componentName = key(array_filter( + $repoMetadataFullJson, + fn ($metadata) => $metadata['distribution_name'] === $name + ))) { + $this->componentDependencies[] = new Component($componentName); + } + } + if (isset($composerJson['require']['google/gax']) + && !isset($composerJson['require']['google/common-protos']) + ) { + $this->componentDependencies[] = new Component('CommonProtos'); + } } /** @@ -245,9 +264,51 @@ public function getMigrationStatuses(): array return array_map(fn($v) => $v->getMigrationStatus(), $this->getComponentPackages()); } - public function getProtoPackages(): array + public function getProtoNamespaces(): array { - return array_map(fn($v) => $v->getProtoPackage(), $this->getComponentPackages()); + $protoNamespaces = []; + $componentPackages = $this->getComponentPackages(); + foreach ($this->namespaces as $namespace => $dir) { + $componentPackages = $dir === 'src' + ? $this->getComponentPackages() + : [new ComponentPackage($this, str_replace('src/', '', $dir))]; + + $protoNamespaces = array_reduce( + $componentPackages, + fn($protoNamespaces, $pkg) => array_merge($protoNamespaces, $pkg->getProtoNamespaces()), + $protoNamespaces + ); + } + + return $protoNamespaces; + } + + public static function getProtoPackageToNamespaceMap(): array + { + $protoNamespaces = []; + foreach (self::getComponents() as $component) { + $componentProtoNamespaces = $component->getProtoNamespaces(); + if ($commonPackages = array_intersect_key($componentProtoNamespaces, $protoNamespaces)) { + foreach ($commonPackages as $package => $namespace) { + if ($namespace !== $protoNamespaces[$package]) { + throw new RuntimeException(sprintf( + 'Package "%s" has conflicting namespaces: "%s" and "%s"', + $package, + $namespace, + $protoNamespaces[$package] + )); + } + } + } + $protoNamespaces = array_merge($protoNamespaces, $componentProtoNamespaces); + } + + return $protoNamespaces; + } + + public function getProtoPaths(): array + { + return array_map(fn($v) => $v->getProtoPath(), $this->getComponentPackages()); } public function getServiceAddresses(): array @@ -284,4 +345,9 @@ private function getPackagePaths(): array } return array_reverse($paths); } + + public function getComponentDependencies(): array + { + return $this->componentDependencies; + } } diff --git a/dev/src/ComponentPackage.php b/dev/src/ComponentPackage.php index 68a9a1eb80e5..b24109de792c 100644 --- a/dev/src/ComponentPackage.php +++ b/dev/src/ComponentPackage.php @@ -48,7 +48,7 @@ public function getName(): string return $this->name; } - public function getProtoPackage(): string + public function getProtoPath(): string { $gapicClientFiles = $this->getV1GapicClientFiles() + $this->getV2ClientFiles(); @@ -78,11 +78,7 @@ public function getServiceAddress(): string $gapicClientClasses = array_map(fn ($fp) => $this->getClassFromFile($fp), $gapicClientFiles); foreach ($gapicClientClasses as $className) { - // Access V1-surface public constant - if (defined($className . '::SERVICE_ADDRESS')) { - return constant($className . '::SERVICE_ADDRESS'); - } - // Access V2-surface private constant + // Access private constants (for v2 surfaces) if ($constants = (new \ReflectionClass($className))->getConstants()) { if (isset($constants['SERVICE_ADDRESS'])) { return $constants['SERVICE_ADDRESS']; @@ -92,6 +88,27 @@ public function getServiceAddress(): string return ''; } + public function getProtoNamespaces(): array + { + $protoPackages = []; + foreach ($this->getFilesInDir('*.php', $this->path) as $classFile) { + $contents = file_get_contents($classFile); + if (preg_match( + '/Generated from protobuf message ([a-z0-9\.]+)(\..*)<\/code>/', + $contents, + $matches + ) && preg_match('/namespace (.*);/', $contents, $nsMatches)) { + // remove namespace (in case it's nested) + $protoPackages[$matches[1]] = str_replace( + str_replace('.', '\\', substr($matches[2], 0, strrpos($matches[2], '.'))), + '', + $nsMatches[1] + ); + } + } + return array_unique($protoPackages); + } + public function getBaseUri(): string { if (file_exists($this->path . 'Connection/Rest.php')) { diff --git a/dev/src/DocFx/Node/ClassNode.php b/dev/src/DocFx/Node/ClassNode.php index f4b90c25647a..513ff8e6cd92 100644 --- a/dev/src/DocFx/Node/ClassNode.php +++ b/dev/src/DocFx/Node/ClassNode.php @@ -28,11 +28,11 @@ class ClassNode use NameTrait; private $childNode; - private array $protoPackages; private string $tocName; public function __construct( - private SimpleXMLElement $xmlNode + private SimpleXMLElement $xmlNode, + private array $protoPackages = [], ) {} public function isProtobufEnumClass(): bool @@ -247,14 +247,6 @@ public function getProtoPackage(): ?string return null; } - public function setProtoPackages(array $protoPackages) - { - $this->protoPackages = $protoPackages; - if ($this->childNode) { - $this->childNode->setProtoPackages($protoPackages); - } - } - public function getTocName() { return isset($this->tocName) ? $this->tocName : $this->getName(); diff --git a/dev/src/DocFx/Page/PageTree.php b/dev/src/DocFx/Page/PageTree.php index 1c5797eb409c..d7d2b595edaf 100644 --- a/dev/src/DocFx/Page/PageTree.php +++ b/dev/src/DocFx/Page/PageTree.php @@ -35,7 +35,8 @@ public function __construct( private string $xmlPath, private string $namespace, private string $packageDescription, - private string $componentPath + private string $componentPath, + private array $componentPackages ) {} public function getPages(): array @@ -65,7 +66,7 @@ private function loadPages(): array continue; } - $classNode = new ClassNode($file->class[0]); + $classNode = new ClassNode($file->class[0], $this->componentPackages); // Skip the protobuf classes with underscores, they're all deprecated // @TODO: Do not generate them in V2 @@ -149,27 +150,6 @@ private function loadPages(): array } } - /** - * Set a map of protobuf package names to PHP namespaces for Xrefs. - * This MUST be done after combining GAPIC clients. - */ - $protoPackages = [ - // shared packages - 'google.longrunning' => 'Google\\LongRunning' - ]; - foreach ($pages as $page) { - $classNode = $page->getClassNode(); - if ($protoPackage = $classNode->getProtoPackage()) { - $package = rtrim(ltrim($classNode->getNamespace(), '\\'), '\\Client'); - $protoPackages[$protoPackage] = $package; - } - } - - // Add the proto packages to every class node - foreach ($pages as $page) { - $page->getClassNode()->setProtoPackages($protoPackages); - } - // Sort pages alphabetically by full class name ksort($pages); diff --git a/dev/tests/Unit/Command/DocFxCommandTest.php b/dev/tests/Unit/Command/DocFxCommandTest.php index a4d99c0f05b0..bccd5057d735 100644 --- a/dev/tests/Unit/Command/DocFxCommandTest.php +++ b/dev/tests/Unit/Command/DocFxCommandTest.php @@ -131,7 +131,7 @@ public function testDocsMetadataFile() $rightContents = preg_replace('/nanos: \d+/', 'nanos: *', $rightContents); file_put_contents($right, $rightContents); - $this->assertFileEqualsWithDiff($left, $right); + $this->assertFileEqualsWithDiff($left, $right, '1' === getenv('UPDATE_FIXTURES')); } public function provideDocFxFiles() @@ -142,6 +142,7 @@ public function provideDocFxFiles() '--out' => self::$tmpDir = sys_get_temp_dir() . '/' . rand(), '--metadata-version' => '1.0.0', '--component-path' => self::$fixturesDir . '/component/Vision', + '--with-cache' => true, ]); $filesAsArguments = []; @@ -163,6 +164,7 @@ public function provideNewClient() '--xml' => self::$fixturesDir . '/phpdoc/newclient.xml', '--out' => $tmpDir = sys_get_temp_dir() . '/' . rand(), '--metadata-version' => '1.0.0', + '--with-cache' => true, ]); return [ diff --git a/dev/tests/Unit/ComponentTest.php b/dev/tests/Unit/ComponentTest.php index ae4f17f28d4a..f3ae5229eefb 100644 --- a/dev/tests/Unit/ComponentTest.php +++ b/dev/tests/Unit/ComponentTest.php @@ -69,9 +69,27 @@ public function provideComponentProperties() [ 'id' => 'cloud-talent', 'apiVersions' => ['V4'], - 'protoPackages' => ['google/cloud/talent/v4'], + 'protoPaths' => ['google/cloud/talent/v4'], ] ] ]; } + + public function testGetProtoNamespaces() + { + // ensure there are no conflicts - this would throw an exception + $allProtoNamespaces = Component::getProtoPackageToNamespaceMap(); + + // verify a few are as expected + $this->assertEquals('Google\Cloud\Bigtable\V2', $allProtoNamespaces['google.bigtable.v2']); + $this->assertEquals('Google\Cloud\Talent\V4', $allProtoNamespaces['google.cloud.talent.v4']); + $this->assertEquals('Grafeas\V1', $allProtoNamespaces['grafeas.v1']); + $this->assertEquals('Google\Cloud\Workflows\Executions\V1', $allProtoNamespaces['google.cloud.workflows.executions.v1']); + $this->assertEquals('Google\Api', $allProtoNamespaces['google.api']); + $this->assertEquals('Google\Cloud\Location', $allProtoNamespaces['google.cloud.location']); + $this->assertEquals('Google\LongRunning', $allProtoNamespaces['google.longrunning']); + $this->assertEquals('Google\Rpc\Context', $allProtoNamespaces['google.rpc.context']); + $this->assertEquals('Google\Cloud\Iam\V1', $allProtoNamespaces['google.iam.v1']); + $this->assertEquals('Google\Cloud\Logging\Type', $allProtoNamespaces['google.logging.type']); + } } diff --git a/dev/tests/Unit/DocFx/NodeTest.php b/dev/tests/Unit/DocFx/NodeTest.php index 310fe7fea7b1..e1ffbdf51666 100644 --- a/dev/tests/Unit/DocFx/NodeTest.php +++ b/dev/tests/Unit/DocFx/NodeTest.php @@ -284,16 +284,15 @@ public function testProtoRefWithXrefUsingPackageName() $description = '[ListBackups][google.bigtable.admin.v2.BigtableTableAdmin.ListBackups]'; $protoPackages = ['google.bigtable.admin.v2' => 'Google\\Cloud\\Bigtable\\Admin\\V2']; - $xref = new class { + $xref = new class($protoPackages) { use XrefTrait; - public $protoPackages; + public function __construct(private array $protoPackages) { + } public function replace(string $description) { return $this->replaceProtoRef($description); } }; - $xref->protoPackages = $protoPackages; - $this->assertEquals( 'ListBackups', $xref->replace($description) @@ -302,9 +301,7 @@ public function replace(string $description) { $classNode = new ClassNode(new SimpleXMLElement(sprintf( '%s', $description - ))); - - $classNode->setProtoPackages($protoPackages); + )), $protoPackages); $this->assertEquals( 'ListBackups', diff --git a/dev/tests/Unit/DocFx/PageTest.php b/dev/tests/Unit/DocFx/PageTest.php index 049812c06327..92f38a7073c7 100644 --- a/dev/tests/Unit/DocFx/PageTest.php +++ b/dev/tests/Unit/DocFx/PageTest.php @@ -43,7 +43,6 @@ public function testFriendlyApiName( file_get_contents(__DIR__ . '/../../fixtures/phpdoc/service.xml') ); $classNode = new ClassNode(new SimpleXMLElement($serviceXml)); - $classNode->setProtoPackages([]); $componentPath = __DIR__ . '/../../fixtures/component/Vision'; $page = new Page($classNode, '', $packageDescription, $componentPath); @@ -68,7 +67,17 @@ public function testLoadPagesProtoPackages() { $structureXml = __DIR__ . '/../../fixtures/phpdoc/structure.xml'; $componentPath = __DIR__ . '/../../fixtures/component/Vision'; - $pageTree = new PageTree($structureXml, 'Google\Cloud\Vision', '', $componentPath); + $protoPackages = [ + 'google.longrunning' => 'Google\LongRunning', + 'google.cloud.vision.v1' => 'Google\Cloud\Vision\V1', + ]; + $pageTree = new PageTree( + $structureXml, + 'Google\Cloud\Vision', + '', + $componentPath, + $protoPackages + ); $pages = $pageTree->getPages(); $this->assertTrue(count($pages) > 0); @@ -77,12 +86,9 @@ public function testLoadPagesProtoPackages() $classNodeReflection = new \ReflectionClass($classNode); $protoPackagesProperty = $classNodeReflection->getProperty('protoPackages'); $protoPackagesProperty->setAccessible(true); - $sharedPackages = [ - 'google.longrunning' => 'Google\LongRunning', - ]; $this->assertEquals( - ['google.cloud.vision.v1' => 'Google\Cloud\Vision\V1'] + $sharedPackages, + $protoPackages, $protoPackagesProperty->getValue($classNode) ); } @@ -108,7 +114,7 @@ public function testHandleSample() { $structureXml = __DIR__ . '/../../fixtures/phpdoc/clientsnippets.xml'; $componentPath = __DIR__ . '/../../fixtures/component/ClientSnippets'; - $pageTree = new PageTree($structureXml, 'Google\Cloud\ClientSnippets', '', $componentPath); + $pageTree = new PageTree($structureXml, 'Google\Cloud\ClientSnippets', '', $componentPath, []); $pages = $pageTree->getPages(); $this->assertCount(1, $pages); diff --git a/dev/tests/fixtures/docfx/Vision/docs.metadata b/dev/tests/fixtures/docfx/Vision/docs.metadata index 24feaaa34fa3..db5afd540963 100644 --- a/dev/tests/fixtures/docfx/Vision/docs.metadata +++ b/dev/tests/fixtures/docfx/Vision/docs.metadata @@ -8,3 +8,5 @@ language: "php" distribution_name: "google/cloud-vision" github_repository: "googleapis/google-cloud-php-vision" issue_tracker: "https://github.com/googleapis/google-cloud-php-vision/issues" +xrefs: "devsite://php/cloud-core" +xrefs: "devsite://php/common-protos"