feat(docs): add support for cross-package references

googleapis · Nov 1, 2024 · 959ef92 · 959ef92
1 parent 068f434
commit 959ef92
Show file tree

Hide file tree

Showing 11 changed files with 150 additions and 61 deletions.
diff --git a/.kokoro/docs/publish.sh b/.kokoro/docs/publish.sh
@@ -37,13 +37,15 @@ do
             --out $DIR/out \
             --metadata-version $VERSION \
             --staging-bucket $STAGING_BUCKET \
+            --with-cache \
             $VERBOSITY
     else
         # dry run
         $PROJECT_DIR/dev/google-cloud docfx \
             --component $COMPONENT \
             --out $DIR/out \
             --metadata-version $VERSION \
+            --with-cache \
             $VERBOSITY
     fi
 done

diff --git a/composer.json b/composer.json
@@ -51,7 +51,7 @@
         "psr/http-message": "^1.0|^2.0",
         "ramsey/uuid": "^4.0",
         "google/gax": "^1.34.0",
-        "google/auth": "^1.34"
+        "google/auth": "^1.42"
     },
     "require-dev": {
         "phpunit/phpunit": "^9.6",

diff --git a/dev/src/Command/ComponentInfoCommand.php b/dev/src/Command/ComponentInfoCommand.php
@@ -44,6 +44,8 @@ class ComponentInfoCommand extends Command
         'php_namespaces' => 'Php Namespace',
         'github_repo' => 'Github Repo',
         'proto_path' => 'Proto Path',
+        'proto_packages' => 'Proto Packages',
+        'proto_namespaces' => 'Proto Namespaces',
         'service_address' => 'Service Address',
         'api_shortname' => 'API Shortname',
         'description' => 'Description',
@@ -216,7 +218,7 @@ private function getComponentDetailRow(
             'migration_mode' => $package ? $package->getMigrationStatus() : implode(",", $component->getMigrationStatuses()),
             'php_namespaces' => implode(",", array_keys($component->getNamespaces())),
             'github_repo' => $component->getRepoName(),
-            'proto_path' => $package ? $package->getProtoPackage() : implode(",", $component->getProtoPackages()),
+            'proto_path' => $package ? $package->getProtoPath() : implode(",", $component->getProtoPaths()),
             'service_address' => $package ? $package->getServiceAddress() : implode(",", $component->getServiceAddresses()),
             'api_shortname' => $package ? $package->getApiShortname() : implode(",", array_filter($component->getApiShortnames())),
             'description' => $component->getDescription(),
@@ -239,6 +241,22 @@ private function getComponentDetailRow(
         if (array_key_exists('downloads', $requestedFields)) {
             $row['downloads'] = number_format($this->packagist->getDownloads($component->getPackageName()));
         }
+        if (
+            array_key_exists('proto_namespaces', $requestedFields)
+            || array_key_exists('proto_packages', $requestedFields)
+        ) {
+            $protoNamespaces = $component->getProtoNamespaces();
+            if (array_key_exists('proto_packages', $requestedFields)) {
+                $row['proto_packages'] = implode(",", array_keys($protoNamespaces));
+            }
+            if (array_key_exists('proto_namespaces', $requestedFields)) {
+                $row['proto_namespaces'] = implode("\n", array_map(
+                    fn ($key, $value) => $key . ' => ' . $value,
+                    array_keys($protoNamespaces),
+                    array_values($protoNamespaces)
+                ));
+            }
+        }
         // call again in case the filters were on the slow fields
         if ($this->filterRow($row, $filters)) {
             return null;
@@ -248,7 +266,7 @@ private function getComponentDetailRow(
 
     private function getAvailableApiVersions(Component $component): string
     {
-        $protos = $component->getProtoPackages();
+        $protos = $component->getProtoPaths();
         $proto = array_shift($protos);
         // Proto packages should be in a version directory
         $versionPath = dirname($proto);

diff --git a/dev/src/Command/DocFxCommand.php b/dev/src/Command/DocFxCommand.php
@@ -25,6 +25,7 @@
 use Symfony\Component\Process\Process;
 use Symfony\Component\Yaml\Yaml;
 use RuntimeException;
+use Google\Auth\Cache\FileSystemCacheItemPool;
 use Google\Cloud\Dev\Component;
 use Google\Cloud\Dev\DocFx\Node\ClassNode;
 use Google\Cloud\Dev\DocFx\Page\PageTree;
@@ -64,6 +65,7 @@ protected function configure()
                 InputOption::VALUE_OPTIONAL,
                 'Specify the path of the desired component. Please note, this option is only intended for testing purposes.'
             )
+            ->addOption('--with-cache', '', InputOption::VALUE_NONE, 'Cache expensive proto namespace lookups to a file')
         ;
     }
 
@@ -108,12 +110,14 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $tocItems = [];
         $packageDescription = $component->getDescription();
         $isBeta = 'stable' !== $component->getReleaseLevel();
+        $packageNamespaces = $this->getProtoPackageToNamespaceMap($input->getOption('with-cache'));
         foreach ($component->getNamespaces() as $namespace => $dir) {
             $pageTree = new PageTree(
                 $xml,
                 $namespace,
                 $packageDescription,
-                $component->getPath()
+                $component->getPath(),
+                $packageNamespaces
             );
 
             foreach ($pageTree->getPages() as $page) {
@@ -262,4 +266,21 @@ private function validate(ClassNode $class, OutputInterface $output): bool
         }
         return $valid;
     }
+
+    private function getProtoPackageToNamespaceMap(bool $useFileCache): array
+    {
+        if (!$useFileCache) {
+            return Component::getProtoPackageToNamespaceMap();
+        }
+
+        $cache = new FileSystemCacheItemPool('.cache');
+        $item = $cache->getItem('phpdoc_proto_package_to_namespace_map');
+
+        if (!$item->isHit()) {
+            $item->set(Component::getProtoPackageToNamespaceMap());
+            $cache->save($item);
+        }
+
+        return $item->get();
+    }
 }
diff --git a/dev/src/Component.php b/dev/src/Component.php
@@ -211,8 +211,9 @@ private function validateComponentFiles(): void
         $this->clientDocumentation = $repoMetadataJson['client_documentation'];
         $this->productDocumentation = $repoMetadataJson['product_documentation'] ?? '';
 
+        $namespaces = [];
         foreach ($composerJson['autoload']['psr-4'] as $namespace => $dir) {
-            if (0 === strpos($dir, 'src')) {
+            if (str_starts_with($dir, 'src')) {
                 $namespaces[rtrim($namespace, '\\')] = $dir;
             }
         }
@@ -245,9 +246,51 @@ public function getMigrationStatuses(): array
         return array_map(fn($v) => $v->getMigrationStatus(), $this->getComponentPackages());
     }
 
-    public function getProtoPackages(): array
+    public function getProtoNamespaces(): array
     {
-        return array_map(fn($v) => $v->getProtoPackage(), $this->getComponentPackages());
+        $protoNamespaces = [];
+        $componentPackages = $this->getComponentPackages();
+        foreach ($this->namespaces as $namespace => $dir) {
+            $componentPackages = $dir === 'src'
+                ? $this->getComponentPackages()
+                : [new ComponentPackage($this, str_replace('src/', '', $dir))];
+
+            $protoNamespaces = array_reduce(
+                $componentPackages,
+                fn($protoNamespaces, $pkg) => array_merge($protoNamespaces, $pkg->getProtoNamespaces()),
+                $protoNamespaces
+            );
+        }
+
+        return $protoNamespaces;
+    }
+
+    public static function getProtoPackageToNamespaceMap(): array
+    {
+        $protoNamespaces = [];
+        foreach (self::getComponents() as $component) {
+            $componentProtoNamespaces = $component->getProtoNamespaces();
+            if ($commonPackages = array_intersect_key($componentProtoNamespaces, $protoNamespaces)) {
+                foreach ($commonPackages as $package => $namespace) {
+                    if ($namespace !== $protoNamespaces[$package]) {
+                        throw new RuntimeException(sprintf(
+                            'Package "%s" has conflicting namespaces: "%s" and "%s"',
+                            $package,
+                            $namespace,
+                            $protoNamespaces[$package]
+                        ));
+                    }
+                }
+            }
+            $protoNamespaces = array_merge($protoNamespaces, $componentProtoNamespaces);
+        }
+
+        return $protoNamespaces;
+    }
+
+    public function getProtoPaths(): array
+    {
+        return array_map(fn($v) => $v->getProtoPath(), $this->getComponentPackages());
     }
 
     public function getServiceAddresses(): array

diff --git a/dev/src/ComponentPackage.php b/dev/src/ComponentPackage.php
@@ -48,7 +48,7 @@ public function getName(): string
         return $this->name;
     }
 
-    public function getProtoPackage(): string
+    public function getProtoPath(): string
     {
         $gapicClientFiles = $this->getV1GapicClientFiles() + $this->getV2ClientFiles();
 
@@ -78,11 +78,7 @@ public function getServiceAddress(): string
         $gapicClientClasses = array_map(fn ($fp) => $this->getClassFromFile($fp), $gapicClientFiles);
 
         foreach ($gapicClientClasses as $className) {
-            // Access V1-surface public constant
-            if (defined($className . '::SERVICE_ADDRESS')) {
-                return constant($className . '::SERVICE_ADDRESS');
-            }
-            // Access V2-surface private constant
+            // Access private constants (for v2 surfaces)
             if ($constants = (new \ReflectionClass($className))->getConstants()) {
                 if (isset($constants['SERVICE_ADDRESS'])) {
                     return $constants['SERVICE_ADDRESS'];
@@ -92,6 +88,22 @@ public function getServiceAddress(): string
         return '';
     }
 
+    public function getProtoNamespaces(): array
+    {
+        $protoPackages = [];
+        foreach ($this->getFilesInDir('*.php', $this->path) as $classFile) {
+            $contents = file_get_contents($classFile);
+            if (preg_match(
+                '/Generated from protobuf message <code>([a-z0-9\.]+)(\..*)<\/code>/',
+                $contents,
+                $matches
+            ) && preg_match('/namespace (.*);/', $contents, $nsMatches)) {
+                $protoPackages[$matches[1]] = str_replace(str_replace('.', '\\', $matches[2]), '', $nsMatches[1]);
+            }
+        }
+        return array_unique($protoPackages);
+    }
+
     public function getBaseUri(): string
     {
         if (file_exists($this->path . 'Connection/Rest.php')) {

diff --git a/dev/src/DocFx/Node/ClassNode.php b/dev/src/DocFx/Node/ClassNode.php
@@ -28,11 +28,11 @@ class ClassNode
     use NameTrait;
 
     private $childNode;
-    private array $protoPackages;
     private string $tocName;
 
     public function __construct(
-        private SimpleXMLElement $xmlNode
+        private SimpleXMLElement $xmlNode,
+        private array $protoPackages = [],
     ) {}
 
     public function isProtobufEnumClass(): bool
@@ -247,14 +247,6 @@ public function getProtoPackage(): ?string
         return null;
     }
 
-    public function setProtoPackages(array $protoPackages)
-    {
-        $this->protoPackages = $protoPackages;
-        if ($this->childNode) {
-            $this->childNode->setProtoPackages($protoPackages);
-        }
-    }
-
     public function getTocName()
     {
         return isset($this->tocName) ? $this->tocName : $this->getName();

diff --git a/dev/src/DocFx/Page/PageTree.php b/dev/src/DocFx/Page/PageTree.php
@@ -35,7 +35,8 @@ public function __construct(
         private string $xmlPath,
         private string $namespace,
         private string $packageDescription,
-        private string $componentPath
+        private string $componentPath,
+        private array $componentPackages
     ) {}
 
     public function getPages(): array
@@ -65,7 +66,7 @@ private function loadPages(): array
                 continue;
             }
 
-            $classNode = new ClassNode($file->class[0]);
+            $classNode = new ClassNode($file->class[0], $this->componentPackages);
 
             // Skip the protobuf classes with underscores, they're all deprecated
             // @TODO: Do not generate them in V2
@@ -149,27 +150,6 @@ private function loadPages(): array
             }
         }
 
-        /**
-         * Set a map of protobuf package names to PHP namespaces for Xrefs.
-         * This MUST be done after combining GAPIC clients.
-         */
-        $protoPackages = [
-            // shared packages
-            'google.longrunning' => 'Google\\LongRunning'
-        ];
-        foreach ($pages as $page) {
-            $classNode = $page->getClassNode();
-            if ($protoPackage = $classNode->getProtoPackage()) {
-                $package = rtrim(ltrim($classNode->getNamespace(), '\\'), '\\Client');
-                $protoPackages[$protoPackage] = $package;
-            }
-        }
-
-        // Add the proto packages to every class node
-        foreach ($pages as $page) {
-            $page->getClassNode()->setProtoPackages($protoPackages);
-        }
-
         // Sort pages alphabetically by full class name
         ksort($pages);
 

diff --git a/dev/tests/Unit/ComponentTest.php b/dev/tests/Unit/ComponentTest.php
@@ -69,9 +69,27 @@ public function provideComponentProperties()
                 [
                     'id' => 'cloud-talent',
                     'apiVersions' => ['V4'],
-                    'protoPackages' => ['google/cloud/talent/v4'],
+                    'protoPaths' => ['google/cloud/talent/v4'],
                 ]
             ]
         ];
     }
+
+    public function testGetProtoNamespaces()
+    {
+        // ensure there are no conflicts - this would throw an exception
+        $allProtoNamespaces = Component::getProtoPackageToNamespaceMap();
+
+        // verify a few are as expected
+        $this->assertEquals('Google\Cloud\Bigtable\V2', $allProtoNamespaces['google.bigtable.v2']);
+        $this->assertEquals('Google\Cloud\Talent\V4', $allProtoNamespaces['google.cloud.talent.v4']);
+        $this->assertEquals('Grafeas\V1', $allProtoNamespaces['grafeas.v1']);
+        $this->assertEquals('Google\Cloud\Workflows\Executions\V1', $allProtoNamespaces['google.cloud.workflows.executions.v1']);
+        $this->assertEquals('Google\Api', $allProtoNamespaces['google.api']);
+        $this->assertEquals('Google\Cloud\Location', $allProtoNamespaces['google.cloud.location']);
+        $this->assertEquals('Google\LongRunning', $allProtoNamespaces['google.longrunning']);
+        $this->assertEquals('Google\Rpc\Context\AttributeContext', $allProtoNamespaces['google.rpc.context']);
+        $this->assertEquals('Google\Cloud\Iam\V1', $allProtoNamespaces['google.iam.v1']);
+        $this->assertEquals('Google\Cloud\Logging\Type', $allProtoNamespaces['google.logging.type']);
+    }
 }
diff --git a/dev/tests/Unit/DocFx/NodeTest.php b/dev/tests/Unit/DocFx/NodeTest.php
@@ -284,16 +284,15 @@ public function testProtoRefWithXrefUsingPackageName()
         $description = '[ListBackups][google.bigtable.admin.v2.BigtableTableAdmin.ListBackups]';
         $protoPackages = ['google.bigtable.admin.v2' => 'Google\\Cloud\\Bigtable\\Admin\\V2'];
 
-        $xref = new class {
+        $xref = new class($protoPackages) {
             use XrefTrait;
-            public $protoPackages;
+            public function __construct(private array $protoPackages) {
+            }
             public function replace(string $description) {
                 return $this->replaceProtoRef($description);
             }
         };
 
-        $xref->protoPackages = $protoPackages;
-
         $this->assertEquals(
             '<xref uid="\Google\Cloud\Bigtable\Admin\V2\Client\BigtableTableAdminClient::listBackups()">ListBackups</xref>',
             $xref->replace($description)
@@ -302,9 +301,7 @@ public function replace(string $description) {
         $classNode = new ClassNode(new SimpleXMLElement(sprintf(
             '<class><docblock><description>%s</description></docblock></class>',
             $description
-        )));
-
-        $classNode->setProtoPackages($protoPackages);
+        )), $protoPackages);
 
         $this->assertEquals(
             '<xref uid="\Google\Cloud\Bigtable\Admin\V2\Client\BigtableTableAdminClient::listBackups()">ListBackups</xref>',