feat(docs): add support for cross-package references

.kokoro/docs/publish.sh

@@ -37,13 +37,15 @@ do
             --out $DIR/out \
             --metadata-version $VERSION \
             --staging-bucket $STAGING_BUCKET \
+            --with-cache \
             $VERBOSITY
     else
         # dry run
         $PROJECT_DIR/dev/google-cloud docfx \
             --component $COMPONENT \
             --out $DIR/out \
             --metadata-version $VERSION \
+            --with-cache \
             $VERBOSITY
     fi
 done

composer.json

@@ -51,7 +51,7 @@
         "psr/http-message": "^1.0|^2.0",
         "ramsey/uuid": "^4.0",
         "google/gax": "^1.34.0",
-        "google/auth": "^1.34"
+        "google/auth": "^1.42"
     },
     "require-dev": {
         "phpunit/phpunit": "^9.6",

dev/src/Command/ComponentInfoCommand.php

@@ -44,6 +44,8 @@ class ComponentInfoCommand extends Command
         'php_namespaces' => 'Php Namespace',
         'github_repo' => 'Github Repo',
         'proto_path' => 'Proto Path',
+        'proto_packages' => 'Proto Packages',
+        'proto_namespaces' => 'Proto Namespaces',
         'service_address' => 'Service Address',
         'api_shortname' => 'API Shortname',
         'description' => 'Description',
@@ -216,7 +218,7 @@ private function getComponentDetailRow(
             'migration_mode' => $package ? $package->getMigrationStatus() : implode(",", $component->getMigrationStatuses()),
             'php_namespaces' => implode(",", array_keys($component->getNamespaces())),
             'github_repo' => $component->getRepoName(),
-            'proto_path' => $package ? $package->getProtoPackage() : implode(",", $component->getProtoPackages()),
+            'proto_path' => $package ? $package->getProtoPath() : implode(",", $component->getProtoPaths()),
             'service_address' => $package ? $package->getServiceAddress() : implode(",", $component->getServiceAddresses()),
             'api_shortname' => $package ? $package->getApiShortname() : implode(",", array_filter($component->getApiShortnames())),
             'description' => $component->getDescription(),
@@ -239,6 +241,22 @@ private function getComponentDetailRow(
         if (array_key_exists('downloads', $requestedFields)) {
             $row['downloads'] = number_format($this->packagist->getDownloads($component->getPackageName()));
         }
+        if (
+            array_key_exists('proto_namespaces', $requestedFields)
+            || array_key_exists('proto_packages', $requestedFields)
+        ) {
+            $protoNamespaces = $component->getProtoNamespaces();
+            if (array_key_exists('proto_packages', $requestedFields)) {
+                $row['proto_packages'] = implode(",", array_keys($protoNamespaces));
+            }
+            if (array_key_exists('proto_namespaces', $requestedFields)) {
+                $row['proto_namespaces'] = implode("\n", array_map(
+                    fn ($key, $value) => $key . ' => ' . $value,
+                    array_keys($protoNamespaces),
+                    array_values($protoNamespaces)
+                ));
+            }
+        }
         // call again in case the filters were on the slow fields
         if ($this->filterRow($row, $filters)) {
             return null;
@@ -248,7 +266,7 @@ private function getComponentDetailRow(
 
     private function getAvailableApiVersions(Component $component): string
     {
-        $protos = $component->getProtoPackages();
+        $protos = $component->getProtoPaths();
         $proto = array_shift($protos);
         // Proto packages should be in a version directory
         $versionPath = dirname($proto);

dev/src/Command/DocFxCommand.php

@@ -25,6 +25,7 @@
 use Symfony\Component\Process\Process;
 use Symfony\Component\Yaml\Yaml;
 use RuntimeException;
+use Google\Auth\Cache\FileSystemCacheItemPool;
 use Google\Cloud\Dev\Component;
 use Google\Cloud\Dev\DocFx\Node\ClassNode;
 use Google\Cloud\Dev\DocFx\Page\PageTree;
@@ -64,6 +65,7 @@ protected function configure()
                 InputOption::VALUE_OPTIONAL,
                 'Specify the path of the desired component. Please note, this option is only intended for testing purposes.'
             )
+            ->addOption('--with-cache', '', InputOption::VALUE_NONE, 'Cache expensive proto namespace lookups to a file')
         ;
     }
 
@@ -108,12 +110,14 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $tocItems = [];
         $packageDescription = $component->getDescription();
         $isBeta = 'stable' !== $component->getReleaseLevel();
+        $packageNamespaces = $this->getProtoPackageToNamespaceMap($input->getOption('with-cache'));
         foreach ($component->getNamespaces() as $namespace => $dir) {
             $pageTree = new PageTree(
                 $xml,
                 $namespace,
                 $packageDescription,
-                $component->getPath()
+                $component->getPath(),
+                $packageNamespaces
             );
 
             foreach ($pageTree->getPages() as $page) {
@@ -174,15 +178,20 @@ protected function execute(InputInterface $input, OutputInterface $output)
 
         if ($metadataVersion = $input->getOption('metadata-version')) {
             $output->write(sprintf('Writing docs.metadata with version <fg=white>%s</>... ', $metadataVersion));
+            $xrefs = array_merge(...array_map(
+                fn ($c) => ['--xrefs', sprintf('devsite://php/%s', $c->getId())],
+                $component->getComponentDependencies(),
+            ));
             $process = new Process([
                 'docuploader', 'create-metadata',
-                '--name', str_replace('google/', '', $component->getPackageName()),
+                '--name', $component->getId(),
                 '--version', $metadataVersion,
                 '--language', 'php',
                 '--distribution-name', $component->getPackageName(),
                 '--product-page', $component->getProductDocumentation(),
                 '--github-repository', $component->getRepoName(),
                 '--issue-tracker', $component->getIssueTracker(),
+                ...$xrefs,
                 $outDir . '/docs.metadata'
             ]);
             $process->mustRun();
@@ -262,4 +271,21 @@ private function validate(ClassNode $class, OutputInterface $output): bool
         }
         return $valid;
     }
+
+    private function getProtoPackageToNamespaceMap(bool $useFileCache): array
+    {
+        if (!$useFileCache) {
+            return Component::getProtoPackageToNamespaceMap();
+        }
+
+        $cache = new FileSystemCacheItemPool('.cache');
+        $item = $cache->getItem('phpdoc_proto_package_to_namespace_map');
+
+        if (!$item->isHit()) {
+            $item->set(Component::getProtoPackageToNamespaceMap());
+            $cache->save($item);
+        }
+
+        return $item->get();
+    }
 }

dev/src/Component.php

@@ -37,6 +37,8 @@ class Component
     private string $clientDocumentation;
     private string $description;
     private array $namespaces;
+    /** @var array<Component> */
+    private array $componentDependencies;
 
     public function __construct(private string $name, string $path = null)
     {
@@ -211,15 +213,32 @@ private function validateComponentFiles(): void
         $this->clientDocumentation = $repoMetadataJson['client_documentation'];
         $this->productDocumentation = $repoMetadataJson['product_documentation'] ?? '';
 
+        $namespaces = [];
         foreach ($composerJson['autoload']['psr-4'] as $namespace => $dir) {
-            if (0 === strpos($dir, 'src')) {
+            if (str_starts_with($dir, 'src')) {
                 $namespaces[rtrim($namespace, '\\')] = $dir;
             }
         }
         if (empty($namespaces)) {
             throw new RuntimeException('composer autoload.psr-4 does not contain a namespace');
         }
         $this->namespaces = $namespaces;
+
+        // find dependencies which are google/cloud components
+        $this->componentDependencies = [];
+        foreach ($composerJson['require'] ?? [] as $name => $version) {
+            if ($componentName = key(array_filter(
+                $repoMetadataFullJson,
+                fn ($metadata) => $metadata['distribution_name'] === $name
+            ))) {
+                $this->componentDependencies[] = new Component($componentName);
+            }
+        }
+        if (isset($composerJson['require']['google/gax'])
+            && !isset($composerJson['require']['google/common-protos'])
+        ) {
+            $this->componentDependencies[] = new Component('CommonProtos');
+        }
     }
 
     /**
@@ -245,9 +264,51 @@ public function getMigrationStatuses(): array
         return array_map(fn($v) => $v->getMigrationStatus(), $this->getComponentPackages());
     }
 
-    public function getProtoPackages(): array
+    public function getProtoNamespaces(): array
     {
-        return array_map(fn($v) => $v->getProtoPackage(), $this->getComponentPackages());
+        $protoNamespaces = [];
+        $componentPackages = $this->getComponentPackages();
+        foreach ($this->namespaces as $namespace => $dir) {
+            $componentPackages = $dir === 'src'
+                ? $this->getComponentPackages()
+                : [new ComponentPackage($this, str_replace('src/', '', $dir))];
+
+            $protoNamespaces = array_reduce(
+                $componentPackages,
+                fn($protoNamespaces, $pkg) => array_merge($protoNamespaces, $pkg->getProtoNamespaces()),
+                $protoNamespaces
+            );
+        }
+
+        return $protoNamespaces;
+    }
+
+    public static function getProtoPackageToNamespaceMap(): array
+    {
+        $protoNamespaces = [];
+        foreach (self::getComponents() as $component) {
+            $componentProtoNamespaces = $component->getProtoNamespaces();
+            if ($commonPackages = array_intersect_key($componentProtoNamespaces, $protoNamespaces)) {
+                foreach ($commonPackages as $package => $namespace) {
+                    if ($namespace !== $protoNamespaces[$package]) {
+                        throw new RuntimeException(sprintf(
+                            'Package "%s" has conflicting namespaces: "%s" and "%s"',
+                            $package,
+                            $namespace,
+                            $protoNamespaces[$package]
+                        ));
+                    }
+                }
+            }
+            $protoNamespaces = array_merge($protoNamespaces, $componentProtoNamespaces);
+        }
+
+        return $protoNamespaces;
+    }
+
+    public function getProtoPaths(): array
+    {
+        return array_map(fn($v) => $v->getProtoPath(), $this->getComponentPackages());
     }
 
     public function getServiceAddresses(): array
@@ -284,4 +345,9 @@ private function getPackagePaths(): array
         }
         return array_reverse($paths);
     }
+
+    public function getComponentDependencies(): array
+    {
+        return $this->componentDependencies;
+    }
 }

dev/src/ComponentPackage.php

@@ -48,7 +48,7 @@ public function getName(): string
         return $this->name;
     }
 
-    public function getProtoPackage(): string
+    public function getProtoPath(): string
     {
         $gapicClientFiles = $this->getV1GapicClientFiles() + $this->getV2ClientFiles();
 
@@ -78,11 +78,7 @@ public function getServiceAddress(): string
         $gapicClientClasses = array_map(fn ($fp) => $this->getClassFromFile($fp), $gapicClientFiles);
 
         foreach ($gapicClientClasses as $className) {
-            // Access V1-surface public constant
-            if (defined($className . '::SERVICE_ADDRESS')) {
-                return constant($className . '::SERVICE_ADDRESS');
-            }
-            // Access V2-surface private constant
+            // Access private constants (for v2 surfaces)
             if ($constants = (new \ReflectionClass($className))->getConstants()) {
                 if (isset($constants['SERVICE_ADDRESS'])) {
                     return $constants['SERVICE_ADDRESS'];
@@ -92,6 +88,27 @@ public function getServiceAddress(): string
         return '';
     }
 
+    public function getProtoNamespaces(): array
+    {
+        $protoPackages = [];
+        foreach ($this->getFilesInDir('*.php', $this->path) as $classFile) {
+            $contents = file_get_contents($classFile);
+            if (preg_match(
+                '/Generated from protobuf message <code>([a-z0-9\.]+)(\..*)<\/code>/',
+                $contents,
+                $matches
+            ) && preg_match('/namespace (.*);/', $contents, $nsMatches)) {
+                // remove namespace (in case it's nested)
+                $protoPackages[$matches[1]] = str_replace(
+                    str_replace('.', '\\', substr($matches[2], 0, strrpos($matches[2], '.'))),
+                    '',
+                    $nsMatches[1]
+                );
+            }
+        }
+        return array_unique($protoPackages);
+    }
+
     public function getBaseUri(): string
     {
         if (file_exists($this->path . 'Connection/Rest.php')) {

dev/src/DocFx/Node/ClassNode.php

@@ -28,11 +28,11 @@ class ClassNode
     use NameTrait;
 
     private $childNode;
-    private array $protoPackages;
     private string $tocName;
 
     public function __construct(
-        private SimpleXMLElement $xmlNode
+        private SimpleXMLElement $xmlNode,
+        private array $protoPackages = [],
     ) {}
 
     public function isProtobufEnumClass(): bool
@@ -247,14 +247,6 @@ public function getProtoPackage(): ?string
         return null;
     }
 
-    public function setProtoPackages(array $protoPackages)
-    {
-        $this->protoPackages = $protoPackages;
-        if ($this->childNode) {
-            $this->childNode->setProtoPackages($protoPackages);
-        }
-    }
-
     public function getTocName()
     {
         return isset($this->tocName) ? $this->tocName : $this->getName();

dev/src/DocFx/Page/PageTree.php

@@ -35,7 +35,8 @@ public function __construct(
         private string $xmlPath,
         private string $namespace,
         private string $packageDescription,
-        private string $componentPath
+        private string $componentPath,
+        private array $componentPackages
     ) {}
 
     public function getPages(): array
@@ -65,7 +66,7 @@ private function loadPages(): array
                 continue;
             }
 
-            $classNode = new ClassNode($file->class[0]);
+            $classNode = new ClassNode($file->class[0], $this->componentPackages);
 
             // Skip the protobuf classes with underscores, they're all deprecated
             // @TODO: Do not generate them in V2
@@ -149,27 +150,6 @@ private function loadPages(): array
             }
         }
 
-        /**
-         * Set a map of protobuf package names to PHP namespaces for Xrefs.
-         * This MUST be done after combining GAPIC clients.
-         */
-        $protoPackages = [
-            // shared packages
-            'google.longrunning' => 'Google\\LongRunning'
-        ];
-        foreach ($pages as $page) {
-            $classNode = $page->getClassNode();
-            if ($protoPackage = $classNode->getProtoPackage()) {
-                $package = rtrim(ltrim($classNode->getNamespace(), '\\'), '\\Client');
-                $protoPackages[$protoPackage] = $package;
-            }
-        }
-
-        // Add the proto packages to every class node
-        foreach ($pages as $page) {
-            $page->getClassNode()->setProtoPackages($protoPackages);
-        }
-
         // Sort pages alphabetically by full class name
         ksort($pages);