Merge pull request #98 from keboola/adamvyborny-CM-782-retry-on-git-c…

…lone-race-codinition Retry on git clone race condition
keboola · Nov 6, 2023 · fa72b19 · fa72b19
2 parents c5677e1 + 47b02a5
commit fa72b19
Show file tree

Hide file tree

Showing 28 changed files with 982 additions and 589 deletions.
diff --git a/composer.json b/composer.json
@@ -8,8 +8,9 @@
         "ext-odbc": "*",
         "keboola/db-adapter-snowflake": "^1.1",
         "keboola/php-component": "^9.1",
+        "keboola/retry": "^0.5",
         "keboola/sandboxes-api-php-client": "^6.21",
-        "keboola/storage-api-client": "^12.10",
+        "keboola/storage-api-client": "^14.12",
         "keboola/table-backend-utils": "^1.11",
         "symfony/filesystem": "^5.4",
         "symfony/process": "^5.4",

diff --git a/composer.lock b/composer.lock
diff --git a/src/Component.php b/src/Component.php
@@ -29,7 +29,10 @@
 use Keboola\SnowflakeDbAdapter\Connection;
 use Keboola\StorageApi\Client as StorageClient;
 use Psr\Log\LoggerInterface;
+use Retry\Policy\CallableRetryPolicy;
+use Retry\RetryProxy;
 use Symfony\Component\Config\Definition\Exception\InvalidConfigurationException;
+use Symfony\Component\Process\Exception\ProcessFailedException;
 use Symfony\Component\Yaml\Yaml;
 
 class Component extends BaseComponent
@@ -47,23 +50,28 @@ class Component extends BaseComponent
     public function __construct(LoggerInterface $logger)
     {
         parent::__construct($logger);
+
+        $retryProxy = new RetryProxy(new CallableRetryPolicy(function (ProcessFailedException $e) {
+            return str_contains($e->getMessage(), 'shallow file has changed since we read it');
+        }));
+
         $this->createProfilesFileService = new DbtProfilesYaml;
         $this->createSourceFileService = new DbtSourcesYaml;
-        $this->gitRepositoryService = new GitRepositoryService($this->getDataDir());
+        $this->gitRepositoryService = new GitRepositoryService($this->getDataDir(), $retryProxy);
         $this->storageClient = new StorageClient([
             'url' => $this->getConfig()->getStorageApiUrl(),
             'token' => $this->getConfig()->getStorageApiToken(),
         ]);
         $this->artifacts = new ArtifactsService(
             $this->storageClient,
             $this->getDataDir(),
-            $this->getConfig()->getArtifactsOptions()
+            $this->getConfig()->getArtifactsOptions(),
         );
         $this->setProjectPath($this->getDataDir());
         $this->dwhProviderFactory = new DwhProviderFactory(
             $this->createSourceFileService,
             $this->createProfilesFileService,
-            $this->getLogger()
+            $this->getLogger(),
         );
     }
 
@@ -104,7 +112,7 @@ public function getOutputManifest(array $workspaceCredentials): OutputManifest
         $manifestConverter = new DbtManifestParser($this->projectPath);
         $connectionConfig = array_intersect_key(
             $workspaceCredentials,
-            array_flip(['host', 'warehouse', 'database', 'user', 'password'])
+            array_flip(['host', 'warehouse', 'database', 'user', 'password']),
         );
         $connection = new Connection($connectionConfig);
 
@@ -118,7 +126,7 @@ public function getOutputManifest(array $workspaceCredentials): OutputManifest
             $manifestManager,
             $manifestConverter,
             $this->getLogger(),
-            $quoteIdentifier
+            $quoteIdentifier,
         );
     }
 
@@ -168,7 +176,7 @@ protected function cloneRepository(Config $config): void
             $config->getGitRepositoryUrl(),
             $config->getGitRepositoryBranch(),
             $config->getGitRepositoryUsername(),
-            $config->getGitRepositoryPassword()
+            $config->getGitRepositoryPassword(),
         );
 
         $branch = $this->gitRepositoryService->getCurrentBranch($this->projectPath);
@@ -177,7 +185,7 @@ protected function cloneRepository(Config $config): void
             'Successfully cloned repository %s from branch %s (%s)',
             $config->getGitRepositoryUrl(),
             $branch['name'],
-            $branch['ref']
+            $branch['ref'],
         ));
     }
 
@@ -335,7 +343,7 @@ protected function actionGitRepository(): array
             $config->getGitRepositoryUrl(),
             $config->getGitRepositoryBranch(),
             $config->getGitRepositoryUsername(),
-            $config->getGitRepositoryPassword()
+            $config->getGitRepositoryPassword(),
         );
 
         $branches = $this->gitRepositoryService->listRemoteBranches($this->projectPath);
@@ -367,7 +375,7 @@ protected function loadConfig(): void
             /** @var BaseConfig $config */
             $config = new $configClass(
                 $rawConfig,
-                new $configDefinitionClass()
+                new $configDefinitionClass(),
             );
             $this->config = $config;
         } catch (InvalidConfigurationException $e) {

diff --git a/src/Configuration/NodeDefinition/DbtNode.php b/src/Configuration/NodeDefinition/DbtNode.php
@@ -46,18 +46,18 @@ protected function build(): void
                     ->always(function ($executeSteps) {
                         if (empty($executeSteps)) {
                             throw new InvalidConfigurationException(
-                                'At least one execute step must be defined'
+                                'At least one execute step must be defined',
                             );
                         }
                         foreach ($executeSteps as $input) {
                             if (substr($input['step'], 0, 4) !== 'dbt ') {
                                 throw new InvalidConfigurationException(
-                                    'Invalid execute step: Command must start with "dbt"'
+                                    'Invalid execute step: Command must start with "dbt"',
                                 );
                             }
                             if (preg_match('/[|&]/', $input['step'])) {
                                 throw new InvalidConfigurationException(
-                                    'Invalid execute step: Command contains disallowed metacharacters'
+                                    'Invalid execute step: Command contains disallowed metacharacters',
                                 );
                             }
                         }

diff --git a/src/DwhProvider/DwhProviderFactory.php b/src/DwhProvider/DwhProviderFactory.php
@@ -27,7 +27,7 @@ class DwhProviderFactory
     public function __construct(
         DbtSourcesYaml $createSourceFileService,
         DbtProfilesYaml $createProfilesFileService,
-        LoggerInterface $logger
+        LoggerInterface $logger,
     ) {
         $this->createProfilesFileService = $createProfilesFileService;
         $this->createSourceFileService = $createSourceFileService;
@@ -72,7 +72,7 @@ public function getProvider(Config $config, string $projectPath): DwhProviderInt
             $this->createProfilesFileService,
             $this->logger,
             $config,
-            $projectPath
+            $projectPath,
         );
     }
 }
diff --git a/src/DwhProvider/LocalSnowflakeProvider.php b/src/DwhProvider/LocalSnowflakeProvider.php
@@ -30,7 +30,7 @@ public function __construct(
         DbtProfilesYaml $createProfilesFileService,
         LoggerInterface $logger,
         Config $config,
-        string $projectPath
+        string $projectPath,
     ) {
         $this->createProfilesFileService = $createProfilesFileService;
         $this->createSourceFileService = $createSourceFileService;
@@ -79,7 +79,7 @@ public function createDbtYamlFiles(array $configurationNames = []): void
             $this->createSourceFileService->dumpYaml(
                 $this->projectPath,
                 $tablesData,
-                $this->config->getFreshness()
+                $this->config->getFreshness(),
             );
         }
     }

diff --git a/src/DwhProvider/RemoteBigQueryProvider.php b/src/DwhProvider/RemoteBigQueryProvider.php
@@ -21,14 +21,14 @@ public function __construct(
         DbtProfilesYaml $createProfilesFileService,
         LoggerInterface $logger,
         Config $config,
-        string $projectPath
+        string $projectPath,
     ) {
         parent::__construct(
             $createSourceFileService,
             $createProfilesFileService,
             $logger,
             $config,
-            $projectPath
+            $projectPath,
         );
 
         $this->temp = new Temp('dbt-big-query');

diff --git a/src/DwhProvider/RemoteSnowflakeProvider.php b/src/DwhProvider/RemoteSnowflakeProvider.php
@@ -31,7 +31,7 @@ public function createDbtYamlFiles(array $configurationNames = []): void
     {
         $this->createProfilesFileService->dumpYaml(
             $this->projectPath,
-            $this->getOutputs($configurationNames, static::getDbtParams())
+            $this->getOutputs($configurationNames, static::getDbtParams()),
         );
         $this->setEnvVars();
 

diff --git a/src/FileDumper/DbtProfilesYaml.php b/src/FileDumper/DbtProfilesYaml.php
@@ -31,7 +31,7 @@ public function dumpYaml(string $projectPath, array $outputs): void
                     'target' => 'dev',
                     'outputs' => $outputs,
                 ],
-            ], 5)
+            ], 5),
         );
     }
 }
diff --git a/src/FileDumper/DbtSourcesYaml.php b/src/FileDumper/DbtSourcesYaml.php
@@ -16,7 +16,7 @@ class DbtSourcesYaml extends FilesystemAwareDumper
     public function dumpYaml(
         string $projectPath,
         array $tablesData,
-        array $freshness
+        array $freshness,
     ): void {
         $modelFolderPath = sprintf('%s/models/_sources/', $projectPath);
         $this->createFolderIfNotExist($modelFolderPath);
@@ -32,14 +32,14 @@ public function dumpYaml(
                             'freshness' => $freshness,
                             'database' => sprintf(
                                 '{{ env_var("DBT_KBC_PROD%s_DATABASE") }}',
-                                isset($tables['projectId']) ? ('_' . $tables['projectId']) : ''
+                                isset($tables['projectId']) ? ('_' . $tables['projectId']) : '',
                             ),
                             'schema' => $bucket,
                             'loaded_at_field' => '"_timestamp"',
                             'tables' => array_map($this->formatTableSources(), $tables['tables']),
                         ],
                     ],
-                ], 8)
+                ], 8),
             );
         }
     }
@@ -64,7 +64,7 @@ static function ($primaryColumn) {
                             'tests' => ['unique', 'not_null'],
                         ];
                     },
-                    $table['primaryKey']
+                    $table['primaryKey'],
                 );
             }
 

diff --git a/src/FileDumper/OutputManifest.php b/src/FileDumper/OutputManifest.php
@@ -36,7 +36,7 @@ public function __construct(
         ManifestManager $manifestManager,
         DbtManifestParser $dbtManifestParser,
         LoggerInterface $logger,
-        bool $quoteIdentifier = false
+        bool $quoteIdentifier = false,
     ) {
         $this->manifestManager = $manifestManager;
         $this->databaseConfig = $databaseConfig;
@@ -67,7 +67,7 @@ public function dump(): void
                 $columnName = $column->getColumnName();
                 $columnsMetadata->{$columnName} = array_merge(
                     $column->getColumnDefinition()->toMetadata(),
-                    (array) ($dbtColumnsMetadata[strtolower($columnName)] ?? [])
+                    (array) ($dbtColumnsMetadata[strtolower($columnName)] ?? []),
                 );
             }
 
@@ -89,7 +89,7 @@ public function dump(): void
                 ->setColumnMetadata($columnsMetadata)
                 ->setPrimaryKeyColumns($this->getPrimaryKeyColumnNames(
                     $dbtPrimaryKey,
-                    $tableDef->getColumnsNames()
+                    $tableDef->getColumnsNames(),
                 ))
             ;
 
@@ -129,21 +129,21 @@ private function getTables(array $sourceTables): array
                  *     kind: string,
                  *     type: string,
                  *     default: string,
-                 *     'null?': string
+                 *     "null?": string
                  * }> $columnsMeta */
                 $columnsMeta = $this->connection->fetchAll(
                     sprintf(
                         'DESC TABLE %s',
                         SnowflakeQuote::createQuotedIdentifierFromParts([
                             $schema,
                             $this->quoteIdentifier ? $tableName : strtoupper($tableName),
-                        ])
-                    )
+                        ]),
+                    ),
                 );
             } catch (RuntimeException $e) {
                 // do nothing for models/tables not existing in the DB
                 $this->logger->warning(
-                    'Table "%s" specified in dbt manifest was not found in the database.'
+                    'Table "%s" specified in dbt manifest was not found in the database.',
                 );
                 continue;
             }
@@ -160,7 +160,7 @@ private function getTables(array $sourceTables): array
                 $tableName,
                 false,
                 new ColumnCollection($columns),
-                []
+                [],
             );
         }
 

diff --git a/src/FileDumper/OutputManifest/DbtManifestParser.php b/src/FileDumper/OutputManifest/DbtManifestParser.php
@@ -32,7 +32,7 @@ public function parse(): array
                 (string) file_get_contents($this->sourceManifestPath),
                 true,
                 512,
-                JSON_THROW_ON_ERROR
+                JSON_THROW_ON_ERROR,
             );
 
             /** @var array<string, array<string, string|array<string, array<string, array<string, mixed>>>>> $modelNodes */
@@ -41,7 +41,7 @@ public function parse(): array
                 function ($node) {
                     /** @var array<string, mixed> $node */
                     return $node['resource_type'] === 'model';
-                }
+                },
             );
 
             foreach ($modelNodes as $tableData) {

diff --git a/src/Helper/DbtCompileHelper.php b/src/Helper/DbtCompileHelper.php
@@ -23,7 +23,7 @@ public static function getCompiledSqlFilesContent(string $directory): array
         reset($filePaths);
 
         $contents = array_map(fn($sqlFile) => trim(
-            (string) file_get_contents($sqlFile->getPathname())
+            (string) file_get_contents($sqlFile->getPathname()),
         ), $filePaths);
 
         $combineArray = (array) array_combine($filenames, $contents);
@@ -39,7 +39,7 @@ public static function getCompiledSqlFilesContent(string $directory): array
     public static function getCompiledSqlPaths(string $directory): array
     {
         $compiledDirInfo = new SplFileInfo(
-            sprintf('%s/%s', $directory, 'compiled')
+            sprintf('%s/%s', $directory, 'compiled'),
         );
 
         try {

diff --git a/src/Helper/DbtDocsHelper.php b/src/Helper/DbtDocsHelper.php
@@ -12,7 +12,7 @@ public static function mergeHtml(string $html, string $catalogJson, string $mani
         $newStr = sprintf(
             'o=[{label: \'manifest\', data: %s},{label: \'catalog\', data: %s}]',
             $manifestJson,
-            $catalogJson
+            $catalogJson,
         );
 
         return (string) str_replace($searchStr, $newStr, $html);

diff --git a/src/Helper/ParseLogFileHelper.php b/src/Helper/ParseLogFileHelper.php
@@ -38,7 +38,7 @@ public function getSqls(): Generator
         foreach ($logs as $log) {
             if (!empty($log['data']['sql'])) {
                 yield $this->queryExcerpt(
-                    trim((string) preg_replace('!/\*.*?\*/!s', '', $log['data']['sql']))
+                    trim((string) preg_replace('!/\*.*?\*/!s', '', $log['data']['sql'])),
                 );
             }
         }
-Original file line number
+Diff line change
@@ Expand Up @@
                         'target' => 'dev',
                         'outputs' => $outputs,
                     ],
-                ], 5)
+                ], 5),
             );
         }
     }