Add EntityCleanupCommand to delete cron, clean api calls or other entity easily through a bundle configuration

CHANGELOG.md

@@ -1,5 +1,12 @@
 CHANGELOG for 1.x
 ===================
+## v1.13.0 - (2024-09-24)
+### Added
+- `EntityCleanupCommand` to delete cron, clean api calls or other entity easily through the bundle configuration **entity_cleanup_command_configs**
+
+### Changed
+- `UpdatableInterface::getUpdatedAt` and `UpdatableInterface::setUpdatedAt` handle nullable datetime
+
 ## v1.12.0 - (2024-09-23)
 ### Added
 - `ArchivableInterface` & trait

composer.json

@@ -22,6 +22,7 @@
     "symfony/form": "^5.4|^6.2",
     "symfony/framework-bundle": "^5.4|^6.2",
     "symfony/security-bundle": "^5.4|^6.2",
+    "symfony/translation": "^5.4|^6.2",
     "theofidry/alice-data-fixtures": "^1.5"
   },
   "require-dev": {

config/services.yaml

@@ -3,6 +3,13 @@ services:
     # Service Fidry\AliceDataFixtures\Loader\PurgerLoader not exist, and it must be aliased. AbstractFixtures need it.
     Fidry\AliceDataFixtures\Loader\PurgerLoader: '@fidry_alice_data_fixtures.doctrine.purger_loader'
     # Command
+    Smart\CoreBundle\Command\EntityCleanupCommand:
+        arguments:
+            - '@Smart\CoreBundle\Monitoring\ProcessMonitor'
+            - '@Smart\CoreBundle\Config\IniOverrideConfig'
+            - '@Doctrine\ORM\EntityManagerInterface'
+            - '@translator'
+        tags: [ 'console.command' ]
     Smart\CoreBundle\Command\CommandPoolHelper:
         arguments:
             - '@Symfony\Component\HttpKernel\KernelInterface'

src/Command/EntityCleanupCommand.php

@@ -0,0 +1,167 @@
+<?php
+
+namespace Smart\CoreBundle\Command;
+
+use App\Entity\Monitoring\ApiCall;
+use App\Entity\Monitoring\Cron;
+use Doctrine\ORM\EntityManagerInterface;
+use Smart\CoreBundle\Config\IniOverrideConfig;
+use Smart\CoreBundle\Entity\ProcessInterface;
+use Smart\CoreBundle\Monitoring\ProcessMonitor;
+use Smart\CoreBundle\Utils\StringUtils;
+use Symfony\Component\Console\Attribute\AsCommand;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Style\SymfonyStyle;
+use Symfony\Component\PropertyAccess\PropertyAccess;
+use Symfony\Contracts\Translation\TranslatorInterface;
+
+/**
+ * @author Mathieu Ducrot <[email protected]>
+ */
+#[AsCommand(
+    name: 'cron:smart-entity-cleanup',
+    description: 'Database entity cleanup command based on the bundle configuration to completely remove entities or partially delete some data.',
+)]
+class EntityCleanupCommand extends Command
+{
+    public const BATCH_SIZE = 50;
+
+    /**
+     * @var array
+     * Example of configuration to put on your smart_core.yaml :
+     *  smart_core:
+     *      entity_cleanup_command_configs:
+     *          smart_entity_cleanup:
+     *              older_than: 1 week
+     *          count_organization:
+     *              older_than: 1 day
+     *          api_organization_update:
+     *              older_than: 1 week
+     *              class: api_call
+     *              where: o.status = 'success'
+     *              properties_to_clean:
+     *                  - logs
+     *                  - data
+     *                  - inputData
+     *                  - headers
+     *                  - outputResponse
+     *          simulation_3_years:
+     *              older_than: 3 years
+     *              older_than_property: updatedAt
+     *              class: App\Entity\Simulation
+     */
+    private array $commandConfigs = [];
+
+    public function __construct(
+        private readonly ProcessMonitor $processMonitor,
+        protected readonly IniOverrideConfig $config,
+        private readonly EntityManagerInterface $entityManager,
+        private readonly TranslatorInterface $translator
+    ) {
+        parent::__construct();
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output): int
+    {
+        $exitCode = Command::SUCCESS;
+        $io = new SymfonyStyle($input, $output);
+        $this->processMonitor->setConsoleIo($io);
+        $this->config->increaseMemoryLimit();
+        $process = $this->processMonitor->start(new Cron($this->getName()), true); // @phpstan-ignore-line
+
+        try {
+            $nbCleanedEntities = 0;
+            $propertyAccessor = PropertyAccess::createPropertyAccessor();
+
+            foreach ($this->commandConfigs as $key => $config) {
+                $targetClass = $this->getTargetClass($config);
+                $olderThanProperty = $config['older_than_property'];
+                $olderThan = $config['older_than'];
+                $dateTime = new \DateTime();
+                $dateTime->modify('-' . $olderThan);
+
+                $qb = $this->entityManager->getRepository($targetClass)->createQueryBuilder('o')
+                    ->andWhere("o.$olderThanProperty <= :started_at")
+                    ->setParameter('started_at', $dateTime)
+                ;
+                // MDT if the entity implements the ProcessInterface, we used the key of the config to filter the type
+                $isProcess = in_array(ProcessInterface::class, class_implements($targetClass));
+                if ($isProcess) {
+                    if ($targetClass === Cron::class) { // @phpstan-ignore-line
+                        $key = str_replace('_', '-', $key);
+                    }
+                    $qb->andWhere('o.type = :type')->setParameter('type', $key);
+                }
+                $whereCondition = $config['where'];
+                if ($whereCondition !== null) {
+                    $qb->andWhere($whereCondition);
+                }
+                $entities = $qb->getQuery()->getResult();
+                $nbEntities = count($entities);
+                $translatedTargetClass = $this->translator->trans('label.' . StringUtils::getEntityShortName($targetClass) . 's');
+                $this->processMonitor->logSection(sprintf(
+                    "%d %s%s à nettoyer depuis %s%s",
+                    $nbEntities,
+                    $translatedTargetClass,
+                    $isProcess ? (' ' . $key) : '',
+                    $olderThan,
+                    $whereCondition ? " (avec la condition $whereCondition)" : ''
+                ));
+
+                $i = 1;
+                $propertiesToClean = $config['properties_to_clean'];
+                $removeEntity = empty($propertiesToClean);
+                foreach ($entities as $entity) {
+                    $this->processMonitor->log(sprintf("%d) #%d début nettoyage ...", $i, $entity->getId()));
+                    if ($removeEntity) {
+                        $this->entityManager->remove($entity);
+                    } else {
+                        foreach ($propertiesToClean as $property) {
+                            $propertyAccessor->setValue($entity, $property, null);
+                        }
+                    }
+                    if ($i % self::BATCH_SIZE === 0) {
+                        $this->entityManager->flush();
+                    }
+                    $nbCleanedEntities++;
+                    $i++;
+                }
+                $this->entityManager->flush();
+                if ($nbEntities > 0) {
+                    $this->processMonitor->log("Fin du nettoyage des $translatedTargetClass.");
+                }
+            }
+
+            $this->processMonitor->logSuccess("$nbCleanedEntities entités nettoyées.");
+        } catch (\Exception $e) {
+            $exitCode = Command::FAILURE;
+            $this->processMonitor->logException($e);
+        } finally {
+            $this->processMonitor->end($process, $exitCode === Command::SUCCESS);
+        }
+
+        return $exitCode;
+    }
+
+    /**
+     * @return class-string<object>
+     */
+    private function getTargetClass(array $config): string
+    {
+        $class = $config['class'];
+        if ($class === 'cron') {
+            return Cron::class; // @phpstan-ignore-line
+        } elseif ($class === 'api_call') {
+            return ApiCall::class; // @phpstan-ignore-line
+        } else {
+            return $class;
+        }
+    }
+
+    public function setCommandConfigs(array $commandConfigs): void
+    {
+        $this->commandConfigs = $commandConfigs;
+    }
+}

src/DependencyInjection/Configuration.php

@@ -2,6 +2,7 @@
 
 namespace Smart\CoreBundle\DependencyInjection;
 
+use Symfony\Component\Config\Definition\Builder\ArrayNodeDefinition;
 use Symfony\Component\Config\Definition\Builder\TreeBuilder;
 use Symfony\Component\Config\Definition\ConfigurationInterface;
 
@@ -22,9 +23,29 @@ public function getConfigTreeBuilder(): TreeBuilder
                     ->prototype('scalar')->end()
                     ->defaultValue([])
                 ->end()
+                ->append($this->getEntityCleanupCommandConfigsDefinition())
             ->end()
         ;
 
         return $treeBuilder;
     }
+
+    private function getEntityCleanupCommandConfigsDefinition(): ArrayNodeDefinition
+    {
+        return (new TreeBuilder('entity_cleanup_command_configs'))->getRootNode()
+            ->requiresAtLeastOneElement()
+            ->useAttributeAsKey('name')
+            ->arrayPrototype()
+                ->children()
+                    ->scalarNode('older_than')->isRequired()->end()
+                    ->scalarNode('older_than_property')->defaultValue('startedAt')->end()
+                    ->scalarNode('class')->defaultValue('cron')->end()
+                    ->scalarNode('where')->defaultNull()->end()
+                    ->arrayNode('properties_to_clean')
+                        ->scalarPrototype()->end()
+                    ->end()
+                ->end()
+            ->end()
+        ;
+    }
 }

src/DependencyInjection/SmartCoreExtension.php

@@ -2,6 +2,7 @@
 
 namespace Smart\CoreBundle\DependencyInjection;
 
+use Smart\CoreBundle\Command\EntityCleanupCommand;
 use Smart\CoreBundle\Monitoring\ApiCallMonitor;
 use Symfony\Component\DependencyInjection\Loader\YamlFileLoader;
 use Symfony\Component\HttpKernel\DependencyInjection\Extension;
@@ -21,5 +22,8 @@ public function load(array $configs, ContainerBuilder $container): void
         $config = $this->processConfiguration(new Configuration(), $configs);
         $apiCallMonitor = $container->getDefinition(ApiCallMonitor::class);
         $apiCallMonitor->addMethodCall('setRestartAllowedRoutes', [$config['monitoring_api_restart_allowed_routes']]);
+
+        $entityCleanupCommand = $container->getDefinition(EntityCleanupCommand::class);
+        $entityCleanupCommand->addMethodCall('setCommandConfigs', [$config['entity_cleanup_command_configs']]);
     }
 }

src/Entity/UpdatableInterface.php

@@ -4,9 +4,9 @@
 
 interface UpdatableInterface
 {
-    public function getUpdatedAt(): \DateTimeInterface;
+    public function getUpdatedAt(): ?\DateTimeInterface;
 
-    public function setUpdatedAt(\DateTimeInterface $updatedAt, bool $initIntegerFields = true): void;
+    public function setUpdatedAt(?\DateTimeInterface $updatedAt, bool $initIntegerFields = true): void;
 
     public function getUpdatedAtMonth(): ?int;
 

src/Entity/UpdatableTrait.php

@@ -11,7 +11,7 @@ trait UpdatableTrait
      * @ORM\Column(type="datetime", nullable=true)
      */
     #[ORM\Column(type: Types::DATETIME_MUTABLE, nullable: true)]
-    protected \DateTimeInterface $updatedAt;
+    protected ?\DateTimeInterface $updatedAt = null;
 
     /**
      * @ORM\Column(type="integer", nullable=true)
@@ -25,15 +25,15 @@ trait UpdatableTrait
     #[ORM\Column(nullable: true)]
     protected ?int $updatedAtYear = null;
 
-    public function getUpdatedAt(): \DateTimeInterface
+    public function getUpdatedAt(): ?\DateTimeInterface
     {
         return $this->updatedAt;
     }
 
-    public function setUpdatedAt(\DateTimeInterface $updatedAt, bool $initIntegerFields = true): void
+    public function setUpdatedAt(?\DateTimeInterface $updatedAt, bool $initIntegerFields = true): void
     {
         $this->updatedAt = $updatedAt;
-        if ($initIntegerFields) {
+        if ($updatedAt !== null && $initIntegerFields) {
             $this->updatedAtMonth = (int) $updatedAt->format('m');
             $this->updatedAtYear = (int) $updatedAt->format('Y');
         }

translations/messages.fr.yaml

@@ -0,0 +1,5 @@
+cron:
+    smart_entity_cleanup.label: Commande de nettoyage
+label:
+    api_calls: Appels API
+    crons: Crons