Add a service to check the broken node before the import and clean th…

…em up.
LOBsTerr · Jun 14, 2024 · 7290080 · 7290080
1 parent a26e2f6
commit 7290080
Show file tree

Hide file tree

Showing 19 changed files with 176 additions and 26 deletions.
diff --git a/composer.json b/composer.json
@@ -17,7 +17,7 @@
   "license": "GPL-2.0-or-later",
   "minimum-stability": "dev",
   "require": {
-    "drupal/core": "^9.5 || ^10",
+    "drupal/core": "^9.5 || ^10 || ^11",
     "drupal/allowed_formats": "^2.0",
     "drupal/entity_data": "^1.0",
     "drupal/migrate_plus": "^6.0",

diff --git a/modules/newsroom_connector_country/newsroom_connector_country.info.yml b/modules/newsroom_connector_country/newsroom_connector_country.info.yml
@@ -3,7 +3,7 @@ description: Provides Newsroom Country vocabulary and importer.
 package: Newsroom
 
 type: module
-core_version_requirement: ^9 || ^10
+core_version_requirement: ^9 || ^10 || ^11
 php: (>=8.0)
 
 dependencies:

diff --git a/modules/newsroom_connector_item/migrations/migrate_plus.migration.newsroom_item.yml b/modules/newsroom_connector_item/migrations/migrate_plus.migration.newsroom_item.yml
@@ -11,7 +11,7 @@ source:
   urls:
     - 'https://ec.europa.eu/newsroom/dae/feed/full/new'
 
-  item_selector: '//channel/item'
+  item_selector: '//channel/item[count(title[@lang="EN"]) > 0]'
 
   ids:
     item_id:

diff --git a/modules/newsroom_connector_item/newsroom_connector_item.info.yml b/modules/newsroom_connector_item/newsroom_connector_item.info.yml
@@ -3,7 +3,7 @@ description: Provides Newsroom Item content type and importer.
 package: Newsroom
 
 type: module
-core_version_requirement: ^9 || ^10
+core_version_requirement: ^9 || ^10 || ^11
 php: (>=8.0)
 
 dependencies:

diff --git a/modules/newsroom_connector_item/newsroom_connector_item.services.yml b/modules/newsroom_connector_item/newsroom_connector_item.services.yml
@@ -1,4 +1,10 @@
 services:
+  newsroom_connector_item.migration_clean_up:
+    class: Drupal\newsroom_connector_item\EventSubscriber\MigrationCleanUpBrokenItems
+    tags:
+      - { name: event_subscriber }
+    arguments: [ '@event_dispatcher', '@entity_type.manager', '@newsroom_connector.migration_manager' ]
+
   newsroom_connector_item.migration_image_sync:
     class: Drupal\newsroom_connector_item\EventSubscriber\MigrationImageSync
     tags:

diff --git a/modules/newsroom_connector_item/src/EventSubscriber/MigrationCleanUpBrokenItems.php b/modules/newsroom_connector_item/src/EventSubscriber/MigrationCleanUpBrokenItems.php
@@ -0,0 +1,107 @@
+<?php
+
+namespace Drupal\newsroom_connector_item\EventSubscriber;
+
+use Drupal\Core\Entity\EntityTypeManagerInterface;
+use Drupal\migrate\Event\MigrateEvents;
+use Drupal\migrate\Event\MigrateImportEvent;
+use Drupal\newsroom_connector\MigrationManager;
+use Drupal\newsroom_connector_item\Plugin\newsroom\NewsroomItemNewsroomProcessor;
+use Symfony\Component\EventDispatcher\EventDispatcherInterface;
+use Symfony\Component\EventDispatcher\EventSubscriberInterface;
+
+/**
+ * Clean up broken nodes.
+ */
+class MigrationCleanUpBrokenItems implements EventSubscriberInterface {
+
+  /**
+   * The event dispatcher.
+   *
+   * @var \Symfony\Component\EventDispatcher\EventDispatcherInterface
+   */
+  protected $dispatcher;
+
+  /**
+   * The entity type manager.
+   *
+   * @var \Drupal\Core\Entity\EntityTypeManagerInterface
+   */
+  protected $entityTypeManager;
+
+  /**
+   * Migration manager.
+   *
+   * @var \Drupal\newsroom_connector\MigrationManager
+   */
+  protected $migrationManager;
+
+  /**
+   * MigrationImportSync constructor.
+   *
+   * @param \Symfony\Component\EventDispatcher\EventDispatcherInterface $dispatcher
+   *   The event dispatcher.
+   * @param \Drupal\Core\Entity\EntityTypeManagerInterface $entity_type_manager
+   *   The entity type manager.
+   */
+  public function __construct(
+    EventDispatcherInterface $dispatcher,
+    EntityTypeManagerInterface $entity_type_manager,
+    MigrationManager $migration_manager,
+  ) {
+    $this->dispatcher = $dispatcher;
+    $this->entityTypeManager = $entity_type_manager;
+    $this->migrationManager = $migration_manager;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public static function getSubscribedEvents() {
+    $events = [];
+    $events[MigrateEvents::PRE_IMPORT][] = ['clean'];
+    return $events;
+  }
+
+  /**
+   * Event callback to sync source and destination.
+   *
+   * @param \Drupal\migrate\Event\MigrateImportEvent $event
+   *   The migration import event.
+   */
+  public function clean(MigrateImportEvent $event) {
+    // We want to get rid off newsroom items, which don't have mappings and we
+    // can't track them, so for us they are broken.
+    $migration = $event->getMigration();
+
+    $entity_type = 'node';
+    $bundle = 'newsroom_item';
+    $bundle_field = 'type';
+
+    if ($migration->id() == NewsroomItemNewsroomProcessor::MIGRATION_ITEM) {
+      $source = clone $migration->getSourcePlugin();
+      $source->rewind();
+
+      while ($source->valid()) {
+        $newsroom_id = $source->current()->getSourceProperty('item_id');
+        if (!empty($newsroom_id)) {
+          $destination_ids = $this->migrationManager->getDestinationIdsBySourceIds($migration, [$newsroom_id]);
+          if (!empty($destination_ids)) {
+            $nodes = $this->migrationManager->getEntitiesByNewsroomId($newsroom_id, $entity_type, $bundle, $bundle_field);
+            foreach ($destination_ids as $destination) {
+              foreach ($nodes as $node) {
+
+                if ($node->id() != $destination['nid']) {
+                  $node->delete();
+                }
+              }
+            }
+          }
+        }
+
+        $source->next();
+      }
+    }
+  }
+
+}
diff --git a/modules/newsroom_connector_item/src/EventSubscriber/MigrationImageSync.php b/modules/newsroom_connector_item/src/EventSubscriber/MigrationImageSync.php
@@ -36,7 +36,10 @@ class MigrationImageSync implements EventSubscriberInterface {
    * @param \Drupal\newsroom_connector\MigrationManager $migration_manager
    *   Migration manager.
    */
-  public function __construct(EventDispatcherInterface $dispatcher, MigrationManager $migration_manager) {
+  public function __construct(
+    EventDispatcherInterface $dispatcher,
+    MigrationManager $migration_manager,
+  ) {
     $this->dispatcher = $dispatcher;
     $this->migrationManager = $migration_manager;
   }

diff --git a/modules/newsroom_connector_media/newsroom_connector_media.info.yml b/modules/newsroom_connector_media/newsroom_connector_media.info.yml
@@ -3,7 +3,7 @@ description: Newsroom media provides media configuration.
 package: Newsroom
 
 type: module
-core_version_requirement: ^9 || ^10
+core_version_requirement: ^9 || ^10 || ^11
 php: (>=8.0)
 
 dependencies:

diff --git a/modules/newsroom_connector_newsletter/newsroom_connector_newsletter.info.yml b/modules/newsroom_connector_newsletter/newsroom_connector_newsletter.info.yml
@@ -3,7 +3,7 @@ description: Provides Newsroom Newsletter vocabulary and importer.
 package: Newsroom
 
 type: module
-core_version_requirement: ^9 || ^10
+core_version_requirement: ^9 || ^10 || ^11
 php: (>=8.0)
 
 dependencies:

diff --git a/modules/newsroom_connector_service/newsroom_connector_service.info.yml b/modules/newsroom_connector_service/newsroom_connector_service.info.yml
@@ -3,7 +3,7 @@ description: Provides Newsroom Service vocabulary and importer.
 package: Newsroom
 
 type: module
-core_version_requirement: ^9 || ^10
+core_version_requirement: ^9 || ^10 || ^11
 php: (>=8.0)
 
 dependencies:

diff --git a/modules/newsroom_connector_team_responsible/newsroom_connector_team_responsible.info.yml b/modules/newsroom_connector_team_responsible/newsroom_connector_team_responsible.info.yml
@@ -3,7 +3,7 @@ description: Provides Newsroom Team Responsible vocabulary and importer.
 package: Newsroom
 
 type: module
-core_version_requirement: ^9 || ^10
+core_version_requirement: ^9 || ^10 || ^11
 php: (>=8.0)
 
 dependencies:

diff --git a/modules/newsroom_connector_topic/migrations/migrate_plus.migration.newsroom_topic.yml b/modules/newsroom_connector_topic/migrations/migrate_plus.migration.newsroom_topic.yml
@@ -10,7 +10,7 @@ source:
   data_parser_plugin: newsroom_simple_xml
   urls: 'https://ec.europa.eu/newsroom/dae/feed/topics'
 
-  item_selector: '//channel/item[infsonewsroom:definedBy="User"]'
+  item_selector: '//channel/item[infsonewsroom:definedBy="User" and string-length(title[@lang="EN"]) > 0]'
 
   ids:
     topic_id:

diff --git a/modules/newsroom_connector_topic/newsroom_connector_topic.info.yml b/modules/newsroom_connector_topic/newsroom_connector_topic.info.yml
@@ -3,7 +3,7 @@ description: Provides Newsroom Topic vocabulary and importer.
 package: Newsroom
 
 type: module
-core_version_requirement: ^9 || ^10
+core_version_requirement: ^9 || ^10 || ^11
 php: (>=8.0)
 
 dependencies:

diff --git a/modules/newsroom_connector_topic/src/Plugin/migrate/NewsroomTopicLanguageDeriver.php b/modules/newsroom_connector_topic/src/Plugin/migrate/NewsroomTopicLanguageDeriver.php
@@ -16,7 +16,7 @@ class NewsroomTopicLanguageDeriver extends BaseNewsroomLanguageDeriver {
   protected function getDerivativeValues(array $base_plugin_definition, LanguageInterface $language, $language_code) {
     $language_id = $language->getId();
 
-    $base_plugin_definition['source']['item_selector'] = '//channel/item[string-length(title[@lang="' . $language_code . '"]) > 0]';
+    $base_plugin_definition['source']['item_selector'] = '//channel/item[infsonewsroom:definedBy="User" and string-length(title[@lang="' . $language_code . '"]) > 0]';
 
     // Name.
     $base_plugin_definition['source']['fields'][] = [

diff --git a/modules/newsroom_connector_type/migrations/migrate_plus.migration.newsroom_type.yml b/modules/newsroom_connector_type/migrations/migrate_plus.migration.newsroom_type.yml
@@ -10,7 +10,7 @@ source:
   data_parser_plugin: newsroom_simple_xml
   urls: 'https://ec.europa.eu/newsroom/dae/feed/item-types'
 
-  item_selector: '//channel/item'
+  item_selector: '//channel/item[string-length(title[@lang="EN"]) > 0]'
 
   ids:
     type_id:

diff --git a/modules/newsroom_connector_type/newsroom_connector_type.info.yml b/modules/newsroom_connector_type/newsroom_connector_type.info.yml
@@ -3,7 +3,7 @@ description: Provides Newsroom Type vocabulary and importer.
 package: Newsroom
 
 type: module
-core_version_requirement: ^9 || ^10
+core_version_requirement: ^9 || ^10 || ^11
 php: (>=8.0)
 
 dependencies:

diff --git a/newsroom_connector.info.yml b/newsroom_connector.info.yml
@@ -3,7 +3,7 @@ description: Newsroom connector
 package: Newsroom
 
 type: module
-core_version_requirement: ^9 || ^10
+core_version_requirement: ^9 || ^10 || ^11
 php: (>=8.0)
 
 dependencies:

diff --git a/src/MigrationManager.php b/src/MigrationManager.php
@@ -48,10 +48,22 @@ class MigrationManager implements MigrationManagerInterface {
    */
   protected $dispatcher;
 
+  /**
+   * Migratinos plugins.
+   *
+   * @var array
+   */
+  protected $migrations = [];
+
   /**
    * {@inheritdoc}
    */
-  public function __construct(MigrationPluginManager $migrationPluginManager, LanguageManagerInterface $languageManager, EntityTypeManagerInterface $entity_type_manager, EventDispatcherInterface $dispatcher) {
+  public function __construct(
+    MigrationPluginManager $migrationPluginManager,
+    LanguageManagerInterface $languageManager,
+    EntityTypeManagerInterface $entity_type_manager,
+    EventDispatcherInterface $dispatcher,
+  ) {
     $this->migrationPluginManager = $migrationPluginManager;
     $this->languageManager = $languageManager;
     $this->entityTypeManager = $entity_type_manager;
@@ -179,7 +191,7 @@ protected function deleteDestinationBySourceId($migration_id, $source_id) {
    * @return array
    *   List of destination IDs.
    */
-  protected function getDestinationIdsBySourceIds(MigrationInterface $migration, array $source_id_values) {
+  public function getDestinationIdsBySourceIds(MigrationInterface $migration, array $source_id_values) {
     $destination_keys = array_keys($migration->getDestinationPlugin()->getIds());
     $indexed_ids = $migration->getIdMap()
       ->lookupDestinationIds($source_id_values);
@@ -194,7 +206,11 @@ protected function getDestinationIdsBySourceIds(MigrationInterface $migration, a
    * {@inheritdoc}
    */
   public function getMigration($migration_id) {
-    return $this->migrationPluginManager->createInstance($migration_id);
+    if (empty($this->migrations[$migration_id])) {
+      $this->migrations[$migration_id] = $this->migrationPluginManager->createInstance($migration_id);
+    }
+
+    return $this->migrations[$migration_id];
   }
 
   /**
@@ -246,19 +262,20 @@ protected function dispatchRowDeleteEvent($event_name, MigrationInterface $migra
    * {@inheritdoc}
    */
   public function getEntityByNewsroomId($newsroom_id, $entity_type, $bundle, $bundle_field) {
-    $entity = NULL;
-    $items = $this->entityTypeManager
+    $items = $this->getEntitiesByNewsroomId($newsroom_id, $entity_type, $bundle, $bundle_field);
+    return reset($items);
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function getEntitiesByNewsroomId($newsroom_id, $entity_type, $bundle, $bundle_field) {
+    return $this->entityTypeManager
       ->getStorage($entity_type)
       ->loadByProperties([
         'field_newsroom_id' => $newsroom_id,
         $bundle_field => $bundle,
       ]);
-
-    if ($item = reset($items)) {
-      $entity = $item;
-    }
-
-    return $entity;
   }
 
 }
diff --git a/src/MigrationManagerInterface.php b/src/MigrationManagerInterface.php
@@ -70,6 +70,23 @@ public function rollback($migration_id, array $source_id_values = []);
    */
   public function getEntityByNewsroomId($newsroom_id, $entity_type, $bundle, $bundle_field);
 
+  /**
+   * Get entities by original newsroom id.
+   *
+   * @param int $newsroom_id
+   *   Original newsroom id.
+   * @param string $entity_type
+   *   Entity type.
+   * @param string $bundle
+   *   Bundle.
+   * @param string $bundle_field
+   *   Bundle field.
+   *
+   * @return \Drupal\Core\Entity\EntityInterface|null
+   *   Entity associated with newsroom id.
+   */
+  public function getEntitiesByNewsroomId($newsroom_id, $entity_type, $bundle, $bundle_field);
+
   /**
    * Get migration.
    *