Merge pull request ManageIQ#5533 from agrare/gce_inventory_refresher

Basic GCE inventory refresher

app/models/manageiq/providers/google/cloud_manager.rb

@@ -1,5 +1,11 @@
 class ManageIQ::Providers::Google::CloudManager < ManageIQ::Providers::CloudManager
+  require_nested :AvailabilityZone
+  require_nested :Flavor
+  require_nested :RefreshParser
+  require_nested :RefreshWorker
   require_nested :Refresher
+  require_nested :Template
+  require_nested :Vm
 
   def self.ems_type
     @ems_type ||= "gce".freeze
@@ -62,19 +68,23 @@ def verify_credentials(auth_type = nil, options = {})
   # Connections
   #
 
-  def connect(options = {})
+  def self.raw_connect(google_project, google_json_key)
     require 'fog/google'
 
-    raise MiqException::MiqHostError, "No credentials defined" if self.missing_credentials?(options[:auth_type])
-
-    auth_token = authentication_token(options[:auth_type])
-
     ::Fog::Compute.new(
       :provider               => "Google",
-      :google_project         => project,
-      :google_json_key_string => auth_token,
+      :google_project         => google_project,
+      :google_json_key_string => google_json_key,
     )
+  end
 
+  def connect(options = {})
+    require 'fog/google'
+
+    raise MiqException::MiqHostError, "No credentials defined" if self.missing_credentials?(options[:auth_type])
+
+    auth_token = authentication_token(options[:auth_type])
+    self.class.raw_connect(project, auth_token)
   end
 
   def gce

app/models/manageiq/providers/google/cloud_manager/availability_zone.rb

@@ -0,0 +1,6 @@
+class ManageIQ::Providers::Google::CloudManager::AvailabilityZone < ::AvailabilityZone
+  def provider_object(connection = nil)
+    connection ||= ext_management_system.connect
+    connection.availability_zones[ems_ref]
+  end
+end

app/models/manageiq/providers/google/cloud_manager/flavor.rb

@@ -0,0 +1,2 @@
+class ManageIQ::Providers::Google::CloudManager::Flavor < ::Flavor
+end

app/models/manageiq/providers/google/cloud_manager/refresh_parser.rb

@@ -0,0 +1,189 @@
+require 'fog/google'
+
+module ManageIQ::Providers
+  module Google
+    class CloudManager::RefreshParser < ManageIQ::Providers::CloudManager::RefreshParser
+      include Vmdb::Logging
+
+      def self.ems_inv_to_hashes(ems, options = nil)
+        new(ems, options).ems_inv_to_hashes
+      end
+
+      def initialize(ems, options = nil)
+        @ems               = ems
+        @connection        = ems.connect
+        @options           = options || {}
+        @data              = {}
+        @data_index        = {}
+      end
+
+      def ems_inv_to_hashes
+        log_header = "Collecting data for EMS : [#{@ems.name}] id: [#{@ems.id}]"
+
+        _log.info("#{log_header}...")
+        get_zones
+        get_flavors
+        get_cloud_networks
+        get_images
+        get_instances
+        _log.info("#{log_header}...Complete")
+
+        @data
+      end
+
+      private
+
+      def get_zones
+        zones = @connection.zones.all
+        process_collection(zones, :availability_zones) { |zone| parse_zone(zone) }
+      end
+
+      def get_flavors
+        flavors = @connection.flavors.all
+        process_collection(flavors, :flavors) { |flavor| parse_flavor(flavor) }
+      end
+
+      def get_cloud_networks
+        networks = @connection.networks.all
+        process_collection(networks, :cloud_networks) { |network| parse_cloud_network(network) }
+      end
+
+      def get_images
+        images = @connection.images.all
+        process_collection(images, :vms) { |image| parse_image(image) }
+      end
+
+      def get_instances
+        instances = @connection.servers.all
+        process_collection(instances, :vms) { |instance| parse_instance(instance) }
+      end
+
+      def process_collection(collection, key)
+        @data[key] ||= []
+
+        collection.each do |item|
+          uid, new_result = yield(item)
+          next if uid.nil?
+
+          @data[key] |= [new_result]
+          @data_index.store_path(key, uid, new_result)
+        end
+      end
+
+      def parse_zone(zone)
+        name = uid = zone.name
+        type = ManageIQ::Providers::Google::CloudManager::AvailabilityZone.name
+
+        new_result = {
+          :type    => type,
+          :ems_ref => uid,
+          :name    => name,
+        }
+
+        return uid, new_result
+      end
+
+      def parse_flavor(flavor)
+        uid = flavor.name
+
+        type = ManageIQ::Providers::Google::CloudManager::Flavor.name
+        new_result = {
+          :type        => type,
+          :ems_ref     => flavor.name,
+          :name        => flavor.name,
+          :description => flavor.description,
+          :enabled     => !flavor.deprecated,
+          :cpus        => flavor.guest_cpus,
+          :cpu_cores   => 1,
+          :memory      => flavor.memory_mb * 1.megabyte,
+        }
+
+        return uid, new_result
+      end
+
+      def parse_cloud_network(network)
+        uid  = network.id
+
+        new_result = {
+          :ems_ref => uid,
+          :name    => network.name,
+          :cidr    => network.ipv4_range,
+          :status  => "active",
+          :enabled => true,
+        }
+
+        return uid, new_result
+      end
+
+      def parse_image(image)
+        uid    = image.id
+        name   = image.name
+        name ||= uid
+        type   = ManageIQ::Providers::Google::CloudManager::Template.name
+
+        new_result = {
+          :type               => type,
+          :uid_ems            => uid,
+          :ems_ref            => uid,
+          :name               => name,
+          :vendor             => "google",
+          :raw_power_state    => "never",
+          :operating_system   => process_os(image),
+          :template           => true,
+          :publicly_available => true,
+        }
+
+        return uid, new_result
+      end
+
+      def process_os(image)
+        {
+          :product_name => OperatingSystem.normalize_os_name(image.name)
+        }
+      end
+
+      def parse_instance(instance)
+        uid    = instance.id
+        name   = instance.name
+        name ||= uid
+
+        flavor_uid = parse_uid_from_url(instance.machine_type)
+        flavor     = @data_index.fetch_path(:flavors, flavor_uid)
+
+        zone_uid   = parse_uid_from_url(instance.zone)
+        zone       = @data_index.fetch_path(:availability_zones, zone_uid)
+
+        type = ManageIQ::Providers::Google::CloudManager::Vm.name
+        new_result = {
+          :type              => type,
+          :uid_ems           => uid,
+          :ems_ref           => uid,
+          :name              => name,
+          :description       => instance.description,
+          :vendor            => "google",
+          :raw_power_state   => instance.state,
+          :flavor            => flavor,
+          :availability_zone => zone,
+          :hardware          => {
+            :cpu_sockets          => flavor[:cpus],
+            :cpu_total_cores      => flavor[:cpu_cores],
+            :cpu_cores_per_socket => 1,
+            :memory_mb            => flavor[:memory] / 1.megabyte,
+            :disks                => [],
+            :networks             => [],
+          }
+        }
+
+        return uid, new_result
+      end
+
+      def parse_uid_from_url(url)
+        # A lot of attributes in gce are full URLs with the
+        # uid being the last component.  This helper method
+        # returns the last component of the url
+        uid = url.split('/')[-1]
+        uid
+      end
+    end
+  end
+end

app/models/manageiq/providers/google/cloud_manager/refresh_worker.rb

@@ -0,0 +1,3 @@
+class ManageIQ::Providers::Google::CloudManager::RefreshWorker < ManageIQ::Providers::BaseManager::RefreshWorker
+  require_nested :Runner
+end

app/models/manageiq/providers/google/cloud_manager/refresh_worker/runner.rb

@@ -0,0 +1,2 @@
+class ManageIQ::Providers::Google::CloudManager::RefreshWorker::Runner < ManageIQ::Providers::BaseManager::RefreshWorker::Runner
+end

app/models/manageiq/providers/google/cloud_manager/refresher.rb

@@ -1,11 +1,13 @@
-class ManageIQ::Providers::Google::CloudManager::Refresher < ManageIQ::Providers::BaseManager::Refresher
-  include ::EmsRefresh::Refreshers::EmsRefresherMixin
+module ManageIQ::Providers::Google
+  class CloudManager::Refresher < ManageIQ::Providers::BaseManager::Refresher
+    include ::EmsRefresh::Refreshers::EmsRefresherMixin
 
-  def parse_inventory(ems, _targets)
-    ::ManageIQ::Providers::Google::CloudManager::RefreshParser.ems_inv_to_hashes(ems, refresher_options)
-  end
+    def parse_inventory(ems, _targets)
+      ManageIQ::Providers::Google::CloudManager::RefreshParser.ems_inv_to_hashes(ems, refresher_options)
+    end
 
-  def post_process_refresh_classes
-    [::Vm]
+    def post_process_refresh_classes
+      [::Vm]
+    end
   end
 end

app/models/manageiq/providers/google/cloud_manager/template.rb

@@ -0,0 +1,6 @@
+class ManageIQ::Providers::Google::CloudManager::Template < ManageIQ::Providers::CloudManager::Template
+  def provider_object(connection = nil)
+    connection ||= ext_management_system.connect
+    connection.images[ems_ref]
+  end
+end

app/models/manageiq/providers/google/cloud_manager/vm.rb

@@ -0,0 +1,37 @@
+class ManageIQ::Providers::Google::CloudManager::Vm < ManageIQ::Providers::CloudManager::Vm
+  def provider_object(connection = nil)
+    connection ||= ext_management_system.connect
+    connection.instances[ems_ref]
+  end
+
+  #
+  # Relationship methods
+  #
+
+  def disconnect_inv
+    super
+
+    # Mark all instances no longer found as terminated
+    power_state == "off"
+    save
+  end
+
+  def disconnected
+    false
+  end
+
+  def disconnected?
+    false
+  end
+
+  def self.calculate_power_state(raw_power_state)
+    case raw_power_state.downcase
+    when "running"
+      "on"
+    when "terminated"
+      "off"
+    else
+      "unknown"
+    end
+  end
+end

app/models/miq_server/worker_management/monitor/class_names.rb

@@ -16,6 +16,7 @@ module MiqServer::WorkerManagement::Monitor::ClassNames
     MiqEmsRefreshCoreWorker
     ManageIQ::Providers::Azure::CloudManager::RefreshWorker
     ManageIQ::Providers::Amazon::CloudManager::RefreshWorker
+    ManageIQ::Providers::Google::CloudManager::RefreshWorker
     ManageIQ::Providers::Foreman::ConfigurationManager::RefreshWorker
     ManageIQ::Providers::Foreman::ProvisioningManager::RefreshWorker
     ManageIQ::Providers::Kubernetes::ContainerManager::RefreshWorker
@@ -77,6 +78,7 @@ module MiqServer::WorkerManagement::Monitor::ClassNames
     MiqStorageMetricsCollectorWorker
     ManageIQ::Providers::Azure::CloudManager::RefreshWorker
     ManageIQ::Providers::Amazon::CloudManager::RefreshWorker
+    ManageIQ::Providers::Google::CloudManager::RefreshWorker
     ManageIQ::Providers::Foreman::ConfigurationManager::RefreshWorker
     ManageIQ::Providers::Foreman::ProvisioningManager::RefreshWorker
     ManageIQ::Providers::Kubernetes::ContainerManager::RefreshWorker

app/models/vm_or_template.rb

@@ -41,6 +41,7 @@ class VmOrTemplate < ActiveRecord::Base
     "amazon"    => "Amazon",
     "redhat"    => "RedHat",
     "openstack" => "OpenStack",
+    "google"    => "Google",
     "unknown"   => "Unknown"
   }
 

config/vmdb.tmpl.yml

@@ -383,6 +383,7 @@ workers:
         :ems_refresh_worker_azure: {}
         :ems_refresh_worker_foreman_configuration: {}
         :ems_refresh_worker_foreman_provisioning: {}
+        :ems_refresh_worker_google: {}
         :ems_refresh_worker_kubernetes: {}
         :ems_refresh_worker_openshift: {}
         :ems_refresh_worker_atomic: {}

lib/miq_automation_engine/service_models/miq_ae_service_manageiq-providers-google-cloud_manager-availability_zone.rb

@@ -0,0 +1,4 @@
+module MiqAeMethodService
+  class MiqAeServiceManageIQ_Providers_Google_CloudManager_AvailabilityZone < MiqAeServiceAvailabilityZone
+  end
+end

lib/miq_automation_engine/service_models/miq_ae_service_manageiq-providers-google-cloud_manager-flavor.rb

@@ -0,0 +1,4 @@
+module MiqAeMethodService
+  class MiqAeServiceManageIQ_Providers_Google_CloudManager_Flavor < MiqAeServiceFlavor
+  end
+end

lib/miq_automation_engine/service_models/miq_ae_service_manageiq-providers-google-cloud_manager-template.rb

@@ -0,0 +1,4 @@
+module MiqAeMethodService
+  class MiqAeServiceManageIQ_Providers_Google_CloudManager_Template < MiqAeServiceManageIQ_Providers_CloudManager_Template
+  end
+end

lib/miq_automation_engine/service_models/miq_ae_service_manageiq-providers-google-cloud_manager-vm.rb

@@ -0,0 +1,4 @@
+module MiqAeMethodService
+  class MiqAeServiceManageIQ_Providers_Google_CloudManager_Vm < MiqAeServiceManageIQ_Providers_CloudManager_Vm
+  end
+end

spec/factories/vm_google.rb

@@ -0,0 +1,5 @@
+FactoryGirl.define do
+  factory :vm_google, :class => "ManageIQ::Providers::Google::CloudManager::Vm", :parent => :vm_cloud do
+    vendor "google"
+  end
+end