Merge remote-tracking branch 'origin/master' into quinteros

ManageIQ · Oct 26, 2023 · 0f7910e · 0f7910e
2 parents 008444e + 98dad42
commit 0f7910e
Show file tree

Hide file tree

Showing 14 changed files with 235 additions and 92 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -31,7 +31,7 @@ jobs:
       TEST_RAILS_VERSION: "${{ matrix.rails-version }}"
       CC_TEST_REPORTER_ID: "${{ secrets.CC_TEST_REPORTER_ID }}"
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:

diff --git a/db/migrate/20180606155924_move_ansible_container_secrets_into_database.rb b/db/migrate/20180606155924_move_ansible_container_secrets_into_database.rb
@@ -14,6 +14,10 @@ def up
     update_authentications
   end
 
+  def self.read_token(file)
+    File.read(file)
+  end
+
   private
 
   def containerized?
@@ -88,7 +92,7 @@ def update_or_create_authentication!(find_args, update_args)
   def request_params
     {
       'Accept'         => "application/json",
-      'Authorization'  => "Bearer #{File.read(TOKEN_FILE)}",
+      'Authorization'  => "Bearer #{self.class.read_token(TOKEN_FILE)}",
       :ssl_ca_cert     => CA_CERT_FILE,
       :ssl_verify_mode => OpenSSL::SSL::VERIFY_PEER
     }

diff --git a/db/migrate/20200427122455_remove_vim_strings_from_notifications.rb b/db/migrate/20200427122455_remove_vim_strings_from_notifications.rb
@@ -7,30 +7,10 @@ class Notification < ActiveRecord::Base
   end
 
   def up
-    say_with_time("Removing VimStrings from Notifications") do
-      base_relation = Notification.in_my_region.where("options LIKE ?", "%ruby/string:VimString%")
-      say_batch_started(base_relation.size)
-
-      loop do
-        count = base_relation.limit(50_000).update_all("options = REGEXP_REPLACE(options, '!ruby/string:VimString', '!ruby/string:String', 'g')")
-        break if count == 0
-
-        say_batch_processed(count)
-      end
-    end
+    regex_replace_column_value(Notification.in_my_region, 'options', '!ruby/string:VimString', '!ruby/string:String')
   end
 
   def down
-    say_with_time("Restoring VimStrings in Notifications") do
-      base_relation = Notification.in_my_region.where("options LIKE ?", "%ruby/string:String%")
-      say_batch_started(base_relation.size)
-
-      loop do
-        count = base_relation.limit(50_000).update_all("options = REGEXP_REPLACE(options, '!ruby/string:String', '!ruby/string:VimString', 'g')")
-        break if count == 0
-
-        say_batch_processed(count)
-      end
-    end
+    regex_replace_column_value(Notification.in_my_region, 'options', '!ruby/string:String', '!ruby/string:VimString')
   end
 end
diff --git a/db/migrate/20200629194033_remove_vim_strings_from_miq_provision.rb b/db/migrate/20200629194033_remove_vim_strings_from_miq_provision.rb
@@ -9,30 +9,10 @@ class MiqRequestTask < ActiveRecord::Base
   end
 
   def up
-    say_with_time("Removing VimStrings from MiqRequestTask") do
-      base_relation = MiqRequestTask.in_my_region.where("phase_context LIKE ?", "%ruby/string:VimString%")
-      say_batch_started(base_relation.size)
-
-      loop do
-        count = base_relation.limit(50_000).update_all("phase_context = REGEXP_REPLACE(phase_context, '!ruby/string:VimString', '!ruby/string:String', 'g')")
-        break if count == 0
-
-        say_batch_processed(count)
-      end
-    end
+    regex_replace_column_value(MiqRequestTask.in_my_region, 'phase_context', '!ruby/string:VimString', '!ruby/string:String')
   end
 
   def down
-    say_with_time("Restoring VimStrings in MiqRequestTask") do
-      base_relation = MiqRequestTask.in_my_region.where("phase_context LIKE ?", "%ruby/string:String%")
-      say_batch_started(base_relation.size)
-
-      loop do
-        count = base_relation.limit(50_000).update_all("phase_context = REGEXP_REPLACE(phase_context, '!ruby/string:String', '!ruby/string:VimString', 'g')")
-        break if count == 0
-
-        say_batch_processed(count)
-      end
-    end
+    regex_replace_column_value(MiqRequestTask.in_my_region, 'phase_context', '!ruby/string:String', '!ruby/string:VimString')
   end
 end
diff --git a/db/migrate/20221111201223_remove_vim_strings_from_custom_attributes.rb b/db/migrate/20221111201223_remove_vim_strings_from_custom_attributes.rb
@@ -7,30 +7,10 @@ class CustomAttribute < ActiveRecord::Base
   end
 
   def up
-    say_with_time("Removing VimStrings from CustomAttribute") do
-      base_relation = CustomAttribute.in_my_region.where("serialized_value LIKE ?", "%ruby/string:VimString%")
-      say_batch_started(base_relation.size)
-
-      loop do
-        count = base_relation.limit(50_000).update_all("serialized_value = REGEXP_REPLACE(serialized_value, '!ruby/string:VimString', '!ruby/string:String', 'g')")
-        break if count == 0
-
-        say_batch_processed(count)
-      end
-    end
+    regex_replace_column_value(CustomAttribute.in_my_region, 'serialized_value', '!ruby/string:VimString', '!ruby/string:String')
   end
 
   def down
-    say_with_time("Restoring VimStrings from CustomAttribute") do
-      base_relation = CustomAttribute.in_my_region.where("serialized_value LIKE ?", "%ruby/string:String%")
-      say_batch_started(base_relation.size)
-
-      loop do
-        count = base_relation.limit(50_000).update_all("serialized_value = REGEXP_REPLACE(serialized_value, '!ruby/string:String', '!ruby/string:VimString', 'g')")
-        break if count == 0
-
-        say_batch_processed(count)
-      end
-    end
+    regex_replace_column_value(CustomAttribute.in_my_region, 'serialized_value', '!ruby/string:String', '!ruby/string:VimString')
   end
 end
diff --git a/db/migrate/20221114165219_remove_vim_string_from_miq_request_task_options.rb b/db/migrate/20221114165219_remove_vim_string_from_miq_request_task_options.rb
@@ -9,30 +9,10 @@ class MiqRequestTask < ActiveRecord::Base
   end
 
   def up
-    say_with_time("Removing VimStrings from MiqRequestTask") do
-      base_relation = MiqRequestTask.in_my_region.where("options LIKE ?", "%ruby/string:VimString%")
-      say_batch_started(base_relation.size)
-
-      loop do
-        count = base_relation.limit(50_000).update_all("options = REGEXP_REPLACE(options, '!ruby/string:VimString', '!ruby/string:String', 'g')")
-        break if count == 0
-
-        say_batch_processed(count)
-      end
-    end
+    regex_replace_column_value(MiqRequestTask, 'options', '!ruby/string:VimString', '!ruby/string:String')
   end
 
   def down
-    say_with_time("Restoring VimStrings from MiqRequestTask") do
-      base_relation = MiqRequestTask.in_my_region.where("options LIKE ?", "%ruby/string:String%")
-      say_batch_started(base_relation.size)
-
-      loop do
-        count = base_relation.limit(50_000).update_all("options = REGEXP_REPLACE(options, '!ruby/string:String', '!ruby/string:VimString', 'g')")
-        break if count == 0
-
-        say_batch_processed(count)
-      end
-    end
+    regex_replace_column_value(MiqRequestTask, 'options', '!ruby/string:String', '!ruby/string:VimString')
   end
 end
diff --git a/db/migrate/20230919143746_remove_vim_strings_from_miq_queue.rb b/db/migrate/20230919143746_remove_vim_strings_from_miq_queue.rb
@@ -0,0 +1,16 @@
+class RemoveVimStringsFromMiqQueue < ActiveRecord::Migration[6.1]
+  disable_ddl_transaction!
+  include MigrationHelper
+
+  class MiqQueue < ActiveRecord::Base
+  end
+
+  def up
+    regex_replace_column_value(MiqQueue, 'args', '!ruby/string:VimString', '!ruby/string:String')
+    regex_replace_column_value(MiqQueue, 'args', '!ruby/hash-with-ivars:VimHash', '!ruby/hash-with-ivars:Hash')
+  end
+
+  # this is very old, not rolling back
+  def down
+  end
+end
diff --git a/db/migrate/20231018141450_add_cpu_cores_columns_to_resource_pools.rb b/db/migrate/20231018141450_add_cpu_cores_columns_to_resource_pools.rb
@@ -0,0 +1,34 @@
+class AddCpuCoresColumnsToResourcePools < ActiveRecord::Migration[6.0]
+  class ResourcePool < ActiveRecord::Base
+    include ActiveRecord::IdRegions
+    self.inheritance_column = :_type_disabled
+  end
+
+  def up
+    add_column :resource_pools, :cpu_cores_available, :float
+    add_column :resource_pools, :cpu_cores_reserve, :float
+    add_column :resource_pools, :cpu_cores_limit, :float
+
+    say_with_time('Removing non-applicable data from IbmCloud::PowerVirtualServer resource pools') do
+      ResourcePool.in_my_region.where(:type => %w[ManageIQ::Providers::IbmCloud::PowerVirtualServers::CloudManager::ResourcePool]).update_all(
+        :cpu_shares  => nil,
+        :cpu_reserve => nil,
+        :cpu_limit   => nil
+      )
+    end
+
+    say_with_time('Removing non-applicable data from IbmPowerHmc resource pools') do
+      ResourcePool.in_my_region.where(:type => %w[ManageIQ::Providers::IbmPowerHmc::InfraManager::ProcessorResourcePool]).update_all(
+        :cpu_shares  => nil,
+        :cpu_reserve => nil,
+        :cpu_limit   => nil
+      )
+    end
+  end
+
+  def down
+    remove_column :resource_pools, :cpu_cores_available
+    remove_column :resource_pools, :cpu_cores_reserve
+    remove_column :resource_pools, :cpu_cores_limit
+  end
+end
diff --git a/lib/migration_helper.rb b/lib/migration_helper.rb
@@ -265,4 +265,24 @@ def previously_migrated_as?(bad_date)
       "DELETE FROM schema_migrations WHERE version = #{connection.quote(bad_date)}"
     ) > 0
   end
+
+  # sometimes a string in a column needs to be renamed to another value
+  # this uses update_all and regex REPLACE to do it database side
+  #
+  # @param model     model with the values
+  # @param column    column with the values
+  # @param old_value previous string that wants to be changed
+  # @param new_value new string that the value will be changed to
+  def regex_replace_column_value(model, column, old_value, new_value, batch_size: 50_000)
+    say_with_time("Removing #{old_value} from #{model.name}.#{column}") do
+      base_relation = model.where("#{column} LIKE ?", "%#{old_value}%")
+      say_batch_started(base_relation.size)
+      loop do
+        count = base_relation.limit(batch_size).update_all("#{column} = REGEXP_REPLACE(#{column}, '#{old_value}', '#{new_value}', 'g')")
+        break if count < batch_size
+
+        say_batch_processed(count)
+      end
+    end
+  end
 end
diff --git a/manageiq-schema.gemspec b/manageiq-schema.gemspec
@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "ancestry"
   spec.add_dependency "activerecord-id_regions", "~> 0.3.2"
   spec.add_dependency "linux_admin",             "~> 2.0"
-  spec.add_dependency "manageiq-password",       "< 2"
+  spec.add_dependency "manageiq-password",       ">= 1.2.0", "< 2"
   spec.add_dependency "more_core_extensions",    ">= 3.5", "< 5"
   spec.add_dependency "pg"
   spec.add_dependency "rails",                   ">=6.0.4", "<7.0"

diff --git a/spec/migrations/20180606155924_move_ansible_container_secrets_into_database_spec.rb b/spec/migrations/20180606155924_move_ansible_container_secrets_into_database_spec.rb
@@ -111,7 +111,7 @@
   end
 
   def expect_request
-    expect(File).to receive(:read).with(token_path).and_return("totally-a-token")
+    expect(described_class).to receive(:read_token).with(token_path).and_return("totally-a-token")
     response = double("RequestIO", :read => secret_json)
     expect(uri_stub).to receive(:open).with({
       'Accept'         => "application/json",

diff --git a/spec/migrations/20230919143746_remove_vim_strings_from_miq_queue_spec.rb b/spec/migrations/20230919143746_remove_vim_strings_from_miq_queue_spec.rb
@@ -0,0 +1,54 @@
+require_migration
+
+# This is mostly necessary for data migrations, so feel free to delete this
+# file if you do no need it.
+describe RemoveVimStringsFromMiqQueue do
+  let(:miq_queue_stub) { migration_stub(:MiqQueue) }
+
+  migration_context :up do
+    it "converts VimStrings to Strings" do
+      queue_args = <<~CONTEXT
+        ---
+        - :event_type: DrsMigrateVM_Task
+          :chain_id: &1 !ruby/string:VimString
+            str: '12345678'
+            xsiType: :SOAP::SOAPInt
+            vimType:
+          :is_task: true
+          :source: VC
+          :message: &3 !ruby/string:VimString
+            str: 'Task: Migrate virtual machine'
+            xsiType: :SOAP::SOAPString
+            vimType:
+          :timestamp: &2 !ruby/string:VimString
+            str: '2022-10-06T06:40:59.901255Z'
+            xsiType: :SOAP::SOAPDateTime
+            vimType:
+          :full_data: !ruby/hash-with-ivars:VimHash
+            elements:
+              key: !ruby/string:VimString
+                str: '12345678'
+                xsiType: :SOAP::SOAPInt
+                vimType:
+              chainId: *1
+              createdTime: *2
+              userName: !ruby/string:VimString
+                str: ''
+                xsiType: :SOAP::SOAPString
+                vimType:
+      CONTEXT
+
+      queue_item = miq_queue_stub.create!(
+        :args => queue_args
+      )
+
+      migrate
+      queue_item.reload
+
+      args = YAML.safe_load(queue_item.args, :permitted_classes => [Symbol, Date, String, Hash], :aliases => true)
+      arg = args.first
+      expect(arg.class).to eq(Hash)
+      expect(arg[:chain_id].class).to eq(String)
+    end
+  end
+end