diff --git a/Gemfile b/Gemfile index f7cda4c..a7db1a5 100644 --- a/Gemfile +++ b/Gemfile @@ -34,6 +34,9 @@ gem 'puma', '5.5.2' gem 'rake', '13.0.6' +# Delayed Job +gem 'delayed_job_active_record' + # Code lint gem 'rubocop', '1.56.1', group: %i[development test] gem 'rubocop-performance', group: %i[development test] diff --git a/Gemfile.lock b/Gemfile.lock index 7530fd2..088517d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -24,6 +24,11 @@ GEM activerecord (>= 5.a) database_cleaner-core (~> 2.0.0) database_cleaner-core (2.0.1) + delayed_job (4.1.11) + activesupport (>= 3.0, < 8.0) + delayed_job_active_record (4.1.8) + activerecord (>= 3.0, < 8.0) + delayed_job (>= 3.0, < 5) diff-lcs (1.5.0) docile (1.4.0) factory_bot (6.2.1) @@ -152,6 +157,7 @@ PLATFORMS DEPENDENCIES database_cleaner + delayed_job_active_record factory_bot faker jwt (= 2.2.2) diff --git a/Rakefile b/Rakefile index 8d3e9f4..62e12e2 100644 --- a/Rakefile +++ b/Rakefile @@ -13,6 +13,25 @@ require 'otr-activerecord' load 'tasks/otr-activerecord.rake' +require_relative 'config/delayed_job' +require_relative 'config/setup' + +namespace :jobs do + desc 'Clear the delayed_job queue.' + task :clear do + Delayed::Job.delete_all + end + + desc 'Start a delayed_job worker.' + task :work do + puts "Starting delayed_job worker - Queues: #{ENV.fetch('QUEUES', 'default')}" + Delayed::Worker.new( + queue: ENV.fetch('QUEUES', 'default'), + quiet: false + ).start + end +end + namespace :db do # Some db tasks require your app code to be loaded; they'll expect to find it here task :environment do diff --git a/bin/console.rb b/bin/console.rb index b6424af..4633219 100755 --- a/bin/console.rb +++ b/bin/console.rb @@ -16,5 +16,6 @@ puts "Starting console: #{ENV.fetch('RAILS_ENV', nil)}" require_relative '../config/setup' +require_relative '../config/delayed_job' IRB.start diff --git a/config.ru b/config.ru index b8d98ff..bfc2438 100755 --- a/config.ru +++ b/config.ru @@ -9,6 +9,7 @@ # frozen_string_literal: true require_relative 'app/github_app' +require_relative 'config/delayed_job' require 'puma' require 'rack/handler/puma' @@ -16,8 +17,19 @@ require 'rack/session/cookie' File.write('.session.key', SecureRandom.hex(32)) +pids = [] +pids << spawn("RACK_ENV=#{ENV.fetch('RACK_ENV', 'development')} rake jobs:work QUEUES=0,1,2,3") +pids << spawn("RACK_ENV=#{ENV.fetch('RACK_ENV', 'development')} rake jobs:work QUEUES=4,5,6") +pids << spawn("RACK_ENV=#{ENV.fetch('RACK_ENV', 'development')} rake jobs:work QUEUES=7,8,9") + use Rack::Session::Cookie, secret: File.read('.session.key'), same_site: true, max_age: 86_400 Rack::Handler::Puma.run Rack::URLMap.new('/' => GithubApp) +pids.each do |pid| + Process.kill('TERM', pid.to_i) +rescue Errno::ESRCH + puts "Process #{pid} already dead" +end + exit 0 diff --git a/config/delayed_job.rb b/config/delayed_job.rb new file mode 100644 index 0000000..eb33989 --- /dev/null +++ b/config/delayed_job.rb @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# delayed_job.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +require_relative '../lib/helpers/github_logger' +require_relative '../database_loader' + +require 'delayed_job' +require 'active_support' + +module Rails + class << self + attr_accessor :logger + end +end + +DELAYED_JOB_TIMER = 5 + +Rails.logger = GithubLogger.instance.create('delayed_job.log', Logger::INFO) +ActiveRecord::Base.logger = GithubLogger.instance.create('delayed_job.log', Logger::INFO) + +# this is used by DJ to guess where tmp/pids is located (default) +RAILS_ROOT = File.expand_path(__FILE__) + +Delayed::Worker.backend = :active_record +Delayed::Worker.destroy_failed_jobs = true +Delayed::Worker.sleep_delay = 5 +Delayed::Worker.max_attempts = 5 +Delayed::Worker.max_run_time = 5.minutes + +config = YAML.load_file('config/database.yml')[ENV.fetch('RACK_ENV', 'development')] +ActiveRecord::Base.establish_connection(config) diff --git a/db/migrate/20240617121935_create_delayed_jobs.rb b/db/migrate/20240617121935_create_delayed_jobs.rb new file mode 100644 index 0000000..d048d92 --- /dev/null +++ b/db/migrate/20240617121935_create_delayed_jobs.rb @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# 20240617121935_create_delayed_jobs.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +class CreateDelayedJobs < ActiveRecord::Migration[6.0] + def change + create_table :delayed_jobs do |t| + t.integer :priority, default: 0, null: false + t.integer :attempts, default: 0, null: false + t.text :handler, null: false + t.text :last_error + t.datetime :run_at + t.datetime :locked_at + t.datetime :failed_at + t.string :locked_by + t.string :queue + t.timestamps null: false + end + end +end diff --git a/db/schema.rb b/db/schema.rb index ff946ad..5dafe7f 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2024_04_17_130601) do +ActiveRecord::Schema[7.0].define(version: 2024_06_17_121935) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -81,6 +81,20 @@ t.index ["stage_id"], name: "index_ci_jobs_on_stage_id" end + create_table "delayed_jobs", force: :cascade do |t| + t.integer "priority", default: 0, null: false + t.integer "attempts", default: 0, null: false + t.text "handler", null: false + t.text "last_error" + t.datetime "run_at", precision: nil + t.datetime "locked_at", precision: nil + t.datetime "failed_at", precision: nil + t.string "locked_by" + t.string "queue" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "github_users", force: :cascade do |t| t.string "github_login" t.string "github_username" diff --git a/lib/github/check.rb b/lib/github/check.rb index 84120b6..a09fc62 100644 --- a/lib/github/check.rb +++ b/lib/github/check.rb @@ -51,11 +51,13 @@ def add_comment(pr_id, comment, repo) end def comment_reaction_thumb_up(repo, comment_id) - @app.create_issue_comment_reaction(repo, comment_id, '+1') + @app.create_issue_comment_reaction(repo, comment_id, '+1', + accept: Octokit::Preview::PREVIEW_TYPES[:reactions]) end def comment_reaction_thumb_down(repo, comment_id) - @app.create_issue_comment_reaction(repo, comment_id, '-1') + @app.create_issue_comment_reaction(repo, comment_id, '-1', + accept: Octokit::Preview::PREVIEW_TYPES[:reactions]) end def create(name) @@ -63,7 +65,7 @@ def create(name) @check_suite.pull_request.repository, name, @check_suite.commit_sha_ref, - accept: 'application/vnd.github.antiope-preview+json' + accept: Octokit::Preview::PREVIEW_TYPES[:checks] ) end @@ -94,7 +96,7 @@ def skipped(check_ref, output = {}) def get_check_run(check_ref) @app.check_run(@check_suite.pull_request.repository, check_ref, - accept: 'application/vnd.github.antiope-preview+json').to_h + accept: Octokit::Preview::PREVIEW_TYPES[:checks]).to_h end def fetch_check_runs @@ -104,7 +106,7 @@ def fetch_check_runs .check_runs_for_ref( @check_suite.pull_request.repository, @check_suite.pull_request.branch_name, - accept: 'application/vnd.github.antiope-preview+json' + accept: Octokit::Preview::PREVIEW_TYPES[:checks] ) .to_h[:check_runs] .map do |check_run| @@ -135,7 +137,7 @@ def fetch_username(username) def basic_status(check_ref, status, output) opts = { status: status, - accept: 'application/vnd.github.antiope-preview+json' + accept: Octokit::Preview::PREVIEW_TYPES[:checks] } opts[:output] = output unless output.empty? diff --git a/lib/github/update_status.rb b/lib/github/update_status.rb index f597b15..a9ec4ba 100644 --- a/lib/github/update_status.rb +++ b/lib/github/update_status.rb @@ -79,7 +79,7 @@ def update_status return [200, 'Success'] unless @job.check_suite.pull_request.current_execution? @job.check_suite - update_build_summary_or_finished + insert_new_delayed_job [200, 'Success'] rescue StandardError => e @@ -88,16 +88,31 @@ def update_status [500, 'Internal Server Error'] end - def update_build_summary_or_finished - summary = Github::Build::Summary.new(@job) - summary.build_summary + def insert_new_delayed_job + queue = @job.check_suite.pull_request.github_pr_id % 10 - return unless @job.finished? + if can_add_new_job? + return CiJobStatus + .delay(run_at: DELAYED_JOB_TIMER.seconds.from_now, queue: queue) + .update(@job.check_suite.id, @job.id) + end + + delete_and_create_delayed_job(queue) + end - logger(Logger::INFO, "Github::PlanExecution::Finished: '#{@job.check_suite.bamboo_ci_ref}'") + def delete_and_create_delayed_job(queue) + fetch_delayed_job.destroy_all + CiJobStatus + .delay(run_at: DELAYED_JOB_TIMER.seconds.from_now, queue: queue) + .update(@job.check_suite.id, @job.id) + end + + def can_add_new_job? + fetch_delayed_job.empty? + end - finished = Github::PlanExecution::Finished.new({ 'bamboo_ref' => @job.check_suite.bamboo_ci_ref }) - finished.finished + def fetch_delayed_job + Delayed::Job.where('handler LIKE ?', "%method_name: :update\nargs:\n- #{@job.check_suite.id}%") end def current_execution? diff --git a/lib/github_ci_app.rb b/lib/github_ci_app.rb index b09faf4..58cf6b3 100644 --- a/lib/github_ci_app.rb +++ b/lib/github_ci_app.rb @@ -36,6 +36,8 @@ require_relative 'helpers/sinatra_payload' require_relative 'helpers/telemetry' +require_relative '../workers/ci_job_status' + # Slack libs require_relative 'slack/slack' diff --git a/spec/lib/github/check_spec.rb b/spec/lib/github/check_spec.rb index a2ad915..89d5b97 100644 --- a/spec/lib/github/check_spec.rb +++ b/spec/lib/github/check_spec.rb @@ -74,7 +74,9 @@ let(:pr_info) { { comment_id: comment_id } } before do - allow(fake_client).to receive(:create_issue_comment_reaction).with(repo, comment_id, '+1').and_return(pr_info) + allow(fake_client).to receive(:create_issue_comment_reaction) + .with(repo, comment_id, '+1', accept: Octokit::Preview::PREVIEW_TYPES[:reactions]) + .and_return(pr_info) end it 'must returns pull request info' do diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index be86bd0..7a2393f 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -26,6 +26,7 @@ SimpleCov.start require_relative '../app/github_app' +require_relative '../config/delayed_job' def app GithubApp @@ -38,8 +39,26 @@ def app config.include FactoryBot::Syntax::Methods config.include WebMock::API + pid = nil + config.before(:all) do DatabaseCleaner.clean + + pid = Thread.new do + Delayed::Worker.new( + min_priority: 0, + max_priority: 10, + quiet: true + ).start + end + end + + config.after(:all) do + pid&.exit + end + + config.before(:each) do + Delayed::Worker.delay_jobs = false end config.after(:each) do diff --git a/workers/ci_job_status.rb b/workers/ci_job_status.rb new file mode 100644 index 0000000..970244c --- /dev/null +++ b/workers/ci_job_status.rb @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# ci_job_status.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +require_relative '../config/setup' + +class CiJobStatus + def self.update(check_suite_id, ci_job_id) + @logger = GithubLogger.instance.create('ci_job_status.log', Logger::INFO) + @logger.info("CiJobStatus::Update: Checksuite #{check_suite_id} -> '#{ci_job_id}'") + + job = CiJob.find(ci_job_id) + + summary = Github::Build::Summary.new(job) + summary.build_summary + + return unless job.finished? + + @logger.info("Github::PlanExecution::Finished: '#{job.check_suite.bamboo_ci_ref}'") + + finished = Github::PlanExecution::Finished.new({ 'bamboo_ref' => job.check_suite.bamboo_ci_ref }) + finished.finished + end +end