MAM-3095-load-test-20k-queues

app/workers/for_you_feed_worker.rb

@@ -7,7 +7,7 @@ class ForYouFeedWorker
   include Redisable
   include Sidekiq::Worker
 
-  sidekiq_options queue: 'mammoth', retry: 0
+  sidekiq_options queue: 'mammoth_default', retry: 0
 
   MAX_ITEMS = 1000
   MINIMUM_ENGAGMENT_ACTIONS = 2

app/workers/scheduler/for_you_statuses_scheduler.rb

@@ -13,7 +13,7 @@ class Scheduler::ForYouStatusesScheduler
   include Sidekiq::Worker
   include JsonLdHelper
 
-  sidekiq_options retry: 0, queue: 'mammoth'
+  sidekiq_options retry: 0, queue: 'mammoth_default'
 
   def perform
     owner_account = Account.local.where(username: FOR_YOU_OWNER_ACCOUNT)

app/workers/update_for_you_worker.rb

@@ -4,7 +4,7 @@ class UpdateForYouWorker
   include Redisable
   include Sidekiq::Worker
 
-  sidekiq_options retry: 0, queue: 'mammoth'
+  sidekiq_options retry: 0, queue: 'mammoth_default'
 
   # Mammoth Curated List(OG List)
 

config/sidekiq.yml

@@ -7,7 +7,8 @@
   - [mailers, 2]
   - [pull]
   - [scheduler]
-  - [mammoth]
+  - [mammoth_critial]
+  - [mammoth_default]
 :scheduler:
   :listened_queues_only: true
   :schedule:

dist/default/mastodon-sidekiq

@@ -0,0 +1,20 @@
+# /etc/default/mastodon-sidekiq
+# Mastodon sidekiq environment variables
+
+# Reduce memory pressure on Linux.
+# This setting halves the amount of memory needed compared to default
+MALLOC_ARENA_MAX=2
+
+# Use the libjemalloc library on Linux
+LD_PRELOAD=libjemalloc.so
+
+# Sidekiq service thread counts
+# The number of threads needs to match the DB_POOL/MAX_THREADS environment
+# setting or we'll run out of database connections.
+# If DB_POOL is set, it takes precedence over MAX_THREADS. If neither are
+# set, mastodon defaults to 5 threads.
+DB_POOL=10
+
+# Default queueset config
+# 
+#QUEUESET="-q default,5 -q scheduler,10 -q mailers,2 -q ingress,5 -q push,5 -q pull,5"

dist/default/mastodon-sidekiq-mammoth_critical

@@ -0,0 +1,20 @@
+# /etc/default/mastodon-sidekiq-%j
+# Mastodon sidekiq environment variables
+
+# Reduce memory pressure on Linux.
+# This setting halves the amount of memory needed compared to default
+MALLOC_ARENA_MAX=2
+
+# Use the libjemalloc library on Linux
+LD_PRELOAD=libjemalloc.so
+
+# Sidekiq service thread counts
+# The number of threads needs to match the DB_POOL/MAX_THREADS environment
+# setting or we'll run out of database connections.
+# If DB_POOL is set, it takes precedence over MAX_THREADS. If neither are
+# set, mastodon defaults to 5 threads.
+DB_POOL=25
+
+# Mammoth_Critical queueset config
+# 
+QUEUESET="-q mammoth_critial,5 -q mammoth_default,1"

dist/default/mastodon-sidekiq-mammoth_default

@@ -0,0 +1,20 @@
+# /etc/default/mastodon-sidekiq-%j
+# Mastodon sidekiq environment variables
+
+# Reduce memory pressure on Linux.
+# This setting halves the amount of memory needed compared to default
+MALLOC_ARENA_MAX=2
+
+# Use the libjemalloc library on Linux
+LD_PRELOAD=libjemalloc.so
+
+# Sidekiq service thread counts
+# The number of threads needs to match the DB_POOL/MAX_THREADS environment
+# setting or we'll run out of database connections.
+# If DB_POOL is set, it takes precedence over MAX_THREADS. If neither are
+# set, mastodon defaults to 5 threads.
+DB_POOL=25
+
+# Mammoth_Default queueset config
+# 
+QUEUESET="-q mammoth_default,1 -q mammoth_critial,1"

dist/scaling-service/mastodon-sidekiq-<queuename>@.service

@@ -0,0 +1,59 @@
+[Unit]
+# Set up a mastodon-sidekiq-<queuename>@.service unit for each queue
+# Configure at least one instance of each queue type to run
+# If we need more processing for a given queue type, enable
+# a second, third, etc. service and start it. Then stop it/disable it
+# if it's no longer needed once the load spike has passed.
+Description=Mastodon Sidekiq %j processor %i
+After=network.target
+
+[Service]
+Type=simple
+User=mastodon
+WorkingDirectory=/home/mastodon/live
+EnvironmentFile=/etc/default/mastodon-sidekiq
+# Queue specific configuration overrides defaults
+EnvironmentFile=-/etc/default/mastodon-sidekiq-%j
+Environment="RAILS_ENV=production"
+
+ExecStart=/usr/bin/bash -l -c '/home/mastodon/.rbenv/shims/bundle exec sidekiq -e production -c $DB_POOL $QUEUESET'
+TimeoutSec=15
+Restart=always
+# Proc filesystem
+ProcSubset=pid
+ProtectProc=invisible
+# Capabilities
+CapabilityBoundingSet=
+# Security
+NoNewPrivileges=true
+# Sandboxing
+ProtectSystem=strict
+PrivateTmp=true
+PrivateDevices=true
+PrivateUsers=true
+ProtectHostname=true
+ProtectKernelLogs=true
+ProtectKernelModules=true
+ProtectKernelTunables=true
+ProtectControlGroups=true
+RestrictAddressFamilies=AF_INET
+RestrictAddressFamilies=AF_INET6
+RestrictAddressFamilies=AF_NETLINK
+RestrictAddressFamilies=AF_UNIX
+RestrictNamespaces=true
+LockPersonality=true
+RestrictRealtime=true
+RestrictSUIDSGID=true
+RemoveIPC=true
+PrivateMounts=true
+ProtectClock=true
+# System Call Filtering
+SystemCallArchitectures=native
+SystemCallFilter=~@cpu-emulation @debug @keyring @ipc @mount @obsolete @privileged @setuid
+SystemCallFilter=@chown
+SystemCallFilter=pipe
+SystemCallFilter=pipe2
+ReadWritePaths=/home/mastodon/live
+
+[Install]
+WantedBy=multi-user.target