CCAP-304: Add support for background jobs (#9)

* Added background jobs with job to deactivate expired auth keys.
* Added bootstrap system
* Added linting for code documentation.
* Added markdown linting.
* Replace unmaintained GitHub action for rubocop

.github/workflows/branch.yaml

@@ -12,10 +12,12 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - run: git fetch origin main --depth=1
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
       - name: RuboCop Linter
-        uses: andrewmcodes/rubocop-linter-action@v3.3.0
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: bundle exec rubocop
 
   spec:
     runs-on: ubuntu-latest

.github/workflows/main.yaml

@@ -14,10 +14,12 @@ jobs:
         with:
           fetch-depth: 1
       - run: git fetch origin main --depth=1
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
       - name: RuboCop Linter
-        uses: andrewmcodes/rubocop-linter-action@v3.3.0
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: bundle exec rubocop
 
   spec:
     runs-on: ubuntu-latest

.gitignore

@@ -3,6 +3,7 @@
 *.env
 !sample.env
 coverage/
+log/
 
 # Ignore Byebug command history file.
 .byebug_history
@@ -17,6 +18,7 @@ coverage/
 
 # Include project IDE settings, but ignore user-specific settings.
 !.idea/
+.idea/dataSources.xml
 *.iml
 *.iws
 out/

.pryrc

@@ -1,25 +1,7 @@
 # frozen_string_literal: true
 
-require 'sequel'
-
 require_relative 'lib/config/application'
-require_relative 'lib/model'
+require_relative 'lib/bootstrap/console'
 
 config = DocumentTransfer::Config::Application.from_environment
-Sequel.connect(config.database_credentials)
-
-DocumentTransfer::Model.load
-
-color = "\e[1;32m"
-color = "\e[1;31m" if config.prod?
-color = "\e[1;33m" if config.prod_like?
-
-Pry.config.prompt_name = "document-transfer(#{config.environment})"
-Pry.config.prompt = Pry::Prompt.new(
-  :document_transfer,
-  'Document transfer console prompt',
-  [
-    proc { |_, _, p| "#{color}[#{p.input_ring.count}] #{p.config.prompt_name} > \e[0m" },
-    proc { |_, _, p| "#{color}[#{p.input_ring.count}] #{p.config.prompt_name} * \e[0m" }
-  ]
-)
+DocumentTransfer::Bootstrap::Console.new(config).bootstrap

.rubocop.yml

@@ -1,8 +1,10 @@
 require:
   - rubocop-factory_bot
+  - rubocop-md
   - rubocop-rake
   - rubocop-rspec
   - rubocop-sequel
+  - rubocop-yard
 
 AllCops:
   NewCops: enable
@@ -18,6 +20,12 @@ AllCops:
 FactoryBot/FactoryAssociationWithStrategy:
   Enabled: false
 
+Layout/InitialIndentation:
+  Exclude:
+    # Runbooks often have indented code blocks and rubocop doesn't like that.
+    # https://github.com/rubocop/rubocop-md/issues/28
+    - doc/runbooks/**/*.md
+
 # Don't let long declarations and invocations to contribute to method length.
 # This is preferable over squeezing them onto a single line.
 Metrics/MethodLength:

Dockerfile

@@ -16,4 +16,4 @@ RUN bundle binstubs --all
 # Copy the application code.
 COPY . .
 
-CMD ["bundle", "exec", "rackup", "--port", "3000", "--host", "0.0.0.0"]
+CMD ["./script/api"]

Gemfile

@@ -13,9 +13,11 @@ group :development do
   gem 'rake', '~> 13.0'
   gem 'rubocop', '~> 1.48'
   gem 'rubocop-factory_bot', '~> 2.23'
+  gem 'rubocop-md', '~> 1.2'
   gem 'rubocop-rake', '~> 0.6'
   gem 'rubocop-rspec', '~> 2.22'
   gem 'rubocop-sequel', '~> 0.3'
+  gem 'rubocop-yard', '~> 0.9.3'
 end
 
 group :test do

Gemfile.lock

@@ -10,15 +10,20 @@ PATH
   remote: .
   specs:
     document-transfer-service (0.1.0)
+      activesupport (~> 7.1.0)
       adal (~> 1.0)
       bcrypt (~> 3.1)
+      daemons (~> 1.4)
+      delayed_job (~> 4.1)
       faraday (~> 2.9)
+      fugit (~> 1.11)
       grape (~> 2.0)
       grape-entity (~> 1.0)
       grape-swagger (~> 2.1)
       grape-swagger-entity (~> 0.5)
       httparty (~> 0.22)
       opentelemetry-exporter-otlp (~> 0.27)
+      opentelemetry-instrumentation-delayed_job (~> 0.22)
       opentelemetry-instrumentation-faraday (~> 0.24)
       opentelemetry-instrumentation-grape (~> 0.1)
       opentelemetry-instrumentation-rack (~> 0.24)
@@ -55,6 +60,9 @@ GEM
     concurrent-ruby (1.3.3)
     connection_pool (2.4.1)
     csv (3.3.0)
+    daemons (1.4.1)
+    delayed_job (4.1.11)
+      activesupport (>= 3.0, < 8.0)
     diff-lcs (1.5.1)
     docile (1.4.0)
     drb (2.2.1)
@@ -73,6 +81,8 @@ GEM
       dry-inflector (~> 1.0)
       dry-logic (~> 1.4)
       zeitwerk (~> 2.6)
+    et-orbi (1.2.11)
+      tzinfo
     factory_bot (6.4.6)
       activesupport (>= 5.0.0)
     faker (3.4.1)
@@ -82,6 +92,9 @@ GEM
       logger
     faraday-net_http (3.1.0)
       net-http
+    fugit (1.11.0)
+      et-orbi (~> 1, >= 1.2.11)
+      raabro (~> 1.4)
     google-protobuf (4.27.2)
       bigdecimal
       rake (>= 13)
@@ -166,6 +179,9 @@ GEM
       opentelemetry-api (~> 1.0)
       opentelemetry-common (~> 0.21)
       opentelemetry-registry (~> 0.1)
+    opentelemetry-instrumentation-delayed_job (0.22.4)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
     opentelemetry-instrumentation-faraday (0.24.5)
       opentelemetry-api (~> 1.0)
       opentelemetry-instrumentation-base (~> 0.22.1)
@@ -193,6 +209,7 @@ GEM
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
+    raabro (1.4.0)
     racc (1.8.0)
     rack (3.1.7)
     rack-test (2.1.0)
@@ -239,6 +256,8 @@ GEM
       rubocop (~> 1.41)
     rubocop-factory_bot (2.26.1)
       rubocop (~> 1.61)
+    rubocop-md (1.2.2)
+      rubocop (>= 1.0)
     rubocop-rake (0.6.0)
       rubocop (~> 1.0)
     rubocop-rspec (2.31.0)
@@ -250,6 +269,9 @@ GEM
       rubocop (~> 1.61)
     rubocop-sequel (0.3.4)
       rubocop (~> 1.0)
+    rubocop-yard (0.9.3)
+      rubocop (~> 1.21)
+      yard
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
     semantic_logger (4.16.0)
@@ -280,6 +302,7 @@ GEM
     uri (0.13.0)
     uri_template (0.7.0)
     webrick (1.8.1)
+    yard (0.9.36)
     zeitwerk (2.6.16)
 
 PLATFORMS
@@ -311,9 +334,11 @@ DEPENDENCIES
   rspec-uuid (~> 0.6)
   rubocop (~> 1.48)
   rubocop-factory_bot (~> 2.23)
+  rubocop-md (~> 1.2)
   rubocop-rake (~> 0.6)
   rubocop-rspec (~> 2.22)
   rubocop-sequel (~> 0.3)
+  rubocop-yard (~> 0.9.3)
   simplecov (~> 0.22)
   sqlite3 (~> 2.0)
 

README.md

@@ -38,7 +38,7 @@ service.
 Using docker compose, the application code will be mounted from your local
 system to the running container. This will allow you to make changes to the
 code and see them reflected in the running service without having to rebuild the
-image.
+image. Using docker compose will launch the api, the worker, and a database. 
 
 To run the service with docker compose, make sure you have [Docker Desktop]
 installed and run the following:
@@ -88,11 +88,19 @@ bundle exec rake db:setup
 You should now be able to start the service with:
 
 ```sh
-bundle exec rackup
+./script/api
 ```
 
 The service should now be available at `http://localhost:9292`.
 
+#### Running the worker
+
+The worker is a separate process that is responsible for processing background
+jobs. Depending on the parts of the service you're using, you may need to run
+the worker alongside the API.
+
+To run the worker, see the [worker documentation][worker] for more information.
+
 #### Updating the service
 
 When you update the service locally, you will need to install any new
@@ -136,3 +144,4 @@ service.
 [ruby-version]: ./.ruby-version
 [rvm]: https://rvm.io/
 [source]: ./doc/sources.md
+[worker]: ./doc/worker.md

Rakefile

@@ -6,6 +6,15 @@ require 'rubocop/rake_task'
 
 require_relative 'lib/document_transfer'
 
+require_relative 'lib/bootstrap/rake'
+require_relative 'lib/config/application'
+
+require_relative 'lib/api/api'
+
+# Bootstrap the application for rake.
+config = DocumentTransfer::Config::Application.from_environment
+DocumentTransfer::Bootstrap::Rake.new(config).bootstrap
+
 task default: %i[spec rubocop]
 
 task :environment do # rubocop:disable Rake/Desc
@@ -16,9 +25,8 @@ GrapeSwagger::Rake::OapiTasks.new('::DocumentTransfer::API::API')
 
 RuboCop::RakeTask.new(:rubocop) do |task|
   task.requires << 'rubocop'
+  task.formatters = %w[pacman]
+  task.formatters << 'github' if ENV.fetch('GITHUB_ACTIONS', false)
 end
 
 RSpec::Core::RakeTask.new(:spec)
-
-# Load our custom tasks.
-DocumentTransfer.load_rake_tasks

config.ru

@@ -1,40 +1,18 @@
 # frozen_string_literal: true
 
-require 'opentelemetry/sdk'
-require 'opentelemetry-exporter-otlp'
-require 'opentelemetry/instrumentation/faraday'
-require 'opentelemetry/instrumentation/grape'
+require 'rackup'
 require 'opentelemetry/instrumentation/rack'
-require 'semantic_logger'
-require 'sequel'
 
-require_relative 'lib/document_transfer'
+require_relative 'lib/bootstrap/api'
+require_relative 'lib/config/application'
 require_relative 'lib/api/api'
 require_relative 'lib/api/middleware'
-require_relative 'lib/config/application'
-require_relative 'lib/model'
 
-# Connect to the database.
+# Bootstrap the application.
 config = DocumentTransfer::Config::Application.from_environment
-Sequel.connect(config.database_credentials)
-
-# Load all models.
-DocumentTransfer::Model.load
-
-# Configure the logger.
-SemanticLogger.default_level = ENV.fetch('LOG_LEVEL',
-                                         DocumentTransfer::DEFAULT_LOG_LEVEL)
-SemanticLogger.application = DocumentTransfer::NAME
-SemanticLogger.add_appender(io: $stdout, formatter: :json)
-
-# Configure telemetry reporting.
-OpenTelemetry::SDK.configure do |c|
-  c.service_name = DocumentTransfer::NAME
-  c.service_version = DocumentTransfer::VERSION
-  c.use_all
-end
+DocumentTransfer::Bootstrap::API.new(config).bootstrap
 
-# Include Rack middleware.
+# Load Rack middleware.
 use Rack::RewindableInput::Middleware
 use(*OpenTelemetry::Instrumentation::Rack::Instrumentation.instance.middleware_args)
 DocumentTransfer::API::Middleware.load.each { |m| use m }