Hardcoded latest versions, gql-cache, import-ssg task (RedHatInsights…

…#413)

* Fix transient profile test failure (RedHatInsights#407)

Signed-off-by: Andrew Kofink <[email protected]>

* Hardcode latest RHEL supported versions (RedHatInsights#408)

* Hardcode latest RHEL supported versions

* Add cache for latest supported versions

* Freeze constant

* Return an AR relation

* Properly search for match ref_id/version

* Task to import supported SSG benchmarks (RedHatInsights#412)

* Task to import supported SSG benchmarks

* Regex way of getting RHEL number

Co-Authored-By: Andrew Kofink <[email protected]>

* Fix rubocop

Co-authored-by: Andrew Kofink <[email protected]>

* Introduce graphql-cache (RedHatInsights#411)

* GQL-cache POC

* Don't cache batched fields

* Cache lazy types

* Wait for promises before saving to cache

* Re-enable promise fields and to_s

* Use hiredis

* Uncomment timeout in schema

* Rubocop fixes

* Fix href/label test

* Add benchmarks API endpoint (RedHatInsights#409)

Signed-off-by: Andrew Kofink <[email protected]>

* Fixes for rake tasks  (RedHatInsights#414)

* Require HostInventoryNotFound

* Execute instead of invoking import SSG task

* Add specific config for cache redis (RedHatInsights#415)

Co-authored-by: Andrew Kofink <[email protected]>
Co-authored-by: Andrew Kofink <[email protected]>

Gemfile

@@ -11,7 +11,7 @@ gem 'puma', '~> 3.12'
 # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder
 # gem 'jbuilder', '~> 2.5'
 # Use Redis adapter to run Action Cable in production
-gem 'redis', '~> 4.0'
+gem 'hiredis'
 # Use ActiveModel has_secure_password
 # gem 'bcrypt', '~> 3.1.7'
 
@@ -68,13 +68,14 @@ gem 'prometheus_exporter'
 
 gem 'activerecord-import'
 gem 'oj'
-gem 'newrelic_rpm'
 gem 'gitlab-sidekiq-fetcher', require: 'sidekiq-reliable-fetch'
 
 gem 'uuid'
 
 gem 'openscap_parser', '~> 1.0.0'
 
+gem 'graphql-cache'
+
 group :development, :test do
   gem 'brakeman'
   gem 'byebug', platforms: %i[mri mingw x64_mingw]

Gemfile.lock

@@ -145,7 +145,10 @@ GEM
     graphql-batch (0.4.2)
       graphql (>= 1.3, < 2)
       promise.rb (~> 0.7.2)
+    graphql-cache (0.6.0)
+      graphql (~> 1, > 1.8)
     hashdiff (1.0.0)
+    hiredis (0.6.3)
     i18n (1.8.2)
       concurrent-ruby (~> 1.0)
     io-console (0.5.4)
@@ -183,7 +186,6 @@ GEM
     msgpack (1.3.1)
     multi_json (1.14.1)
     multipart-post (2.1.1)
-    newrelic_rpm (6.8.0.360)
     nio4r (2.5.2)
     nokogiri (1.10.9)
       mini_portile2 (~> 2.4.0)
@@ -370,11 +372,12 @@ DEPENDENCIES
   graphiql-rails
   graphql
   graphql-batch
+  graphql-cache
+  hiredis
   irb (>= 1.2)
   listen (>= 3.0.5, < 3.2)
   minitest-reporters
   mocha
-  newrelic_rpm
   oj
   openscap_parser (~> 1.0.0)
   pg
@@ -387,7 +390,6 @@ DEPENDENCIES
   racecar
   rack-cors
   rails (~> 5.2.1)
-  redis (~> 4.0)
   rspec-rails
   rswag-api
   rswag-specs

app/controllers/benchmarks_controller.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# API for Benchmarks
+class BenchmarksController < ApplicationController
+  def index
+    render json: benchmarks
+  end
+
+  def show
+    render json: benchmark
+  end
+
+  private
+
+  def benchmarks
+    @benchmarks ||= BenchmarkSerializer.new(authorize(scope_search), metadata)
+  end
+
+  def benchmark
+    @benchmark ||= BenchmarkSerializer.new(
+      authorize(resource.find(params[:id]))
+    )
+  end
+
+  def scope
+    policy_scope(resource)
+  end
+
+  def resource
+    Xccdf::Benchmark
+  end
+end

app/graphql/schema.rb

@@ -15,6 +15,5 @@ class Schema < GraphQL::Schema
   mutation Types::Mutation
   lazy_resolve(Promise, :sync)
   use GraphQL::Batch
-  use GraphQL::Execution::Interpreter
-  use GraphQL::Analysis::AST
+  use GraphQL::Cache
 end

app/graphql/types/base_object.rb

@@ -9,6 +9,7 @@ class BaseObject < GraphQL::Types::Relay::BaseObject
     global_id_field :global_id
     field :node, field: GraphQL::Relay::Node.field
     field :nodes, field: GraphQL::Relay::Node.plural_field
+    field_class ::GraphQL::Cache::Field
 
     class << self
       def model_class(new_model_class = nil)

app/graphql/types/benchmark.rb

@@ -6,13 +6,13 @@ class Benchmark < Types::BaseObject
     graphql_name 'Benchmark'
     description 'A representation of a SCAP Security Guide version'
 
-    field :id, ID, null: false
-    field :description, String, null: true
-    field :title, String, null: false
-    field :ref_id, String, null: false
-    field :version, String, null: false
-    field :profiles, [::Types::Profile], null: true
-    field :rules, [::Types::Rule], null: true
+    field :id, ID, null: false, cache: true
+    field :description, String, null: true, cache: true
+    field :title, String, null: false, cache: true
+    field :ref_id, String, null: false, cache: true
+    field :version, String, null: false, cache: true
+    field :profiles, [::Types::Profile], null: true, cache: true
+    field :rules, [::Types::Rule], null: true, cache: true
 
     def profiles
       object.profiles.canonical

app/graphql/types/business_objective.rb

@@ -6,7 +6,7 @@ class BusinessObjective < Types::BaseObject
     graphql_name 'BusinessObjective'
     description 'A Business Objective registered in Insights Compliance'
 
-    field :id, ID, null: false
-    field :title, String, null: false
+    field :id, ID, null: false, cache: true
+    field :title, String, null: false, cache: true
   end
 end

app/graphql/types/interfaces/rules_preload.rb

@@ -4,6 +4,7 @@
 # rule results, compliant,and references
 module RulesPreload
   include GraphQL::Schema::Interface
+  field_class GraphQL::Cache::Field
 
   def rules(args = {})
     context[:parent_profile_id] ||= {}

app/graphql/types/profile.rb

@@ -13,59 +13,61 @@ class Profile < Types::BaseObject
     graphql_name 'Profile'
     description 'A Profile registered in Insights Compliance'
 
-    field :id, ID, null: false
-    field :name, String, null: false
-    field :description, String, null: true
-    field :ref_id, String, null: false
-    field :compliance_threshold, Float, null: false
-    field :benchmark_id, ID, null: false
-    field :account_id, ID, null: false
-    field :policy, Types::Profile, null: true
-    field :rules, [::Types::Rule], null: true, extras: [:lookahead] do
+    field :id, ID, null: false, cache: true
+    field :name, String, null: false, cache: true
+    field :description, String, null: true, cache: true
+    field :ref_id, String, null: false, cache: true
+    field :compliance_threshold, Float, null: false, cache: true
+    field :benchmark_id, ID, null: false, cache: true
+    field :account_id, ID, null: false, cache: true
+    field :policy, Types::Profile, null: true, cache: true
+    field :rules, [::Types::Rule], null: true, cache: true,
+                                   extras: [:lookahead] do
       argument :system_id, String,
                'System ID to filter by', required: false
       argument :identifier, String,
                'Rule identifier to filter by', required: false
       argument :references, [String],
                'Rule references to filter by', required: false
     end
-    field :hosts, [::Types::System], null: true
-    field :benchmark, ::Types::Benchmark, null: true
-    field :business_objective, ::Types::BusinessObjective, null: true
-    field :business_objective_id, ID, null: true
-    field :total_host_count, Int, null: false
-    field :external, Boolean, null: false
-
-    field :score, Float, null: false do
+    field :hosts, [::Types::System], null: true, cache: true
+    field :benchmark, ::Types::Benchmark, null: true, cache: true
+    field :business_objective, ::Types::BusinessObjective, null: true,
+                                                           cache: true
+    field :business_objective_id, ID, null: true, cache: true
+    field :total_host_count, Int, null: false, cache: true
+    field :external, Boolean, null: false, cache: true
+
+    field :score, Float, null: false, cache: true do
       argument :system_id, String,
                'Latest TestResult score for this system and profile',
                required: false
     end
 
-    field :compliant, Boolean, null: false do
+    field :compliant, Boolean, null: false, cache: true do
       argument :system_id, String, 'Is a system compliant with this profile?',
                required: false
     end
 
-    field :rules_passed, Int, null: false do
+    field :rules_passed, Int, null: false, cache: true do
       argument :system_id, String,
                'Rules passed for a system and a profile', required: false
     end
 
-    field :rules_failed, Int, null: false do
+    field :rules_failed, Int, null: false, cache: true do
       argument :system_id, String,
                'Rules failed for a system and a profile', required: false
     end
 
-    field :last_scanned, String, null: false do
+    field :last_scanned, String, null: false, cache: true do
       argument :system_id, String,
                'Last time this profile was scanned for a system',
                required: false
     end
 
-    field :compliant_host_count, Int, null: false
+    field :compliant_host_count, Int, null: false, cache: true
 
-    field :major_os_version, String, null: false
+    field :major_os_version, String, null: false, cache: true
 
     def compliant_host_count
       ::CollectionLoader.for(object.class, :hosts).load(object).then do |hosts|

app/graphql/types/query.rb

@@ -62,7 +62,7 @@ def business_objectives
     def latest_benchmarks
       Pundit.authorize(
         context[:current_user],
-        ::Xccdf::Benchmark.latest,
+        ::Xccdf::Benchmark.latest_supported,
         :index?, policy_class: BenchmarkPolicy
       )
     end

app/graphql/types/rule.rb

@@ -6,17 +6,17 @@ class Rule < Types::BaseObject
     graphql_name 'Rule'
     description 'A Rule registered in Insights Compliance'
 
-    field :id, ID, null: false
-    field :title, String, null: false
-    field :ref_id, String, null: false
-    field :rationale, String, null: true
-    field :description, String, null: true
-    field :severity, String, null: false
-    field :remediation_available, Boolean, null: false
-    field :profiles, [::Types::Profile], null: true
-    field :identifier, String, null: true
-    field :references, String, null: true
-    field :compliant, Boolean, null: false do
+    field :id, ID, null: false, cache: true
+    field :title, String, null: false, cache: true
+    field :ref_id, String, null: false, cache: true
+    field :rationale, String, null: true, cache: true
+    field :description, String, null: true, cache: true
+    field :severity, String, null: false, cache: true
+    field :remediation_available, Boolean, null: false, cache: true
+    field :profiles, [::Types::Profile], null: true, cache: true
+    field :identifier, String, null: true, cache: true
+    field :references, String, null: true, cache: true
+    field :compliant, Boolean, null: false, cache: true do
       argument :system_id, String, 'Is a system compliant?',
                required: false
       argument :profile_id, String, 'Is a system compliant with this profile?',
@@ -35,7 +35,7 @@ def references
       if context[:"rule_references_#{object.id}"].nil?
         ::CollectionLoader.for(::Rule, :rule_references)
                           .load(object).then do |references|
-          references.map { |ref| [ref.href, ref.label] }.to_json
+          references.map { |ref| { href: ref.href, label: ref.label } }.to_json
         end
       else
         references_from_context

app/graphql/types/system.rb

@@ -7,26 +7,26 @@ class System < Types::BaseObject
     graphql_name 'System'
     description 'A System registered in Insights Compliance'
 
-    field :id, ID, null: false
-    field :name, String, null: false
-    field :profiles, [::Types::Profile], null: true
-    field :compliant, Boolean, null: false do
+    field :id, ID, null: false, cache: true
+    field :name, String, null: false, cache: true
+    field :profiles, [::Types::Profile], null: true, cache: true
+    field :compliant, Boolean, null: false, cache: true do
       argument :profile_id, String, 'Filter results by profile ID',
                required: false
     end
-    field :profile_names, String, null: false
-    field :rules_passed, Int, null: false do
+    field :profile_names, String, null: false, cache: true
+    field :rules_passed, Int, null: false, cache: true do
       argument :profile_id, String, 'Filter results by profile ID',
                required: false
     end
-    field :rules_failed, Int, null: false do
+    field :rules_failed, Int, null: false, cache: true do
       argument :profile_id, String, 'Filter results by profile ID',
                required: false
     end
-    field :rule_objects_failed, [::Types::Rule], null: true do
+    field :rule_objects_failed, [::Types::Rule], null: true, cache: true do
       description 'Rules failed by a system'
     end
-    field :last_scanned, String, null: true do
+    field :last_scanned, String, null: true, cache: true do
       argument :profile_id, String, 'Filter results by profile ID',
                required: false
     end

app/graphql/types/test_result.rb

@@ -7,11 +7,11 @@ class TestResult < Types::BaseObject
     graphql_name 'TestResult'
     description 'A TestResult as recorded in Insights Compliance'
 
-    field :id, ID, null: false
-    field :start_time, String, null: true
-    field :end_time, String, null: false
-    field :score, Float, null: false
-    field :profile, ::Types::Profile, null: false
-    field :host, ::Types::System, null: false
+    field :id, ID, null: false, cache: true
+    field :start_time, String, null: true, cache: true
+    field :end_time, String, null: false, cache: true
+    field :score, Float, null: false, cache: true
+    field :profile, ::Types::Profile, null: false, cache: true
+    field :host, ::Types::System, null: false, cache: true
   end
 end

app/models/xccdf/benchmark.rb

@@ -6,11 +6,22 @@ module Xccdf
   # for a given set of software in a specific release of the SCAP Security
   # Guide (i.e. RHEL 7, v0.1.43)
   class Benchmark < ApplicationRecord
+    scoped_search on: %i[id ref_id title version]
+    scoped_search relation: :profiles, on: :id, rename: :profile_ids,
+                  alias: :profile_id
+    scoped_search relation: :rules, on: :id, rename: :rule_ids, alias: :rule_id
+
     has_many :profiles, dependent: :destroy
     has_many :rules, dependent: :destroy
     validates :ref_id, uniqueness: { scope: %i[version] }, presence: true
     validates :version, presence: true
 
+    LATEST_SUPPORTED_VERSIONS = {
+      'xccdf_org.ssgproject.content_benchmark_RHEL-6': '0.1.28',
+      'xccdf_org.ssgproject.content_benchmark_RHEL-7': '0.1.46',
+      'xccdf_org.ssgproject.content_benchmark_RHEL-8': '0.1.46'
+    }.freeze
+
     class << self
       def from_openscap_parser(op_benchmark)
         benchmark = find_or_initialize_by(
@@ -43,6 +54,16 @@ def find_latest(benchmarks)
           Gem::Version.new(benchmark.version)
         end
       end
+
+      def latest_supported
+        Rails.cache.fetch('latest_supported_benchmarks') do
+          LATEST_SUPPORTED_VERSIONS.inject(
+            where('1=0')
+          ) do |acc, (ref_id, version)|
+            acc.or(where(ref_id: ref_id, version: version))
+          end.compact
+        end
+      end
     end
 
     def inferred_os_major_version

app/serializers/benchmark_serializer.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+# JSON API serialization for an OpenSCAP Benchmark
+class BenchmarkSerializer
+  include FastJsonapi::ObjectSerializer
+  attributes :ref_id, :title, :version, :description
+end

config/environments/production.rb

@@ -50,7 +50,7 @@
   config.log_tags = [:request_id]
 
   # Use a different cache store in production.
-  config.cache_store = :redis_cache_store, { url: "redis://#{Settings.redis_url}/0" }
+  config.cache_store = :redis_cache_store, { url: "redis://#{Settings.redis_cache_url}" }
 
   # Use a real queuing backend for Active Job (and separate queues per environment)
   # config.active_job.queue_adapter     = :resque