Split workers & use workflows
parent
d4be6af200
commit
54a7edd061
|
@ -0,0 +1,13 @@
|
|||
source 'https://rubygems.org'
|
||||
|
||||
gem 'dotenv'
|
||||
|
||||
gem 'sidekiq'
|
||||
gem 'sidekiq-workflow', git: 'https://git.imirhil.fr/aeris/sidekiq-workflow.git', branch: :master
|
||||
|
||||
gem 'simpleidn'
|
||||
gem 'cryptcheck', '~> 2.0.0', path: '../engine'
|
||||
|
||||
group :development do
|
||||
gem 'amazing_print'
|
||||
end
|
|
@ -0,0 +1,85 @@
|
|||
GIT
|
||||
remote: https://git.imirhil.fr/aeris/sidekiq-workflow.git
|
||||
revision: 9600075c17e83f2013f0b7d6e35698bea6b0ee40
|
||||
branch: master
|
||||
specs:
|
||||
sidekiq-workflow (0.0.0)
|
||||
colorize (~> 0.8)
|
||||
redis (~> 4.3)
|
||||
redlock (~> 1.2)
|
||||
sidekiq
|
||||
terminal-table (~> 3.0)
|
||||
|
||||
PATH
|
||||
remote: ../engine
|
||||
specs:
|
||||
cryptcheck (2.0.0)
|
||||
amazing_print
|
||||
colorize
|
||||
httparty
|
||||
nokogiri
|
||||
parallel
|
||||
ruby-progressbar
|
||||
thor
|
||||
|
||||
GEM
|
||||
remote: https://rubygems.org/
|
||||
specs:
|
||||
amazing_print (1.2.1)
|
||||
colorize (0.8.1)
|
||||
connection_pool (2.2.5)
|
||||
dotenv (2.7.6)
|
||||
foreman (0.87.2)
|
||||
http_accept_language (2.1.1)
|
||||
httparty (0.20.0)
|
||||
mime-types (~> 3.0)
|
||||
multi_xml (>= 0.5.2)
|
||||
mime-types (3.4.1)
|
||||
mime-types-data (~> 3.2015)
|
||||
mime-types-data (3.2022.0105)
|
||||
mini_portile2 (2.4.0)
|
||||
multi_xml (0.6.0)
|
||||
nokogiri (1.10.10)
|
||||
mini_portile2 (~> 2.4.0)
|
||||
parallel (1.19.2)
|
||||
pg (1.2.3)
|
||||
rack (2.0.9)
|
||||
rack-protection (2.2.0)
|
||||
rack
|
||||
recursive-open-struct (1.1.3)
|
||||
redis (4.4.0)
|
||||
redlock (1.2.2)
|
||||
redis (>= 3.0.0, < 5.0)
|
||||
ruby-progressbar (1.11.0)
|
||||
sidekiq (5.2.8)
|
||||
connection_pool (~> 2.2, >= 2.2.2)
|
||||
rack (< 2.1.0)
|
||||
rack-protection (>= 1.5.0)
|
||||
redis (>= 3.3.5, < 5)
|
||||
simpleidn (0.2.1)
|
||||
unf (~> 0.1.4)
|
||||
terminal-table (3.0.2)
|
||||
unicode-display_width (>= 1.1.1, < 3)
|
||||
thor (1.2.1)
|
||||
unf (0.1.4)
|
||||
unf_ext
|
||||
unf_ext (0.0.8.1)
|
||||
unicode-display_width (2.1.0)
|
||||
|
||||
PLATFORMS
|
||||
x86_64-linux
|
||||
|
||||
DEPENDENCIES
|
||||
amazing_print
|
||||
cryptcheck (~> 2.0.0)!
|
||||
dotenv
|
||||
foreman
|
||||
http_accept_language
|
||||
pg
|
||||
recursive-open-struct
|
||||
sidekiq
|
||||
sidekiq-workflow!
|
||||
simpleidn
|
||||
|
||||
BUNDLED WITH
|
||||
2.3.9
|
|
@ -0,0 +1,4 @@
|
|||
web: bundle exec guard -i
|
||||
webpack: bundle exec webpack-dev-server
|
||||
sidekiq: bundle exec sidekiq -q default
|
||||
sidekiq_1_0: BUNDLE_GEMFILE=Gemfile-2.3 bin/sidekiq 1.0 -q tls_1_0
|
|
@ -1,59 +1,66 @@
|
|||
class CheckController < ApplicationController
|
||||
before_action :check_host, except: %i(index)
|
||||
helper_method :tls_type, :type
|
||||
before_action :check_host, except: %i[index]
|
||||
before_action :exclude_robots_indexing
|
||||
helper_method :tls_type, :type
|
||||
|
||||
def show
|
||||
enqueue_host unless @analysis
|
||||
@host = SimpleIDN.to_unicode @host
|
||||
respond_to do |format|
|
||||
format.html do
|
||||
return render :processing if @analysis.pending
|
||||
@result = @analysis.result.collect { |r| RecursiveOpenStruct.new r, recurse_over_arrays: true }
|
||||
end
|
||||
format.json { render json: @analysis }
|
||||
end
|
||||
end
|
||||
def show
|
||||
enqueue_host unless @analysis
|
||||
@host = SimpleIDN.to_unicode @host
|
||||
respond_to do |format|
|
||||
format.html do
|
||||
return render :processing if @analysis.pending
|
||||
@result = @analysis.result.collect { |r| RecursiveOpenStruct.new r, recurse_over_arrays: true }
|
||||
end
|
||||
format.json { render json: @analysis }
|
||||
end
|
||||
end
|
||||
|
||||
def refresh
|
||||
unless @analysis.pending
|
||||
if Rails.env == 'production'
|
||||
refresh_allowed = @analysis.updated_at + Rails.configuration.refresh_delay
|
||||
if Time.now < refresh_allowed
|
||||
flash[:warning] = "Merci d’attendre au moins #{l refresh_allowed} pour rafraîchir"
|
||||
return redirect_to action: :show, id: @host
|
||||
end
|
||||
end
|
||||
enqueue_host
|
||||
end
|
||||
redirect_to action: :show
|
||||
end
|
||||
def refresh
|
||||
unless @analysis.pending
|
||||
if Rails.env == 'production'
|
||||
refresh_allowed = @analysis.updated_at + Rails.configuration.refresh_delay
|
||||
if Time.now < refresh_allowed
|
||||
flash[:warning] = "Merci d’attendre au moins #{l refresh_allowed} pour rafraîchir"
|
||||
return redirect_to action: :show, id: @host
|
||||
end
|
||||
end
|
||||
enqueue_host
|
||||
end
|
||||
redirect_to action: :show
|
||||
end
|
||||
|
||||
protected
|
||||
def default_args
|
||||
end
|
||||
protected
|
||||
|
||||
def enqueue_host
|
||||
@analysis = Analysis.pending! self.type, @host, @args
|
||||
self.worker.perform_async @analysis.host, *@analysis.args
|
||||
end
|
||||
def exclude_robots_indexing
|
||||
response.set_header 'X-Robots-Tag',
|
||||
%i[none noarchive nosnippet notranslate noimageindex].join(',')
|
||||
end
|
||||
|
||||
def check_host
|
||||
@id = params[:id]
|
||||
def default_args
|
||||
end
|
||||
|
||||
if @id.end_with? '.json'
|
||||
@id = @id.sub /\.json$/, ''
|
||||
request.format = :json
|
||||
end
|
||||
def enqueue_host
|
||||
@analysis = Analysis.pending! self.type, @host, @args
|
||||
CheckWorkflow.start! self.type, @analysis.host, *@analysis.args
|
||||
end
|
||||
|
||||
@host, @args = @id.split ':'
|
||||
@host = SimpleIDN.to_ascii @host.downcase
|
||||
if /[^a-zA-Z0-9.-]/ =~ @host
|
||||
flash[:danger] = "Hôte #{@host} invalide"
|
||||
redirect_to action: :index
|
||||
return false
|
||||
end
|
||||
@args ||= default_args
|
||||
def check_host
|
||||
@id = params[:id]
|
||||
|
||||
@analysis = Analysis[self.type, @host, @args]
|
||||
end
|
||||
if @id.end_with? '.json'
|
||||
@id = @id.sub /\.json$/, ''
|
||||
request.format = :json
|
||||
end
|
||||
|
||||
@host, @args = @id.split ':'
|
||||
@host = SimpleIDN.to_ascii @host.downcase
|
||||
if /[^a-zA-Z0-9.-]/ =~ @host
|
||||
flash[:danger] = "Hôte #{@host} invalide"
|
||||
redirect_to action: :index
|
||||
return false
|
||||
end
|
||||
@args ||= default_args
|
||||
|
||||
@analysis = Analysis[self.type, @host, @args]
|
||||
end
|
||||
end
|
||||
|
|
|
@ -4,10 +4,6 @@ class HttpsController < CheckController
|
|||
:https
|
||||
end
|
||||
|
||||
def worker
|
||||
HTTPSWorker
|
||||
end
|
||||
|
||||
def tls_type
|
||||
'HTTPS'
|
||||
end
|
||||
|
|
|
@ -4,10 +4,6 @@ class SmtpController < CheckController
|
|||
:smtp
|
||||
end
|
||||
|
||||
def worker
|
||||
SMTPWorker
|
||||
end
|
||||
|
||||
def tls_type
|
||||
'STARTTLS'
|
||||
end
|
||||
|
|
|
@ -4,10 +4,6 @@ class SshController < CheckController
|
|||
:ssh
|
||||
end
|
||||
|
||||
def worker
|
||||
SSHWorker
|
||||
end
|
||||
|
||||
def tls_type
|
||||
'SSH'
|
||||
end
|
||||
|
|
|
@ -4,10 +4,6 @@ class TlsController < CheckController
|
|||
:tls
|
||||
end
|
||||
|
||||
def worker
|
||||
TLSWorker
|
||||
end
|
||||
|
||||
def tls_type
|
||||
'TLS'
|
||||
end
|
||||
|
|
|
@ -5,10 +5,6 @@ class XmppController < CheckController
|
|||
:xmpp
|
||||
end
|
||||
|
||||
def worker
|
||||
XMPPWorker
|
||||
end
|
||||
|
||||
def tls_type
|
||||
'STARTTLS'
|
||||
end
|
||||
|
|
|
@ -0,0 +1,30 @@
|
|||
require 'simpleidn'
|
||||
|
||||
class CheckJob
|
||||
include Sidekiq::Workflow::Worker
|
||||
sidekiq_options queue: :tls_1_0, retry: false
|
||||
|
||||
def perform(type, host, *args, **_)
|
||||
host = SimpleIDN.to_ascii host.downcase
|
||||
result = self.analyze type, host, *args
|
||||
args = nil if args.empty?
|
||||
set_payload({ type: type, host: host, args: args, result: result })
|
||||
end
|
||||
|
||||
def analyze(type, *args)
|
||||
case type.to_sym
|
||||
when :https
|
||||
CryptCheck::Tls::Https.analyze *args
|
||||
when :smtp
|
||||
CryptCheck::Tls::Smtp.analyze *args
|
||||
when :xmpp
|
||||
CryptCheck::Tls::Xmpp.analyze *args
|
||||
when :tls
|
||||
CryptCheck::Tls.analyze *args
|
||||
when :ssh
|
||||
CryptCheck::Ssh.analyze *args
|
||||
else
|
||||
raise "Unsupported analysis #{type}"
|
||||
end
|
||||
end
|
||||
end
|
|
@ -0,0 +1,13 @@
|
|||
class PersistJob
|
||||
include Sidekiq::Workflow::Worker
|
||||
sidekiq_options retry: false
|
||||
|
||||
def perform(*_, **_)
|
||||
result = self.get_payload CheckJob, :json
|
||||
type = result.fetch 'type'
|
||||
host = result.fetch 'host'
|
||||
args = result.fetch 'args'
|
||||
result = result.fetch 'result'
|
||||
Analysis.post! type, host, args, result
|
||||
end
|
||||
end
|
|
@ -1,11 +0,0 @@
|
|||
class CheckWorker
|
||||
include Sidekiq::Worker
|
||||
sidekiq_options retry: false
|
||||
|
||||
def perform(host, *args)
|
||||
host = SimpleIDN.to_ascii host.downcase
|
||||
result = self.analyze host, *args
|
||||
args = nil if args.empty?
|
||||
Analysis.post! self.type, host, args, result
|
||||
end
|
||||
end
|
|
@ -1,12 +0,0 @@
|
|||
class HTTPSWorker < CheckWorker
|
||||
sidekiq_options retry: false
|
||||
|
||||
protected
|
||||
def analyze(host, port)
|
||||
CryptCheck::Tls::Https.analyze host, port
|
||||
end
|
||||
|
||||
def type
|
||||
:https
|
||||
end
|
||||
end
|
|
@ -1,12 +0,0 @@
|
|||
class SMTPWorker < CheckWorker
|
||||
sidekiq_options retry: false
|
||||
|
||||
protected
|
||||
def analyze(host)
|
||||
CryptCheck::Tls::Smtp.analyze host
|
||||
end
|
||||
|
||||
def type
|
||||
:smtp
|
||||
end
|
||||
end
|
|
@ -1,26 +0,0 @@
|
|||
class SSHWorker < CheckWorker
|
||||
sidekiq_options retry: false
|
||||
|
||||
protected
|
||||
def analyze(host, port=22)
|
||||
CryptCheck::Ssh.analyze host, port
|
||||
end
|
||||
|
||||
def type
|
||||
:ssh
|
||||
end
|
||||
|
||||
def to_json(server)
|
||||
{
|
||||
kex: server.kex,
|
||||
encryption: server.encryption,
|
||||
hmac: server.hmac,
|
||||
compression: server.compression,
|
||||
key_: server.key
|
||||
}
|
||||
end
|
||||
|
||||
def grade_to_json(grade)
|
||||
nil
|
||||
end
|
||||
end
|
|
@ -1,12 +0,0 @@
|
|||
class TLSWorker < CheckWorker
|
||||
sidekiq_options retry: false
|
||||
|
||||
protected
|
||||
def analyze(host, port)
|
||||
CryptCheck::Tls.analyze host, port
|
||||
end
|
||||
|
||||
def type
|
||||
:tls
|
||||
end
|
||||
end
|
|
@ -1,12 +0,0 @@
|
|||
class XMPPWorker < CheckWorker
|
||||
sidekiq_options retry: false
|
||||
|
||||
protected
|
||||
def analyze(host, type)
|
||||
CryptCheck::Tls::Xmpp.analyze host, type
|
||||
end
|
||||
|
||||
def type
|
||||
:xmpp
|
||||
end
|
||||
end
|
|
@ -0,0 +1,6 @@
|
|||
class CheckWorkflow < Sidekiq::Workflow
|
||||
def configure(*args, **kwargs)
|
||||
check = job CheckJob, *args, **kwargs
|
||||
job PersistJob, after: check
|
||||
end
|
||||
end
|
28
bin/sidekiq
28
bin/sidekiq
|
@ -1,20 +1,8 @@
|
|||
#!/usr/bin/env ruby
|
||||
$:.unshift File.expand_path File.join File.dirname(__FILE__), '../../cryptcheck/lib'
|
||||
require 'rubygems'
|
||||
require 'bundler/setup'
|
||||
|
||||
$TESTING = false
|
||||
$CELLULOID_DEBUG = false
|
||||
|
||||
require 'sidekiq/cli'
|
||||
|
||||
begin
|
||||
cli = Sidekiq::CLI.instance
|
||||
cli.parse
|
||||
cli.run
|
||||
rescue => e
|
||||
raise e if $DEBUG
|
||||
STDERR.puts e.message
|
||||
STDERR.puts e.backtrace.join("\n")
|
||||
exit 1
|
||||
end
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
DIR="$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")"
|
||||
RAILS_DIR="$(readlink -f "$DIR/..")"
|
||||
ENGINE_DIR="$(readlink -f "$RAILS_DIR/../engine")"
|
||||
eval "$("$ENGINE_DIR/bin/load-rbenv" "$1")"
|
||||
shift
|
||||
bundle exec sidekiq -r "$RAILS_DIR/config/sidekiq.rb" $*
|
||||
|
|
|
@ -0,0 +1,3 @@
|
|||
if redis_url = ENV['REDIS_URL']
|
||||
Sidekiq::Workflow.configure url: redis_url
|
||||
end
|
|
@ -0,0 +1,29 @@
|
|||
#!/usr/bin/env ruby
|
||||
require 'rubygems'
|
||||
require 'bundler/setup'
|
||||
env = ENV.fetch 'RAILS_ENV', 'development'
|
||||
Bundler.require env
|
||||
|
||||
require 'dotenv'
|
||||
Dotenv.load ".env.#{ENV['RAILS_ENV']}.local", ".env.#{ENV['RAILS_ENV']}", '.env.local', '.env'
|
||||
|
||||
class Hash
|
||||
def compact
|
||||
select { |_, value| !value.nil? }
|
||||
end
|
||||
end
|
||||
|
||||
require 'redis'
|
||||
Redis.exists_returns_integer = true
|
||||
redis_url = ENV.fetch 'REDIS_URL'
|
||||
|
||||
require 'sidekiq'
|
||||
Sidekiq.configure_server { |c| c.redis = { url: redis_url } }
|
||||
Sidekiq.configure_client { |c| c.redis = { url: redis_url } }
|
||||
require 'sidekiq/workflow'
|
||||
redis_url = ENV['REDIS_URL']
|
||||
Sidekiq::Workflow.configure url: redis_url
|
||||
|
||||
require 'cryptcheck'
|
||||
load File.join __dir__, '../app/jobs/check_job.rb'
|
||||
load File.join __dir__, '../app/jobs/persist_job.rb'
|
Loading…
Reference in New Issue