aeris
/
alice
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Remove useless checks

Browse Source
webui
aeris 5 years ago
parent 65452a7895
commit 8c7df91bb5
14 changed files with 188 additions and 289 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 8
      app/lib/http.rb
  2. 29
      app/models/check.rb
  3. 7
      app/models/group.rb
  4. 74
      app/models/site.rb
  5. 22
      app/models/target.rb
  6. 105
      bin/cli.rb
  7. 2
      bin/import.rb
  8. 1
      config/boot.rb
  9. 11
      config/database.yml
  10. 2
      db/migrate/20180510000002_create_sites.rb
  11. 15
      db/migrate/20180510000004_create_checks.rb
  12. 18
      db/schema.rb
  13. 14
      spec/helpers/sites_helper_spec.rb
  14. 169
      spec/models/site_spec.rb

8
app/lib/http.rb
Unescape View File

@ -59,12 +59,16 @@ class Http
DATE_FORMAT = '%Y%m%d_%H%M%S'.freeze
def self.prefix(url)
Digest::SHA256.hexdigest url
end
def cache(response)
return unless ENV['DEBUG_HTTP']
prefix = Digest::SHA256.hexdigest @url
prefix = self.class.prefix @url
dir = File.join Rails.root, 'tmp', 'cache', 'http'
FileUtils.mkdir_p dirs unless Dir.exist? dir
FileUtils.mkdir_p dir unless Dir.exist? dir
body = response.body
last = Dir[File.join dir, "#{prefix}_*"].sort.last

29
app/models/check.rb
Unescape View File

@ -1,29 +0,0 @@
class Check < ApplicationRecord
belongs_to :site
belongs_to :target
def to_s
self.target.to_s
end
def changed?(reference, content, debug: false)
target = self.target
reference = target.extract reference
content = target.extract content
changed = reference != content
if changed
puts Utils.diff reference, content if debug
return true
end
false
end
def diff(reference, content, context: 3, **kwargs)
target = self.target
reference = target.extract reference
content = target.extract content
Diffy::Diff.new reference, content, context: context, **kwargs
end
end

7
app/models/group.rb
Unescape View File

@ -8,4 +8,11 @@ class Group < ApplicationRecord
def self.[](name)
self.where(name: name).first
end
def all_targets
targets = self.targets
template = self.template
targets += template.targets if template
targets
end
end

74
app/models/site.rb
Unescape View File

@ -2,7 +2,6 @@ class Site < ApplicationRecord
belongs_to :group, optional: true
belongs_to :template, optional: true
has_many :targets
has_many :checks
validates :url, presence: true
@ -14,47 +13,24 @@ class Site < ApplicationRecord
Http.new self.url
end
def inherited_targets
def all_targets
targets = self.targets
group = self.group
targets += group.targets if group
template = self.template
targets = template.targets if template
targets += template.all_targets if template
group = self.group
targets += group.all_targets if group
targets
end
def create_checks!
self.inherited_targets.each do |target|
self.checks.create! target: target
end
end
def reset!
self.update! reference: nil, content: nil, checked_at: nil, changed_at: nil, last_error: nil
end
def reference!(content)
self.update! reference: content, content: content, checked_at: Time.now, changed_at: nil, last_error: nil
end
def read!
return unless self.content
self.reference! self.content
end
def clear!
self.update! content: nil, checked_at: Time.now, changed_at: nil, last_error: nil
end
def diff(context: 3, **kwargs)
reference = self.reference
content = self.content
Diffy::Diff.new reference, content, context: context, **kwargs
end
def changed?(reference, content, debug: false)
checks = self.checks
if checks.empty?
def content_changed?(reference, content, debug: false)
targets = self.all_targets
if targets.empty?
if reference != content
puts Utils.diff reference, content if debug
return true
@ -62,27 +38,28 @@ class Site < ApplicationRecord
return false
end
checks.each do |check|
changed = check.changed? reference, content, debug: debug
targets.each do |target|
changed = target.content_changed? reference, content, debug: debug
return true if changed
end
false
end
def diff!(reference, content, debug: false)
self.checked_at = Time.now
def diff!(reference, content, date: Time.now, debug: false)
self.checked_at = date
state = :unchanged
begin
changed = self.changed? reference, content, debug: debug
changed = self.content_changed? reference, content, debug: debug
if changed
self.content = content
self.reference = content
self.changed_at = self.checked_at
state = :changed
end
self.last_error = nil
rescue => e
raise
$stderr.puts e
self.last_error = e
state = :error
@ -98,29 +75,10 @@ class Site < ApplicationRecord
self.update! name: grab.title unless self.name
unless self.reference
self.reference! content
self.update! reference: content
return :reference
else
return self.diff! self.content, content, debug: debug
end
end
def recalculate!(debug: false)
reference = self.reference
content = self.content || reference
changed_at = self.changed_at
state = :unchanged
changed = self.checks.find { |c| c.changed? reference, content, debug: debug }
if changed
state = :changed
changed_at ||= self.checked_at
else
changed_at = nil
return self.diff! self.reference, content, debug: debug
end
self.update! reference: reference, content: content, changed_at: changed_at
state
end
end

22
app/models/target.rb
Unescape View File

@ -5,8 +5,9 @@ class Target < ApplicationRecord
has_many :checks
def to_s
return self.name if self.name
s = []
s << self.name if self.name
s << "from: #{self.from}" if self.from
s << "to: #{self.to}" if self.to
s << "css: #{self.css}" if self.css
@ -56,4 +57,23 @@ class Target < ApplicationRecord
content = self.extract_css content
content
end
def content_changed?(reference, content, debug: false)
reference = self.extract reference
content = self.extract content
changed = reference != content
if changed
puts Utils.diff reference, content if debug
return true
end
false
end
def diff(reference, content, context: 3, **kwargs)
reference = self.extract reference
content = self.extract content
Diffy::Diff.new reference, content, context: context, **kwargs
end
end

105
bin/cli.rb
Unescape View File

@ -4,7 +4,6 @@ require 'optparse'
# Force resolution to avoid cycle in autoloading
Http
Check
Target
Site
Group
@ -30,10 +29,6 @@ def display(item)
end
class App < Thor
desc 'check <url>*', 'Check given sites for changes'
method_option :reset, type: :boolean, default: false, aliases: '-r', desc: 'Reset sites before check'
method_option :debug, type: :boolean, default: false, aliases: '-d', desc: 'Activate debug'
COLORS = {
reference: :blue,
unchanged: :green,
@ -41,14 +36,16 @@ class App < Thor
error: { background: :red }
}.freeze
desc 'check <url>*', 'Check given sites for changes'
method_option :debug, type: :boolean, default: false, aliases: '-d', desc: 'Activate debug'
def check(urls = nil)
reset = options[:reset]
debug = options[:debug]
results = Hash.new 0
self.process urls do |site|
site.reset! if reset
result = site.check! debug: debug
results[result] += 1
color = COLORS[result]
@ -67,68 +64,45 @@ class App < Thor
self.process urls, &:read!
end
desc 'clear <url>*', 'Clear given sites'
def clear(urls = nil)
self.process urls, &:clear!
end
desc 'diff <url>*', 'Display diff of the given sites'
def diff(urls = nil)
sites = self.sites urls
sites.each do |site|
next unless site.changed_at
puts "#{site.url.colorize :yellow}"
checks = site.checks
display site if checks.empty?
checks.each do |check|
next unless check.changed_at
puts " #{check.target}"
display check
end
end
end
desc 'recalculate <url>*', 'Recalculate state of given sites'
method_option :debug, type: :boolean, default: false, aliases: '-d', desc: 'Activate debug'
def recalculate(urls = nil)
debug = options[:debug]
results = Hash.new 0
desc 'redo <url>*', 'Redo diff from cache'
def redo(urls = nil)
cache = 'tmp/cache/http'
self.process urls do |site|
result = site.recalculate! debug: debug
color = COLORS[result]
results[result] += 1
result.to_s.colorize color
end
results.each do |k, v|
color = COLORS[k]
puts "#{k.to_s.colorize color}: #{v}"
site._changes.delete_all
reference = nil
fp = Http.prefix site.url
Dir["#{cache}/#{fp}_*"].sort.each do |file|
name = File.basename file
date = name.split('_', 2).last
date = DateTime.strptime date, Http::DATE_FORMAT
content = File.read file
unless reference
site.update! reference: content
else
status = site.diff! reference, content, date: date
ap site: site.url, date: date, status: status
end
reference = content
end
nil
end
end
desc 'reset <url>*', 'Reset state of given sites'
def reset(urls = nil)
self.process urls, &:reset!
end
desc 'redo <url> <date1> <date2>', 'Redo check from cache'
def redo(url, date1 = nil, date2 = nil)
site = Site.where(url: url).first
fp = Digest::SHA256.hexdigest url
dir = File.join Rails.root, 'tmp/cache/http'
reference = File.join dir, "#{fp}_#{date1}"
reference = File.read reference
content = File.join dir, "#{fp}_#{date2}"
content = File.read content
ap site.changed? reference, content, debug: true
end
# desc 'redo <url> <date1> <date2>', 'Redo check from cache'
#
# def redo(url, date1 = nil, date2 = nil)
# site = Site.where(url: url).first
# fp = Digest::SHA256.hexdigest url
# dir = File.join Rails.root, 'tmp/cache/http'
# reference = File.join dir, "#{fp}_#{date1}"
# reference = File.read reference
# content = File.join dir, "#{fp}_#{date2}"
# content = File.read content
#
# ap site.changed? reference, content, debug: true
# end
protected
@ -139,7 +113,7 @@ class App < Thor
def process(urls)
sites = self.sites urls
Parallel.each sites, in_threads: 16 do |site|
Parallel.each sites, in_threads: 1 do |site|
ActiveRecord::Base.transaction do
url = site.url.colorize :yellow
begin
@ -148,6 +122,7 @@ class App < Thor
result
rescue => e
puts "#{url} #{e.to_s.colorize :red}"
raise
nil
end
end

2
bin/import.rb
Unescape View File

@ -81,7 +81,6 @@ def import_sites(sites, group = nil, skip_title: true)
site = Site.create! url: url, name: name, group: group
create_targets site, params
site.create_checks!
rescue => e
$stderr.puts "Unable to import site #{url.colorize :yellow}: #{e.to_s.colorize :red}"
raise
@ -90,7 +89,6 @@ def import_sites(sites, group = nil, skip_title: true)
end
ActiveRecord::Base.transaction do
Check.destroy_all
Target.destroy_all
Site.destroy_all
Group.destroy_all

1
config/boot.rb
Unescape View File

@ -1,3 +1,4 @@
ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
require 'bundler/setup' # Set up gems listed in the Gemfile.
require 'bootsnap/setup'

11
config/database.yml
Unescape View File

@ -5,21 +5,22 @@
# gem 'sqlite3'
#
default: &default
adapter: sqlite3
adapter: postgresql
host: localhost
user: postgres
password: postgres
database: webmon_<%= Rails.env %>
encoding: unicode
pool: 20
timeout: 5000
development:
<<: *default
database: db/development.sqlite3
# Warning: The database defined as "test" will be erased and
# re-generated from your development database when you run "rake".
# Do not set this db to the same as development or production.
test:
<<: *default
database: db/test.sqlite3
production:
<<: *default
database: db/production.sqlite3

2
db/migrate/20180510000002_create_sites.rb
Unescape View File

@ -5,14 +5,12 @@ class CreateSites < ActiveRecord::Migration[5.1]
t.string :name, index: true
t.text :reference
t.text :content
t.belongs_to :group, index: true, foreign_key: true
t.belongs_to :template, index: true, foreign_key: true
t.string :last_error
t.datetime :checked_at
t.datetime :changed_at
end
end
end

15
db/migrate/20180510000004_create_checks.rb
Unescape View File

@ -1,15 +0,0 @@
class CreateChecks < ActiveRecord::Migration[5.1]
def change
create_table :checks do |t|
t.text :reference
t.text :content
t.belongs_to :target, index: true, foreign_key: true
t.belongs_to :site, index: true, foreign_key: true
t.string :last_error
t.datetime :checked_at
t.datetime :changed_at
end
end
end

18
db/schema.rb
Unescape View File

@ -10,23 +10,11 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20180510000004) do
ActiveRecord::Schema.define(version: 2018_05_10_000003) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
create_table "checks", force: :cascade do |t|
t.text "reference"
t.text "content"
t.bigint "target_id"
t.bigint "site_id"
t.string "last_error"
t.datetime "checked_at"
t.datetime "changed_at"
t.index ["site_id"], name: "index_checks_on_site_id"
t.index ["target_id"], name: "index_checks_on_target_id"
end
create_table "groups", force: :cascade do |t|
t.string "name", null: false
t.bigint "template_id"
@ -38,12 +26,10 @@ ActiveRecord::Schema.define(version: 20180510000004) do
t.string "url", null: false
t.string "name"
t.text "reference"
t.text "content"
t.bigint "group_id"
t.bigint "template_id"
t.string "last_error"
t.datetime "checked_at"
t.datetime "changed_at"
t.index ["group_id"], name: "index_sites_on_group_id"
t.index ["name"], name: "index_sites_on_name"
t.index ["template_id"], name: "index_sites_on_template_id"
@ -67,8 +53,6 @@ ActiveRecord::Schema.define(version: 20180510000004) do
t.index ["name"], name: "index_templates_on_name", unique: true
end
add_foreign_key "checks", "sites"
add_foreign_key "checks", "targets"
add_foreign_key "groups", "templates"
add_foreign_key "sites", "groups"
add_foreign_key "sites", "templates"

14
spec/helpers/sites_helper_spec.rb
Unescape View File

@ -0,0 +1,14 @@
require 'rails_helper'
# Specs in this file have access to a helper object that includes
# the SitesHelper. For example:
#
# describe SitesHelper do
# describe "string concat" do
# it "concats two strings with spaces" do
# expect(helper.concat_strings("this","that")).to eq("this that")
# end
# end
# end
RSpec.describe SitesHelper, type: :helper do
end

169
spec/models/site_spec.rb
Unescape View File

@ -1,117 +1,100 @@
RSpec.describe Site, type: :model do
REFERENCE_TARGET = '<div id="content">bar</div>'
REFERENCE = "<html><body>foo #{REFERENCE_TARGET}</body></html>"
CHANGE_OUTSIDE_TARGET = '<html><body>baz <div id="content">bar</div></body></html>'
CHANGE_TARGET = '<div id="content">baz</div>'
CHANGE_INSIDE_TARGET = "<html><body>foo #{CHANGE_TARGET}</body></html>"
let :site do
Site.create! url: 'http://localhost/'
end
let :check do
site.checks.first
end
def add_check(**args)
target = site.targets.create! args
site.checks.create! target: target
end
def stub_page(content)
allow(site).to receive(:grab) { OpenStruct.new body: content }
end
def check!(content)
stub_page content
site.check!
end
def reference_and_check!(content)
site.reference! REFERENCE
self.check! content
end
it 'must not change if no change with no check' do
status = reference_and_check! REFERENCE
expect(status).to be :unchanged
expect(site.changed_at).to be_nil
expect(site.content).to eq REFERENCE
end
it 'must not change if no change with checks' do
check = add_check css: '#content'
context '#check!' do
REFERENCE_TARGET = '<div id="content">bar</div>'
REFERENCE = "<html><body>foo #{REFERENCE_TARGET}</body></html>"
CHANGE_OUTSIDE_TARGET = '<html><body>baz <div id="content">bar</div></body></html>'
CHANGE_TARGET = '<div id="content">baz</div>'
CHANGE_INSIDE_TARGET = "<html><body>foo #{CHANGE_TARGET}</body></html>"
let :site do
site = Site.create! url: 'http://localhost/'
site.reference = REFERENCE
site
end
status = reference_and_check! REFERENCE
expect(status).to be :unchanged
let :target do
site.targets.first
end
expect(site.changed_at).to be_nil
expect(site.content).to eq REFERENCE
def expect_changed(content, reference = REFERENCE)
status = self.check! content
expect(status).to be :changed
expect(site.reference).to eq content
# expect(check.changed_at).to be_nil
# expect(check.content).to eq REFERENCE_TARGET
end
expect(site.changed_at).not_to be_nil
it 'must change if change with no check' do
status = reference_and_check! CHANGE_OUTSIDE_TARGET
expect(status).to be :changed
# changes = site._changes
# expect(changes.size).to be 1
# change = changes.first
#
# expect(change.from).to eq reference
# expect(change.to).to eq content
end
expect(site.changed_at).not_to be_nil
expect(site.content).to eq CHANGE_OUTSIDE_TARGET
end
def expect_unchanged(content, changed_at = nil)
status = self.check! content
expect(status).to be :unchanged
it 'must not change if change but no check changed' do
check = add_check css: '#content'
status = reference_and_check! CHANGE_OUTSIDE_TARGET
expect(status).to be :unchanged
# expect(site.changed_at).to eq changed_at
#
# changes = site._changes
# expect(changes.size).to be 0
end
expect(site.changed_at).to be_nil
expect(site.content).to be REFERENCE
def add_target(**args)
site.targets.create! args
end
# expect(check.changed_at).to be_nil
# expect(check.content).to eq REFERENCE_TARGET
end
def stub_page(content)
allow(site).to receive(:grab) { OpenStruct.new body: content }
end
it 'must change if check changed' do
check = add_check css: '#content'
status = reference_and_check! CHANGE_INSIDE_TARGET
expect(status).to be :changed
def check!(content)
stub_page content
# site._changes.delete_all
site.check!
end
expect(site.changed_at).not_to be_nil
expect(site.content).to eq CHANGE_INSIDE_TARGET
it 'must not change if no change with no target' do
expect_unchanged REFERENCE
end
# expect(check.changed_at).not_to be_nil
# expect(check.content).to eq CHANGE_TARGET
end
it 'must not change if no change with target' do
add_target css: '#content'
expect_unchanged REFERENCE
end
it 'must stay changed if no change after a change' do
first_date = Date.parse '2018-01-01'
second_date = first_date + 1
third_date = second_date + 1
it 'must change if change with no target' do
expect_changed CHANGE_OUTSIDE_TARGET
end
Timecop.freeze first_date do
status = reference_and_check! CHANGE_OUTSIDE_TARGET
expect(status).to be :changed
it 'must not change if change but no target changed' do
add_target css: '#content'
expect_unchanged CHANGE_OUTSIDE_TARGET
end
expect(site.changed_at).to eq first_date
expect(site.content).to eq CHANGE_OUTSIDE_TARGET
it 'must change if target changed' do
add_target css: '#content'
expect_changed CHANGE_INSIDE_TARGET
end
Timecop.freeze second_date do
status = check! CHANGE_OUTSIDE_TARGET
expect(status).to be :unchanged
it 'must stay changed if no change after a change' do
first_date = Date.parse '2018-01-01'
second_date = first_date + 1
third_date = second_date + 1
expect(site.changed_at).to eq first_date
expect(site.content).to eq CHANGE_OUTSIDE_TARGET
end
Timecop.freeze first_date do
expect_changed CHANGE_OUTSIDE_TARGET
end
Timecop.freeze third_date do
status = check! CHANGE_INSIDE_TARGET
expect(status).to be :changed
Timecop.freeze second_date do
expect_unchanged CHANGE_OUTSIDE_TARGET, first_date
end
expect(site.changed_at).to eq third_date
expect(site.content).to eq CHANGE_INSIDE_TARGET
Timecop.freeze third_date do
expect_changed CHANGE_INSIDE_TARGET, CHANGE_OUTSIDE_TARGET
end
end
end
end

Write Preview
Loading…
Cancel
Save