Browse Source

Remove useless checks

webui
aeris 2 years ago
parent
commit
8c7df91bb5
14 changed files with 188 additions and 289 deletions
  1. +6
    -2
      app/lib/http.rb
  2. +0
    -29
      app/models/check.rb
  3. +7
    -0
      app/models/group.rb
  4. +16
    -58
      app/models/site.rb
  5. +21
    -1
      app/models/target.rb
  6. +40
    -65
      bin/cli.rb
  7. +0
    -2
      bin/import.rb
  8. +1
    -0
      config/boot.rb
  9. +6
    -5
      config/database.yml
  10. +0
    -2
      db/migrate/20180510000002_create_sites.rb
  11. +0
    -15
      db/migrate/20180510000004_create_checks.rb
  12. +1
    -17
      db/schema.rb
  13. +14
    -0
      spec/helpers/sites_helper_spec.rb
  14. +76
    -93
      spec/models/site_spec.rb

+ 6
- 2
app/lib/http.rb View File

@@ -59,12 +59,16 @@ class Http

DATE_FORMAT = '%Y%m%d_%H%M%S'.freeze

def self.prefix(url)
Digest::SHA256.hexdigest url
end

def cache(response)
return unless ENV['DEBUG_HTTP']

prefix = Digest::SHA256.hexdigest @url
prefix = self.class.prefix @url
dir = File.join Rails.root, 'tmp', 'cache', 'http'
FileUtils.mkdir_p dirs unless Dir.exist? dir
FileUtils.mkdir_p dir unless Dir.exist? dir

body = response.body
last = Dir[File.join dir, "#{prefix}_*"].sort.last


+ 0
- 29
app/models/check.rb View File

@@ -1,29 +0,0 @@
class Check < ApplicationRecord
belongs_to :site
belongs_to :target

def to_s
self.target.to_s
end

def changed?(reference, content, debug: false)
target = self.target
reference = target.extract reference
content = target.extract content
changed = reference != content

if changed
puts Utils.diff reference, content if debug
return true
end

false
end

def diff(reference, content, context: 3, **kwargs)
target = self.target
reference = target.extract reference
content = target.extract content
Diffy::Diff.new reference, content, context: context, **kwargs
end
end

+ 7
- 0
app/models/group.rb View File

@@ -8,4 +8,11 @@ class Group < ApplicationRecord
def self.[](name)
self.where(name: name).first
end

def all_targets
targets = self.targets
template = self.template
targets += template.targets if template
targets
end
end

+ 16
- 58
app/models/site.rb View File

@@ -2,7 +2,6 @@ class Site < ApplicationRecord
belongs_to :group, optional: true
belongs_to :template, optional: true
has_many :targets
has_many :checks

validates :url, presence: true

@@ -14,47 +13,24 @@ class Site < ApplicationRecord
Http.new self.url
end

def inherited_targets
def all_targets
targets = self.targets
group = self.group
targets += group.targets if group
template = self.template
targets = template.targets if template
targets += template.all_targets if template
group = self.group
targets += group.all_targets if group
targets
end

def create_checks!
self.inherited_targets.each do |target|
self.checks.create! target: target
end
end

def reset!
self.update! reference: nil, content: nil, checked_at: nil, changed_at: nil, last_error: nil
end

def reference!(content)
self.update! reference: content, content: content, checked_at: Time.now, changed_at: nil, last_error: nil
end

def read!
return unless self.content
self.reference! self.content
end

def clear!
self.update! content: nil, checked_at: Time.now, changed_at: nil, last_error: nil
end

def diff(context: 3, **kwargs)
reference = self.reference
content = self.content
Diffy::Diff.new reference, content, context: context, **kwargs
end

def changed?(reference, content, debug: false)
checks = self.checks
if checks.empty?
def content_changed?(reference, content, debug: false)
targets = self.all_targets
if targets.empty?
if reference != content
puts Utils.diff reference, content if debug
return true
@@ -62,27 +38,28 @@ class Site < ApplicationRecord
return false
end

checks.each do |check|
changed = check.changed? reference, content, debug: debug
targets.each do |target|
changed = target.content_changed? reference, content, debug: debug
return true if changed
end

false
end

def diff!(reference, content, debug: false)
self.checked_at = Time.now
def diff!(reference, content, date: Time.now, debug: false)
self.checked_at = date
state = :unchanged

begin
changed = self.changed? reference, content, debug: debug
changed = self.content_changed? reference, content, debug: debug
if changed
self.content = content
self.reference = content
self.changed_at = self.checked_at
state = :changed
end
self.last_error = nil
rescue => e
raise
$stderr.puts e
self.last_error = e
state = :error
@@ -98,29 +75,10 @@ class Site < ApplicationRecord
self.update! name: grab.title unless self.name

unless self.reference
self.reference! content
self.update! reference: content
return :reference
else
return self.diff! self.content, content, debug: debug
end
end

def recalculate!(debug: false)
reference = self.reference
content = self.content || reference
changed_at = self.changed_at
state = :unchanged

changed = self.checks.find { |c| c.changed? reference, content, debug: debug }
if changed
state = :changed
changed_at ||= self.checked_at
else
changed_at = nil
return self.diff! self.reference, content, debug: debug
end

self.update! reference: reference, content: content, changed_at: changed_at

state
end
end

+ 21
- 1
app/models/target.rb View File

@@ -5,8 +5,9 @@ class Target < ApplicationRecord
has_many :checks

def to_s
return self.name if self.name

s = []
s << self.name if self.name
s << "from: #{self.from}" if self.from
s << "to: #{self.to}" if self.to
s << "css: #{self.css}" if self.css
@@ -56,4 +57,23 @@ class Target < ApplicationRecord
content = self.extract_css content
content
end

def content_changed?(reference, content, debug: false)
reference = self.extract reference
content = self.extract content
changed = reference != content

if changed
puts Utils.diff reference, content if debug
return true
end

false
end

def diff(reference, content, context: 3, **kwargs)
reference = self.extract reference
content = self.extract content
Diffy::Diff.new reference, content, context: context, **kwargs
end
end

+ 40
- 65
bin/cli.rb View File

@@ -4,7 +4,6 @@ require 'optparse'

# Force resolution to avoid cycle in autoloading
Http
Check
Target
Site
Group
@@ -30,10 +29,6 @@ def display(item)
end

class App < Thor
desc 'check <url>*', 'Check given sites for changes'
method_option :reset, type: :boolean, default: false, aliases: '-r', desc: 'Reset sites before check'
method_option :debug, type: :boolean, default: false, aliases: '-d', desc: 'Activate debug'

COLORS = {
reference: :blue,
unchanged: :green,
@@ -41,14 +36,16 @@ class App < Thor
error: { background: :red }
}.freeze

desc 'check <url>*', 'Check given sites for changes'
method_option :debug, type: :boolean, default: false, aliases: '-d', desc: 'Activate debug'


def check(urls = nil)
reset = options[:reset]
debug = options[:debug]

results = Hash.new 0

self.process urls do |site|
site.reset! if reset
result = site.check! debug: debug
results[result] += 1
color = COLORS[result]
@@ -67,68 +64,45 @@ class App < Thor
self.process urls, &:read!
end

desc 'clear <url>*', 'Clear given sites'

def clear(urls = nil)
self.process urls, &:clear!
end

desc 'diff <url>*', 'Display diff of the given sites'

def diff(urls = nil)
sites = self.sites urls
sites.each do |site|
next unless site.changed_at
puts "#{site.url.colorize :yellow}"
checks = site.checks
display site if checks.empty?
checks.each do |check|
next unless check.changed_at
puts " #{check.target}"
display check
end
end
end

desc 'recalculate <url>*', 'Recalculate state of given sites'
method_option :debug, type: :boolean, default: false, aliases: '-d', desc: 'Activate debug'

def recalculate(urls = nil)
debug = options[:debug]
results = Hash.new 0
desc 'redo <url>*', 'Redo diff from cache'

def redo(urls = nil)
cache = 'tmp/cache/http'
self.process urls do |site|
result = site.recalculate! debug: debug
color = COLORS[result]
results[result] += 1
result.to_s.colorize color
end

results.each do |k, v|
color = COLORS[k]
puts "#{k.to_s.colorize color}: #{v}"
site._changes.delete_all
reference = nil
fp = Http.prefix site.url
Dir["#{cache}/#{fp}_*"].sort.each do |file|
name = File.basename file
date = name.split('_', 2).last
date = DateTime.strptime date, Http::DATE_FORMAT
content = File.read file

unless reference
site.update! reference: content
else
status = site.diff! reference, content, date: date
ap site: site.url, date: date, status: status
end
reference = content
end
nil
end
end

desc 'reset <url>*', 'Reset state of given sites'

def reset(urls = nil)
self.process urls, &:reset!
end

desc 'redo <url> <date1> <date2>', 'Redo check from cache'

def redo(url, date1 = nil, date2 = nil)
site = Site.where(url: url).first
fp = Digest::SHA256.hexdigest url
dir = File.join Rails.root, 'tmp/cache/http'
reference = File.join dir, "#{fp}_#{date1}"
reference = File.read reference
content = File.join dir, "#{fp}_#{date2}"
content = File.read content

ap site.changed? reference, content, debug: true
end
# desc 'redo <url> <date1> <date2>', 'Redo check from cache'
#
# def redo(url, date1 = nil, date2 = nil)
# site = Site.where(url: url).first
# fp = Digest::SHA256.hexdigest url
# dir = File.join Rails.root, 'tmp/cache/http'
# reference = File.join dir, "#{fp}_#{date1}"
# reference = File.read reference
# content = File.join dir, "#{fp}_#{date2}"
# content = File.read content
#
# ap site.changed? reference, content, debug: true
# end

protected

@@ -139,7 +113,7 @@ class App < Thor

def process(urls)
sites = self.sites urls
Parallel.each sites, in_threads: 16 do |site|
Parallel.each sites, in_threads: 1 do |site|
ActiveRecord::Base.transaction do
url = site.url.colorize :yellow
begin
@@ -148,6 +122,7 @@ class App < Thor
result
rescue => e
puts "#{url} #{e.to_s.colorize :red}"
raise
nil
end
end


+ 0
- 2
bin/import.rb View File

@@ -81,7 +81,6 @@ def import_sites(sites, group = nil, skip_title: true)

site = Site.create! url: url, name: name, group: group
create_targets site, params
site.create_checks!
rescue => e
$stderr.puts "Unable to import site #{url.colorize :yellow}: #{e.to_s.colorize :red}"
raise
@@ -90,7 +89,6 @@ def import_sites(sites, group = nil, skip_title: true)
end

ActiveRecord::Base.transaction do
Check.destroy_all
Target.destroy_all
Site.destroy_all
Group.destroy_all


+ 1
- 0
config/boot.rb View File

@@ -1,3 +1,4 @@
ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)

require 'bundler/setup' # Set up gems listed in the Gemfile.
require 'bootsnap/setup'

+ 6
- 5
config/database.yml View File

@@ -5,21 +5,22 @@
# gem 'sqlite3'
#
default: &default
adapter: sqlite3
adapter: postgresql
host: localhost
user: postgres
password: postgres
database: webmon_<%= Rails.env %>
encoding: unicode
pool: 20
timeout: 5000

development:
<<: *default
database: db/development.sqlite3

# Warning: The database defined as "test" will be erased and
# re-generated from your development database when you run "rake".
# Do not set this db to the same as development or production.
test:
<<: *default
database: db/test.sqlite3

production:
<<: *default
database: db/production.sqlite3

+ 0
- 2
db/migrate/20180510000002_create_sites.rb View File

@@ -5,14 +5,12 @@ class CreateSites < ActiveRecord::Migration[5.1]
t.string :name, index: true

t.text :reference
t.text :content

t.belongs_to :group, index: true, foreign_key: true
t.belongs_to :template, index: true, foreign_key: true

t.string :last_error
t.datetime :checked_at
t.datetime :changed_at
end
end
end

+ 0
- 15
db/migrate/20180510000004_create_checks.rb View File

@@ -1,15 +0,0 @@
class CreateChecks < ActiveRecord::Migration[5.1]
def change
create_table :checks do |t|
t.text :reference
t.text :content

t.belongs_to :target, index: true, foreign_key: true
t.belongs_to :site, index: true, foreign_key: true

t.string :last_error
t.datetime :checked_at
t.datetime :changed_at
end
end
end

+ 1
- 17
db/schema.rb View File

@@ -10,23 +10,11 @@
#
# It's strongly recommended that you check this file into your version control system.

ActiveRecord::Schema.define(version: 20180510000004) do
ActiveRecord::Schema.define(version: 2018_05_10_000003) do

# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"

create_table "checks", force: :cascade do |t|
t.text "reference"
t.text "content"
t.bigint "target_id"
t.bigint "site_id"
t.string "last_error"
t.datetime "checked_at"
t.datetime "changed_at"
t.index ["site_id"], name: "index_checks_on_site_id"
t.index ["target_id"], name: "index_checks_on_target_id"
end

create_table "groups", force: :cascade do |t|
t.string "name", null: false
t.bigint "template_id"
@@ -38,12 +26,10 @@ ActiveRecord::Schema.define(version: 20180510000004) do
t.string "url", null: false
t.string "name"
t.text "reference"
t.text "content"
t.bigint "group_id"
t.bigint "template_id"
t.string "last_error"
t.datetime "checked_at"
t.datetime "changed_at"
t.index ["group_id"], name: "index_sites_on_group_id"
t.index ["name"], name: "index_sites_on_name"
t.index ["template_id"], name: "index_sites_on_template_id"
@@ -67,8 +53,6 @@ ActiveRecord::Schema.define(version: 20180510000004) do
t.index ["name"], name: "index_templates_on_name", unique: true
end

add_foreign_key "checks", "sites"
add_foreign_key "checks", "targets"
add_foreign_key "groups", "templates"
add_foreign_key "sites", "groups"
add_foreign_key "sites", "templates"


+ 14
- 0
spec/helpers/sites_helper_spec.rb View File

@@ -0,0 +1,14 @@
require 'rails_helper'

# Specs in this file have access to a helper object that includes
# the SitesHelper. For example:
#
# describe SitesHelper do
# describe "string concat" do
# it "concats two strings with spaces" do
# expect(helper.concat_strings("this","that")).to eq("this that")
# end
# end
# end
RSpec.describe SitesHelper, type: :helper do
end

+ 76
- 93
spec/models/site_spec.rb View File

@@ -1,117 +1,100 @@
RSpec.describe Site, type: :model do
REFERENCE_TARGET = '<div id="content">bar</div>'
REFERENCE = "<html><body>foo #{REFERENCE_TARGET}</body></html>"
CHANGE_OUTSIDE_TARGET = '<html><body>baz <div id="content">bar</div></body></html>'
CHANGE_TARGET = '<div id="content">baz</div>'
CHANGE_INSIDE_TARGET = "<html><body>foo #{CHANGE_TARGET}</body></html>"

let :site do
Site.create! url: 'http://localhost/'
end

let :check do
site.checks.first
end

def add_check(**args)
target = site.targets.create! args
site.checks.create! target: target
end

def stub_page(content)
allow(site).to receive(:grab) { OpenStruct.new body: content }
end

def check!(content)
stub_page content
site.check!
end

def reference_and_check!(content)
site.reference! REFERENCE
self.check! content
end

it 'must not change if no change with no check' do
status = reference_and_check! REFERENCE
expect(status).to be :unchanged

expect(site.changed_at).to be_nil
expect(site.content).to eq REFERENCE
end

it 'must not change if no change with checks' do
check = add_check css: '#content'
context '#check!' do
REFERENCE_TARGET = '<div id="content">bar</div>'
REFERENCE = "<html><body>foo #{REFERENCE_TARGET}</body></html>"
CHANGE_OUTSIDE_TARGET = '<html><body>baz <div id="content">bar</div></body></html>'
CHANGE_TARGET = '<div id="content">baz</div>'
CHANGE_INSIDE_TARGET = "<html><body>foo #{CHANGE_TARGET}</body></html>"

let :site do
site = Site.create! url: 'http://localhost/'
site.reference = REFERENCE
site
end

status = reference_and_check! REFERENCE
expect(status).to be :unchanged
let :target do
site.targets.first
end

expect(site.changed_at).to be_nil
expect(site.content).to eq REFERENCE
def expect_changed(content, reference = REFERENCE)
status = self.check! content
expect(status).to be :changed
expect(site.reference).to eq content

# expect(check.changed_at).to be_nil
# expect(check.content).to eq REFERENCE_TARGET
end
expect(site.changed_at).not_to be_nil

it 'must change if change with no check' do
status = reference_and_check! CHANGE_OUTSIDE_TARGET
expect(status).to be :changed
# changes = site._changes
# expect(changes.size).to be 1
# change = changes.first
#
# expect(change.from).to eq reference
# expect(change.to).to eq content
end

expect(site.changed_at).not_to be_nil
expect(site.content).to eq CHANGE_OUTSIDE_TARGET
end
def expect_unchanged(content, changed_at = nil)
status = self.check! content
expect(status).to be :unchanged

it 'must not change if change but no check changed' do
check = add_check css: '#content'
status = reference_and_check! CHANGE_OUTSIDE_TARGET
expect(status).to be :unchanged
# expect(site.changed_at).to eq changed_at
#
# changes = site._changes
# expect(changes.size).to be 0
end

expect(site.changed_at).to be_nil
expect(site.content).to be REFERENCE
def add_target(**args)
site.targets.create! args
end

# expect(check.changed_at).to be_nil
# expect(check.content).to eq REFERENCE_TARGET
end
def stub_page(content)
allow(site).to receive(:grab) { OpenStruct.new body: content }
end

it 'must change if check changed' do
check = add_check css: '#content'
status = reference_and_check! CHANGE_INSIDE_TARGET
expect(status).to be :changed
def check!(content)
stub_page content
# site._changes.delete_all
site.check!
end

expect(site.changed_at).not_to be_nil
expect(site.content).to eq CHANGE_INSIDE_TARGET
it 'must not change if no change with no target' do
expect_unchanged REFERENCE
end

# expect(check.changed_at).not_to be_nil
# expect(check.content).to eq CHANGE_TARGET
end
it 'must not change if no change with target' do
add_target css: '#content'
expect_unchanged REFERENCE
end

it 'must stay changed if no change after a change' do
first_date = Date.parse '2018-01-01'
second_date = first_date + 1
third_date = second_date + 1
it 'must change if change with no target' do
expect_changed CHANGE_OUTSIDE_TARGET
end

Timecop.freeze first_date do
status = reference_and_check! CHANGE_OUTSIDE_TARGET
expect(status).to be :changed
it 'must not change if change but no target changed' do
add_target css: '#content'
expect_unchanged CHANGE_OUTSIDE_TARGET
end

expect(site.changed_at).to eq first_date
expect(site.content).to eq CHANGE_OUTSIDE_TARGET
it 'must change if target changed' do
add_target css: '#content'
expect_changed CHANGE_INSIDE_TARGET
end

Timecop.freeze second_date do
status = check! CHANGE_OUTSIDE_TARGET
expect(status).to be :unchanged
it 'must stay changed if no change after a change' do
first_date = Date.parse '2018-01-01'
second_date = first_date + 1
third_date = second_date + 1

expect(site.changed_at).to eq first_date
expect(site.content).to eq CHANGE_OUTSIDE_TARGET
end
Timecop.freeze first_date do
expect_changed CHANGE_OUTSIDE_TARGET
end

Timecop.freeze third_date do
status = check! CHANGE_INSIDE_TARGET
expect(status).to be :changed
Timecop.freeze second_date do
expect_unchanged CHANGE_OUTSIDE_TARGET, first_date
end

expect(site.changed_at).to eq third_date
expect(site.content).to eq CHANGE_INSIDE_TARGET
Timecop.freeze third_date do
expect_changed CHANGE_INSIDE_TARGET, CHANGE_OUTSIDE_TARGET
end
end
end
end

Loading…
Cancel
Save