Better changes detection

.rspec

@@ -0,0 +1,2 @@
+--require spec_helper
+--require rails_helper

Gemfile

@@ -4,25 +4,30 @@ gem 'rails', '~> 5.1.6'
 gem 'puma', '~> 3.7'
 
 gem 'sqlite3'
+gem 'pg'
 
 gem 'nokogiri'
 gem 'httparty'
 gem 'awesome_print'
+gem 'thor'
 gem 'colorize'
 gem 'pry-rails'
 gem 'parallel'
+gem 'diffy'
 
 group :development, :test do
-  gem 'pry-byebug'
+	gem 'pry-byebug'
+
+	gem 'rspec-rails'
 end
 
 group :development do
-  gem 'listen', '>= 3.0.5', '< 3.2'
-  gem 'better_errors'
-  gem 'binding_of_caller'
+	gem 'listen', '>= 3.0.5', '< 3.2'
+	gem 'better_errors'
+	gem 'binding_of_caller'
 
-  gem 'spring'
-  gem 'spring-watcher-listen', '~> 2.0.0'
+	gem 'spring'
+	gem 'spring-watcher-listen', '~> 2.0.0'
 end
 
 gem 'tzinfo-data', platforms: %i[mingw mswin x64_mingw jruby]

Gemfile.lock

@@ -53,6 +53,8 @@ GEM
     concurrent-ruby (1.0.5)
     crass (1.0.4)
     debug_inspector (0.0.3)
+    diff-lcs (1.3)
+    diffy (3.2.1)
     erubi (1.7.1)
     ffi (1.9.23)
     globalid (0.4.1)
@@ -79,6 +81,7 @@ GEM
     nokogiri (1.8.2)
       mini_portile2 (~> 2.3.0)
     parallel (1.12.1)
+    pg (1.0.0)
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
@@ -118,6 +121,23 @@ GEM
     rb-fsevent (0.10.3)
     rb-inotify (0.9.10)
       ffi (>= 0.5.0, < 2)
+    rspec-core (3.7.1)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-rails (3.7.2)
+      actionpack (>= 3.0)
+      activesupport (>= 3.0)
+      railties (>= 3.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.1)
     ruby_dep (1.5.0)
     spring (2.0.2)
       activesupport (>= 4.2)
@@ -148,17 +168,21 @@ DEPENDENCIES
   better_errors
   binding_of_caller
   colorize
+  diffy
   httparty
   listen (>= 3.0.5, < 3.2)
   nokogiri
   parallel
+  pg
   pry-byebug
   pry-rails
   puma (~> 3.7)
   rails (~> 5.1.6)
+  rspec-rails
   spring
   spring-watcher-listen (~> 2.0.0)
   sqlite3
+  thor
   tzinfo-data
 
 BUNDLED WITH

app/lib/utils.rb

@@ -0,0 +1,16 @@
+module Utils
+	def self.utf8!(text)
+		return nil unless text
+		return text if text.encoding == Encoding::UTF_8
+		text.force_encoding 'utf-8'
+	end
+
+	def self.diff(a, b, context: 3, limit: 30)
+		a = self.utf8! a
+		b = self.utf8! b
+		diff = Diffy::Diff.new a, b, context: context
+		diff = diff.to_s :color
+		return '...(too much diff)...'.colorize :light_red if diff.lines.size > limit
+		diff
+	end
+end

app/models/check.rb

@@ -0,0 +1,69 @@
+class Check < ApplicationRecord
+	belongs_to :site
+	belongs_to :target
+
+	def reference!(content)
+		target    = self.target
+		reference = target.extract content
+		self.update! reference: reference, content: nil, checked_at: Time.now, changed_at: nil, last_error: nil
+	end
+
+	def diff!(content, debug: false)
+		return :previously_changed if self.changed_at
+
+		self.checked_at = Time.now
+		state           = :unchanged
+
+		begin
+			target    = self.target
+			reference = Utils.utf8! self.reference
+			content   = target.extract content
+			changed   = reference != content
+			if changed
+				puts Utils.diff reference, content if debug
+				state           = :changed
+				self.content    = content
+				self.changed_at = self.checked_at
+			end
+			self.last_error = nil
+		rescue => e
+			raise
+			$stderr.puts e
+			state           = :error
+			self.last_error = e
+		end
+
+		self.save!
+		state
+	end
+
+	def recalculate!(debug: false)
+		state = :unchanged
+
+		target    = self.target
+		reference = self.site.reference
+		content   = self.site.content || reference
+
+		reference = target.extract reference
+		content   = target.extract content
+
+		changed_at = self.changed_at
+
+		if reference == content
+			content    = nil
+			changed_at = nil
+		else
+			puts Utils.diff reference, content if debug
+			state      = :changed
+			changed_at ||= self.checked_at
+		end
+
+		self.update! reference: reference, content: content, changed_at: changed_at
+
+		state
+	end
+
+	def clear!
+		self.update! reference: nil, content: nil, checked_at: nil, changed_at: nil, last_error: nil
+	end
+end

app/models/group.rb

@@ -1,9 +1,7 @@
 class Group < ApplicationRecord
-	attribute :targets, :targets
-
 	belongs_to :template, optional: true
-	has_many :targets
 	has_many :sites
+	has_many :targets
 
 	validates :name, uniqueness: true
 

app/models/site.rb

@@ -1,12 +1,15 @@
 class Site < ApplicationRecord
-	attribute :targets, :targets
-
 	belongs_to :group, optional: true
 	belongs_to :template, optional: true
 	has_many :targets
+	has_many :checks
 
 	validates :url, presence: true
 
+	def self.[](url)
+		self.where(url: url).first
+	end
+
 	def self.grab(url)
 		response = HTTParty.get url, timeout: 10.seconds
 		raise "Receive #{response.code}" unless response.success?
@@ -27,66 +30,118 @@ class Site < ApplicationRecord
 		tag&.text
 	end
 
-	def all_targets
+	def inherited_targets
 		targets  = self.targets
 		group    = self.group
-		targets  += groups.targets if group
+		targets  += group.targets if group
 		template = self.template
 		targets  = template.targets if template
 		targets
 	end
 
-	def check
+	def create_checks!
+		self.inherited_targets.each do |target|
+			self.checks.create! target: target
+		end
+	end
+
+	def reference!(content)
+		self.update! reference: content, content: nil, checked_at: Time.now, changed_at: nil, last_error: nil
+		self.checks.each { |c| c.reference! content }
+	end
+
+	STATES = %i[unchanged previously_changed changed error].freeze
+
+	def update_state(current, state)
+		current_index = STATES.index current
+		state_index   = STATES.index state
+		current_index < state_index ? state : current
+	end
+
+	def diff!(content, debug: false)
 		self.checked_at = Time.now
-		state           = :no_changes
-		error           = nil
+		state           = :unchanged
 
 		begin
-			reference = self.reference
-
-			response = self.class.grab self.url
-			content  = response.body
-			unless reference
-				self.reference = content
-				state          = :new
+			reference = Utils.utf8! self.reference
+			checks    = self.checks
+			if checks.empty?
+				if reference != content
+					puts Utils.diff reference, content if debug
+					state = :changed
+				end
 			else
-				self.content = content
-				unchanged    = true
-
-				content_type = response.content_type
-				case content_type
-				when 'text/html'
-					targets = self.targets
-					if targets
-						targets.each do |target|
-							target_content   = target.extract content
-							target_reference = target.extract reference
-							target_unchanged = target_content == target_reference
-							unless target_unchanged
-								unchanged = target_unchanged
-								break
-							end
-						end
-					else
-						unchanged = content == reference
-					end
-				else
-					unchanged = content == reference
+				checks.each do |check|
+					check_state = check.diff! content, debug: debug
+					state       = self.update_state state, check_state
 				end
+			end
 
-				unless unchanged
-					self.changed_at = self.checked_at
-					state           = :changes
-				end
+			if state == :changed
+				self.content    = content
+				self.changed_at = self.checked_at
 			end
+
 			self.last_error = nil
 		rescue => e
-			self.last_error = e.to_s
-			error           = e
+			$stderr.puts e
+			self.last_error = e
 		end
 
 		self.save!
-		raise error if error
 		state
 	end
+
+	def check(debug: false)
+		return :previously_changed if self.changed_at
+		reference = self.reference
+		response  = self.class.grab self.url
+		content   = response.body
+		# case response.content_type
+		# when 'text/html'
+		# 	content = content.force_encoding 'utf-8'
+		# end
+		unless reference
+			self.reference! content
+			return :reference
+		else
+			return self.diff! content, debug: debug
+		end
+	end
+
+	def recalculate!(debug: false)
+		state = :unchanged
+
+		reference  = self.reference
+		content    = self.content || reference
+		changed_at = self.changed_at
+
+		states = self.checks.collect { |c| c.recalculate! debug: debug }.uniq
+		state  = :changed if states.include? :changed
+		if states.empty? && reference != content
+			state = :changed
+			puts Utils.diff reference, content if debug
+		end
+
+		if state == :changed
+			changed_at ||= self.checked_at
+		else
+			content    = nil
+			changed_at = nil
+		end
+
+		self.update! reference: reference, content: content, changed_at: changed_at
+
+		state
+	end
+
+	def read!
+		return unless self.content
+		self.reference! self.content
+	end
+
+	def reset!
+		self.update! reference: nil, content: nil, checked_at: nil, changed_at: nil, last_error: nil
+		self.checks.each &:clear!
+	end
 end

app/models/target.rb

@@ -1,2 +1,59 @@
 class Target < ApplicationRecord
+	has_many :templates
+	has_many :groups
+	has_many :sites
+	has_many :checks
+
+	def to_s
+		s = []
+		s << self.name if self.name
+		s << "from: #{self.from}" if self.from
+		s << "to: #{self.to}" if self.to
+		s << "css: #{self.css}" if self.css
+		s.join ' '
+	end
+
+	def extract_boundary(content)
+		return nil unless content
+		if self.from
+			i = content.index self.from
+			unless i
+				# $stderr.puts "Unable to find `from` #{self.from}"
+				return nil
+				raise "Unable to find `from` #{self.from}"
+			end
+			content = content[i..-1]
+		end
+
+		if self.to
+			i = content.index self.to
+			unless i
+				# $stderr.puts "Unable to find `to` #{self.to}"
+				return nil
+				raise "Unable to find `to` #{self.to}"
+			end
+			content = content[0..i+self.to.size]
+		end
+		content
+	end
+
+	def extract_css(content)
+		return nil unless content
+		return content unless self.css
+		content = Nokogiri::HTML.parse content
+		node    = content.at self.css
+		unless node
+			# $stderr.puts "Unable to find `css` #{self.css}"
+			return nil
+			raise "Unable to find `css` #{self.css}"
+		end
+		node.to_s
+	end
+
+	def extract(content)
+		return nil unless content
+		content = self.extract_boundary content
+		content = self.extract_css content
+		content
+	end
 end

app/models/targets.rb

@@ -1,88 +0,0 @@
-class Targets < ActiveRecord::Type::Value
-	class Target
-		def initialize(target)
-			@from = target['from']
-			@to   = target['to']
-			@css  = target['css']
-		end
-
-		def extract_boundary(content)
-			if @from
-				i = content.index @from
-				raise "Unable to find `from` #{@from}" unless i
-				content = content[i..-1]
-			end
-
-			if @to
-				i = content.index @to
-				raise "Unable to find `to` #{@to}" unless i
-				content = content[0..i]
-			end
-			content
-		end
-
-		def extract_css(content)
-			return content unless @css
-			content = Nokogiri::HTML.parse content
-			node    = content.at @css
-			raise "Unable to find `css` #{@css}" unless node
-			node.to_s
-		end
-
-		def extract(content)
-			content = self.extract_boundary content
-			content = self.extract_css content
-			content
-		end
-
-		def to_h
-			json         = {}
-			json['from'] = @from if @from
-			json['to']   = @to if @to
-			json['css']  = @css if @css
-			json
-		end
-
-		def empty?
-			!(@from || @to || @css)
-		end
-	end
-
-	def self.detect(object)
-		targets = object['targets']
-		if targets
-			targets = targets.collect { |t| create t }.flatten
-			return nil if targets.empty?
-			targets
-		end
-
-		target = create object
-		return nil unless target
-
-		[target]
-	end
-
-	def type
-		:string
-	end
-
-	def deserialize(value)
-		return nil unless value
-		value = YAML.load value
-		value.collect { |t| Target.new t }
-	end
-
-	def serialize(value)
-		return nil unless value
-		value = value.collect &:to_h
-		YAML.dump value
-	end
-
-	private
-
-	def self.create(target)
-		target = Target.new target
-		return nil if target.empty?
-		target
-	end
-end

app/models/template.rb

@@ -1,7 +1,6 @@
 class Template < ApplicationRecord
-	attribute :targets, :targets
-
 	has_many :targets
+	has_many :sites
 
 	validates :name, uniqueness: true
 

bin/check.rb

@@ -1,18 +0,0 @@
-#!./bin/rails runner
-Parallel.each Site.all, in_threads: 5 do |site|
-	ActiveRecord::Base.transaction do
-		print "Checking #{site.url.colorize :yellow}..."
-		begin
-			result = site.check
-			color  = case result
-					 when :new
-						 :blue
-					 when :changes
-						 :green
-					 end
-			puts " #{result.to_s.colorize color}"
-		rescue => e
-			puts " #{e.to_s.colorize :red}"
-		end
-	end
-end

bin/cli.rb

@@ -0,0 +1,113 @@
+#!./bin/rails runner
+require 'ostruct'
+require 'optparse'
+
+# Force resolution to avoid cycle in autoloading
+Check
+Target
+Site
+Group
+Template
+
+def fp(content)
+	return nil unless content
+	Digest::SHA1.hexdigest content
+end
+
+def display(item)
+	reference = item.reference
+	content   = item.content
+	ap reference: fp(reference),
+	   content: fp(content),
+	   checked_at: item.checked_at,
+	   changed_at: item.changed_at,
+	   last_error: item.last_error
+
+	if reference && content && reference != content
+		puts Utils.diff reference, content
+	end
+end
+
+class App < Thor
+	desc 'check', 'Check given sites for changes'
+	method_option :reset, type: :boolean, default: false, aliases: '-r', desc: 'Reset sites before check'
+	method_option :debug, type: :boolean, default: false, aliases: '-d', desc: 'Activate debug'
+
+	COLORS = {
+			reference:          :blue,
+			unchanged:          :green,
+			previously_changed: :light_red,
+			changed:            :red,
+			error:              { background: :red }
+	}.freeze
+
+	def check(urls = nil)
+		reset = options[:reset]
+		debug = options[:debug]
+
+		self.process urls do |site|
+			site.reset! if reset
+			result = site.check debug: debug
+			color  = COLORS[result]
+			result.to_s.colorize color
+		end
+	end
+
+	desc 'read', 'Mark given sites as read'
+
+	def read(urls = nil)
+		self.process urls, &:read!
+	end
+
+	desc 'diff', 'Display diff of the given sites'
+
+	def diff(urls = nil)
+		sites = self.sites urls
+		sites.each do |site|
+			next unless site.changed_at
+			puts "#{site.url.colorize :yellow}"
+			checks = site.checks
+			display site if checks.empty?
+			checks.each do |check|
+				next unless check.changed_at
+				puts "  #{check.target}"
+				display check
+			end
+		end
+	end
+
+	desc 'recalculate', 'Recalculate state of given sites'
+	method_option :debug, type: :boolean, default: false, aliases: '-d', desc: 'Activate debug'
+
+	def recalculate(urls = nil)
+		debug = options[:debug]
+		self.process urls do |site|
+			result = site.recalculate! debug: debug
+			color  = COLORS[result]
+			result.to_s.colorize color
+		end
+	end
+
+	protected
+
+	def sites(url)
+		return Site.where url: url if url
+		Site.all
+	end
+
+	def process(urls)
+		sites = self.sites urls
+		Parallel.each sites, in_threads: 16 do |site|
+			ActiveRecord::Base.transaction do
+				url = site.url.colorize :yellow
+				begin
+					result = yield site
+					puts "#{url} #{result}"
+				rescue => e
+					puts "#{url} #{e.to_s.colorize :red}"
+				end
+			end
+		end
+	end
+end
+App.start

bin/diff.rb

@@ -0,0 +1,67 @@
+#!./bin/rails runner
+# id = ARGV.first
+# check = Check.find id
+# reference = File.join Dir.tmpdir, "#{id}-reference"
+# File.write reference, check.reference
+# content = File.join Dir.tmpdir, "#{id}-content"
+# File.write content, check.content
+# system 'kompare', reference, content
+
+# puts 'Recalculating...'
+# Site.all.each do |site|
+# 	puts site.url.colorize :yellow
+# 	site.checks.each do |check|
+# 		puts '  ' + check.target.to_s
+# 		check.recalculate!
+# 	end
+# end
+
+def fp(content)
+	return nil unless content
+	Digest::SHA1.hexdigest content
+end
+
+def display(item)
+	reference = item.reference&.force_encoding 'utf-8'
+	content   = item.content&.force_encoding 'utf-8'
+	ap reference: fp(reference),
+	   content: fp(content),
+	   checked_at: item.checked_at,
+	   changed_at: item.changed_at,
+	   last_error: item.last_error
+
+	if reference && content && reference != content
+		diff = Diffy::Diff.new reference, content, context: 3
+		diff = diff.to_s :color
+		if diff.lines.size > 30
+			puts '...(too much diff)...'.colorize :light_red
+		else
+			puts diff
+		end
+	end
+end
+
+url   = ARGV.first
+sites = if url
+			if url == 'all'
+				Site.all
+			else
+				Site.where url: url
+			end
+		else
+			Site.where.not changed_at: nil
+		end
+
+sites.each do |site|
+	site.recalculate!
+	next unless site.changed_at
+	puts site.url.colorize :yellow
+	checks = site.checks
+	display site if checks.empty?
+	checks.each do |check|
+		# check.recalculate!
+		next unless check.changed_at
+		puts "  #{check.target}"
+		display check
+	end
+end

bin/import.rb

@@ -1,13 +1,31 @@
 #!./bin/rails runner
 import = YAML.load_file ARGV.first
 
+def create_target(parent, params)
+	css  = params['css']
+	from = params['from']
+	to   = params['to']
+	return nil unless css || from || to
+	parent.targets.create! name: params['name'],
+						   css:  css,
+						   from: from, to: to
+end
+
+def create_targets(parent, params)
+	targets = params['targets']
+	return targets.collect { |t| create_target parent, t } if targets
+	target = create_target parent, params
+	return nil unless target
+	[target]
+end
+
 def import_templates(templates)
 	return unless templates
 	templates.each do |name, params|
 		puts "Importing template #{name.colorize :yellow}"
-		targets = Targets.detect params
 		begin
-			Template.create! name: name, targets: targets
+			template = Template.create! name: name
+			create_targets template, params
 		rescue => e
 			$stderr.puts "Unable to import template #{name.colorize :yellow}: #{e.to_s.colorize :red}"
 		end
@@ -23,10 +41,9 @@ def import_groups(groups)
 		template      = Template[template_name] if template_name
 		$stderr.puts "Template #{template_name.colorize :yellow} not found for group #{name.colorize :yellow}" if template_name && !template
 
-		targets = Targets.detect params
-
-		group = begin
-			Group.create! name: name, template: template, targets: targets
+		begin
+			group = Group.create! name: name, template: template
+			create_targets group, params
 		rescue => e
 			$stderr.puts "Unable to import group #{name.colorize :yellow}: #{e.to_s.colorize :red}"
 			next
@@ -41,7 +58,7 @@ def import_sites(sites, group = nil, skip_title: true)
 	sites.each do |params|
 		case params
 		when String
-			url  = params
+			url    = params
 			params = {}
 		else
 			url = params['url']
@@ -58,13 +75,13 @@ def import_sites(sites, group = nil, skip_title: true)
 
 			unless group
 				group_name = params['group']
-				group = Group[group_name] if group_name
+				group      = Group[group_name] if group_name
 				$stderr.puts "Group #{group_name.colorize :yellow} not found for site #{url.colorize :yellow}" if group_name && !group
 			end
 
-			targets = Targets.detect params
-
-			Site.create! url: url, name: name, group: group, targets: targets
+			site = Site.create! url: url, name: name, group: group
+			create_targets site, params
+			site.create_checks!
 		rescue => e
 			$stderr.puts "Unable to import site #{url.colorize :yellow}: #{e.to_s.colorize :red}"
 			raise
@@ -73,6 +90,8 @@ def import_sites(sites, group = nil, skip_title: true)
 end
 
 ActiveRecord::Base.transaction do
+	Check.destroy_all
+	Target.destroy_all
 	Site.destroy_all
 	Group.destroy_all
 	Template.destroy_all

config/database.yml

@@ -6,7 +6,7 @@
 #
 default: &default
   adapter: sqlite3
-  pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
+  pool: 20
   timeout: 5000
 
 development:

config/initializers/types.rb

@@ -1 +0,0 @@
-ActiveRecord::Type.register :targets, Targets

db/migrate/20180510000000_create_targets.rb

@@ -1,11 +0,0 @@
-class CreateTargets < ActiveRecord::Migration[5.1]
-  def change
-    create_table :targets do |t|
-      t.belongs_to :template, index: true, foreign_key: true
-      t.belongs_to :group, index: true, foreign_key: true
-      t.belongs_to :site, index: true, foreign_key: true
-
-      t.timestamps
-    end
-  end
-end

db/migrate/20180510000002_create_groups.rb → db/migrate/20180510000001_create_groups.rb

@@ -1,13 +1,9 @@
 class CreateGroups < ActiveRecord::Migration[5.1]
 	def change
 		create_table :groups do |t|
-			t.string :name, null: false, unique: true
-			t.string :targets
+			t.string :name, null: false, index: { unique: true }
 
 			t.belongs_to :template, index: true, foreign_key: true
-
-			t.index :name, unique: true
-			t.index :template
 		end
 	end
 end

db/migrate/20180510000003_create_sites.rb → db/migrate/20180510000002_create_sites.rb

@@ -4,8 +4,6 @@ class CreateSites < ActiveRecord::Migration[5.1]
 			t.string :url, null: false
 			t.string :name, index: true
 
-			t.string :targets
-
 			t.binary :reference
 			t.binary :content
 

db/migrate/20180510000003_create_targets.rb

@@ -0,0 +1,14 @@
+class CreateTargets < ActiveRecord::Migration[5.1]
+	def change
+		create_table :targets do |t|
+			t.string :name
+			t.string :css
+			t.string :from
+			t.string :to
+
+			t.belongs_to :template, index: true, foreign_key: true
+			t.belongs_to :group, index: true, foreign_key: true
+			t.belongs_to :site, index: true, foreign_key: true
+		end
+	end
+end

db/migrate/20180510000004_create_checks.rb

@@ -0,0 +1,15 @@
+class CreateChecks < ActiveRecord::Migration[5.1]
+	def change
+		create_table :checks do |t|
+			t.binary :reference
+			t.binary :content
+
+			t.belongs_to :target, index: true, foreign_key: true
+			t.belongs_to :site, index: true, foreign_key: true
+
+			t.string :last_error
+			t.datetime :checked_at
+			t.datetime :changed_at
+		end
+	end
+end

db/schema.rb

@@ -10,25 +10,37 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180510000003) do
+ActiveRecord::Schema.define(version: 20180510000004) do
+
+  # These are extensions that must be enabled in order to support this database
+  enable_extension "plpgsql"
+
+  create_table "checks", force: :cascade do |t|
+    t.binary "reference"
+    t.binary "content"
+    t.bigint "target_id"
+    t.bigint "site_id"
+    t.string "last_error"
+    t.datetime "checked_at"
+    t.datetime "changed_at"
+    t.index ["site_id"], name: "index_checks_on_site_id"
+    t.index ["target_id"], name: "index_checks_on_target_id"
+  end
 
   create_table "groups", force: :cascade do |t|
     t.string "name", null: false
-    t.string "targets"
-    t.integer "template_id"
+    t.bigint "template_id"
     t.index ["name"], name: "index_groups_on_name", unique: true
     t.index ["template_id"], name: "index_groups_on_template_id"
-    t.index [nil], name: "index_groups_on_template"
   end
 
   create_table "sites", force: :cascade do |t|
     t.string "url", null: false
     t.string "name"
-    t.string "targets"
     t.binary "reference"
     t.binary "content"
-    t.integer "group_id"
-    t.integer "template_id"
+    t.bigint "group_id"
+    t.bigint "template_id"
     t.string "last_error"
     t.datetime "checked_at"
     t.datetime "changed_at"
@@ -38,11 +50,13 @@ ActiveRecord::Schema.define(version: 20180510000003) do
   end
 
   create_table "targets", force: :cascade do |t|
-    t.integer "template_id"
-    t.integer "group_id"
-    t.integer "site_id"
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
+    t.string "name"
+    t.string "css"
+    t.string "from"
+    t.string "to"
+    t.bigint "template_id"
+    t.bigint "group_id"
+    t.bigint "site_id"
     t.index ["group_id"], name: "index_targets_on_group_id"
     t.index ["site_id"], name: "index_targets_on_site_id"
     t.index ["template_id"], name: "index_targets_on_template_id"
@@ -53,4 +67,12 @@ ActiveRecord::Schema.define(version: 20180510000003) do
     t.index ["name"], name: "index_templates_on_name", unique: true
   end
 
+  add_foreign_key "checks", "sites"
+  add_foreign_key "checks", "targets"
+  add_foreign_key "groups", "templates"
+  add_foreign_key "sites", "groups"
+  add_foreign_key "sites", "templates"
+  add_foreign_key "targets", "groups"
+  add_foreign_key "targets", "sites"
+  add_foreign_key "targets", "templates"
 end

spec/models/site_spec.rb

@@ -0,0 +1,82 @@
+
+RSpec.describe Site, type: :model do
+	REFERENCE = '<html><body>foo <div id="content">bar</div></body></html>'
+	CHANGE_OUTSIDE_TARGET = '<html><body>baz <div id="content">bar</div></body></html>'
+	CHANGE_INSIDE_TARGET = '<html><body>foo <div id="content">baz</div></body></html>'
+
+	let :site do
+		Site.create! url: 'http://localhost/'
+	end
+
+	let :check do
+		site.checks.first
+	end
+
+	def add_check(**args)
+		target = site.targets.create! args
+		site.checks.create! target: target
+	end
+
+	def stub_page(content)
+		allow(Site).to receive(:grab) { OpenStruct.new body: content }
+	end
+
+	def check!(content)
+		site.reference! REFERENCE
+		stub_page content
+		site.check
+	end
+
+	it 'must not change if no change with no check' do
+		status = check! REFERENCE
+		expect(status).to be :unchanged
+
+		expect(site.changed_at).to be_nil
+		expect(site.content).to be_nil
+	end
+
+	it 'must not change if no change with checks' do
+		check = add_check css: '#content'
+
+		status = check! REFERENCE
+		expect(status).to be :unchanged
+
+		expect(site.changed_at).to be_nil
+		expect(site.content).to be_nil
+
+		expect(check.changed_at).to be_nil
+		expect(check.content).to be_nil
+	end
+
+	it 'must change if change with no check' do
+		status = check! CHANGE_OUTSIDE_TARGET
+		expect(status).to be :changed
+
+		expect(site.changed_at).not_to be_nil
+		expect(site.content).not_to be_nil
+	end
+
+	it 'must not change if change but no check changed' do
+		check = add_check css: '#content'
+		status = check! CHANGE_OUTSIDE_TARGET
+		expect(status).to be :unchanged
+
+		expect(site.changed_at).to be_nil
+		expect(site.content).to be_nil
+
+		expect(check.changed_at).to be_nil
+		expect(check.content).to be_nil
+	end
+
+	it 'must change if check changed' do
+		check = add_check css: '#content'
+		status = check! CHANGE_INSIDE_TARGET
+		expect(status).to be :changed
+
+		expect(site.changed_at).not_to be_nil
+		expect(site.content).not_to be_nil
+
+		expect(check.changed_at).not_to be_nil
+		expect(check.content).not_to be_nil
+	end
+end

spec/rails_helper.rb

@@ -0,0 +1,57 @@
+# This file is copied to spec/ when you run 'rails generate rspec:install'
+require 'spec_helper'
+ENV['RAILS_ENV'] ||= 'test'
+require File.expand_path('../../config/environment', __FILE__)
+# Prevent database truncation if the environment is production
+abort("The Rails environment is running in production mode!") if Rails.env.production?
+require 'rspec/rails'
+# Add additional requires below this line. Rails is not loaded until this point!
+
+# Requires supporting ruby files with custom matchers and macros, etc, in
+# spec/support/ and its subdirectories. Files matching `spec/**/*_spec.rb` are
+# run as spec files by default. This means that files in spec/support that end
+# in _spec.rb will both be required and run as specs, causing the specs to be
+# run twice. It is recommended that you do not name files matching this glob to
+# end with _spec.rb. You can configure this pattern with the --pattern
+# option on the command line or in ~/.rspec, .rspec or `.rspec-local`.
+#
+# The following line is provided for convenience purposes. It has the downside
+# of increasing the boot-up time by auto-requiring all files in the support
+# directory. Alternatively, in the individual `*_spec.rb` files, manually
+# require only the support files necessary.
+#
+# Dir[Rails.root.join('spec/support/**/*.rb')].each { |f| require f }
+
+# Checks for pending migrations and applies them before tests are run.
+# If you are not using ActiveRecord, you can remove this line.
+ActiveRecord::Migration.maintain_test_schema!
+
+RSpec.configure do |config|
+  # Remove this line if you're not using ActiveRecord or ActiveRecord fixtures
+  config.fixture_path = "#{::Rails.root}/spec/fixtures"
+
+  # If you're not using ActiveRecord, or you'd prefer not to run each of your
+  # examples within a transaction, remove the following line or assign false
+  # instead of true.
+  config.use_transactional_fixtures = true
+
+  # RSpec Rails can automatically mix in different behaviours to your tests
+  # based on their file location, for example enabling you to call `get` and
+  # `post` in specs under `spec/controllers`.
+  #
+  # You can disable this behaviour by removing the line below, and instead
+  # explicitly tag your specs with their type, e.g.:
+  #
+  #     RSpec.describe UsersController, :type => :controller do
+  #       # ...
+  #     end
+  #
+  # The different available types are documented in the features, such as in
+  # https://relishapp.com/rspec/rspec-rails/docs
+  config.infer_spec_type_from_file_location!
+
+  # Filter lines from Rails gems in backtraces.
+  config.filter_rails_from_backtrace!
+  # arbitrary gems may also be filtered via:
+  # config.filter_gems_from_backtrace("gem name")
+end

spec/spec_helper.rb

@@ -0,0 +1,96 @@
+# This file was generated by the `rails generate rspec:install` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+  # This option will default to `:apply_to_host_groups` in RSpec 4 (and will
+  # have no way to turn it off -- the option exists only for backwards
+  # compatibility in RSpec 3). It causes shared context metadata to be
+  # inherited by the metadata hash of host groups and examples, rather than
+  # triggering implicit auto-inclusion in groups with matching metadata.
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # This allows you to limit a spec run to individual examples or groups
+  # you care about by tagging them with `:focus` metadata. When nothing
+  # is tagged with `:focus`, all examples get run. RSpec also provides
+  # aliases for `it`, `describe`, and `context` that include `:focus`
+  # metadata: `fit`, `fdescribe` and `fcontext`, respectively.
+  config.filter_run_when_matching :focus
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = "spec/examples.txt"
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+  config.disable_monkey_patching!
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = "doc"
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end