Compress content

app/lib/http.rb

@@ -59,16 +59,17 @@ class Http
 		prefix = self.class.prefix @url
 
 		body = response.body
-		last = Dir[File.join dir, "#{prefix}_*"].sort.last
+		last = Dir[File.join dir, "#{prefix}_*.xz"].sort.last
 		if last
-			old = Digest::SHA256.file(last).hexdigest
+			last = self.class.cache last
+			old = Digest::SHA256.hexdigest last
 			new = Digest::SHA256.hexdigest body
 			return if old == new
 		end
 
 		time = Time.now.strftime DATE_FORMAT
 		file = prefix + '_' + time + '.xz'
-		file = File.join dir, file
+		file = File.join HTTP_CACHE_DIR, file
 		body = XZ.compress body, level: 9
 		File.binwrite file, body
 	end

app/models/attributes/compressed_text.rb

@@ -0,0 +1,19 @@
+require 'xz'
+require 'active_record/connection_adapters/postgresql_adapter'
+
+class CompressedText < ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea
+	def type
+		:compressed_text
+	end
+
+	def serialize(value)
+		return if value.nil?
+		value = XZ.compress value, level: 9
+		super value
+	end
+
+	def deserialize(value)
+		return if value.nil?
+		XZ.decompress super
+	end
+end

app/models/site.rb

@@ -3,6 +3,7 @@ class Site < ApplicationRecord
 	belongs_to :template, optional: true
 	has_many :targets, dependent: :delete_all
 	has_many :diffs, dependent: :delete_all
+	attribute :reference, :compressed_text
 
 	validates :url, presence: true
 
@@ -59,7 +60,7 @@ class Site < ApplicationRecord
 						}
 					end
 				end
-				self.diffs.create! content: diffs
+				self.diffs.create! content: diffs, created_at: date
 			end
 			self.last_error = nil
 		rescue => e

config/application.rb

@@ -28,6 +28,8 @@ module Webmon
 		# Middleware like session, flash, cookies can be added back manually.
 		# Skip views, helpers and assets when generating a new resource.
 		# config.api_only = true
+		#
+		config.autoload_paths += %w(app/models/attributes)
 
 		config.assets.precompile << Proc.new { |_, fn| fn =~ /vendor\/assets\/images/ }
 	end

config/initializers/types.rb

@@ -0,0 +1 @@
+ActiveRecord::Type.register :compressed_text, CompressedText, adapter: :postgresql

db/migrate/20180510000002_create_sites.rb

@@ -4,7 +4,7 @@ class CreateSites < ActiveRecord::Migration[5.1]
 			t.string :url, null: false
 			t.string :name, index: true
 
-			t.text :reference
+			t.binary :reference
 
 			t.belongs_to :group, index: true, foreign_key: true
 			t.belongs_to :template, index: true, foreign_key: true

db/migrate/20181127204747_create_diffs.rb

@@ -0,0 +1,10 @@
+class CreateDiffs < ActiveRecord::Migration[5.2]
+	def change
+		create_table :diffs do |t|
+			t.json :content, null: false
+
+			t.belongs_to :site, index: true, foreign_key: true, null: false
+			t.datetime :created_at, null: false
+		end
+	end
+end

db/schema.rb

@@ -32,7 +32,7 @@ ActiveRecord::Schema.define(version: 2018_11_27_204747) do
   create_table "sites", force: :cascade do |t|
     t.string "url", null: false
     t.string "name"
-    t.text "reference"
+    t.binary "reference"
     t.bigint "group_id"
     t.bigint "template_id"
     t.string "last_error"