Compress content

master
aeris 2018-12-09 20:01:03 +01:00
parent ab4295d492
commit 14d983ec93
8 changed files with 40 additions and 6 deletions

View File

@ -59,16 +59,17 @@ class Http
prefix = self.class.prefix @url
body = response.body
last = Dir[File.join dir, "#{prefix}_*"].sort.last
last = Dir[File.join dir, "#{prefix}_*.xz"].sort.last
if last
old = Digest::SHA256.file(last).hexdigest
last = self.class.cache last
old = Digest::SHA256.hexdigest last
new = Digest::SHA256.hexdigest body
return if old == new
end
time = Time.now.strftime DATE_FORMAT
file = prefix + '_' + time + '.xz'
file = File.join dir, file
file = File.join HTTP_CACHE_DIR, file
body = XZ.compress body, level: 9
File.binwrite file, body
end

View File

@ -0,0 +1,19 @@
require 'xz'
require 'active_record/connection_adapters/postgresql_adapter'
class CompressedText < ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea
def type
:compressed_text
end
def serialize(value)
return if value.nil?
value = XZ.compress value, level: 9
super value
end
def deserialize(value)
return if value.nil?
XZ.decompress super
end
end

View File

@ -3,6 +3,7 @@ class Site < ApplicationRecord
belongs_to :template, optional: true
has_many :targets, dependent: :delete_all
has_many :diffs, dependent: :delete_all
attribute :reference, :compressed_text
validates :url, presence: true
@ -59,7 +60,7 @@ class Site < ApplicationRecord
}
end
end
self.diffs.create! content: diffs
self.diffs.create! content: diffs, created_at: date
end
self.last_error = nil
rescue => e

View File

@ -28,6 +28,8 @@ module Webmon
# Middleware like session, flash, cookies can be added back manually.
# Skip views, helpers and assets when generating a new resource.
# config.api_only = true
#
config.autoload_paths += %w(app/models/attributes)
config.assets.precompile << Proc.new { |_, fn| fn =~ /vendor\/assets\/images/ }
end

View File

@ -0,0 +1 @@
ActiveRecord::Type.register :compressed_text, CompressedText, adapter: :postgresql

View File

@ -4,7 +4,7 @@ class CreateSites < ActiveRecord::Migration[5.1]
t.string :url, null: false
t.string :name, index: true
t.text :reference
t.binary :reference
t.belongs_to :group, index: true, foreign_key: true
t.belongs_to :template, index: true, foreign_key: true

View File

@ -0,0 +1,10 @@
class CreateDiffs < ActiveRecord::Migration[5.2]
def change
create_table :diffs do |t|
t.json :content, null: false
t.belongs_to :site, index: true, foreign_key: true, null: false
t.datetime :created_at, null: false
end
end
end

View File

@ -32,7 +32,7 @@ ActiveRecord::Schema.define(version: 2018_11_27_204747) do
create_table "sites", force: :cascade do |t|
t.string "url", null: false
t.string "name"
t.text "reference"
t.binary "reference"
t.bigint "group_id"
t.bigint "template_id"
t.string "last_error"