diff --git a/app/lib/http.rb b/app/lib/http.rb index 1240708..3ffb739 100644 --- a/app/lib/http.rb +++ b/app/lib/http.rb @@ -59,16 +59,17 @@ class Http prefix = self.class.prefix @url body = response.body - last = Dir[File.join dir, "#{prefix}_*"].sort.last + last = Dir[File.join dir, "#{prefix}_*.xz"].sort.last if last - old = Digest::SHA256.file(last).hexdigest + last = self.class.cache last + old = Digest::SHA256.hexdigest last new = Digest::SHA256.hexdigest body return if old == new end time = Time.now.strftime DATE_FORMAT file = prefix + '_' + time + '.xz' - file = File.join dir, file + file = File.join HTTP_CACHE_DIR, file body = XZ.compress body, level: 9 File.binwrite file, body end diff --git a/app/models/attributes/compressed_text.rb b/app/models/attributes/compressed_text.rb new file mode 100644 index 0000000..244d4d2 --- /dev/null +++ b/app/models/attributes/compressed_text.rb @@ -0,0 +1,19 @@ +require 'xz' +require 'active_record/connection_adapters/postgresql_adapter' + +class CompressedText < ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea + def type + :compressed_text + end + + def serialize(value) + return if value.nil? + value = XZ.compress value, level: 9 + super value + end + + def deserialize(value) + return if value.nil? + XZ.decompress super + end +end diff --git a/app/models/site.rb b/app/models/site.rb index 9a3e24c..d4df7b6 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -3,6 +3,7 @@ class Site < ApplicationRecord belongs_to :template, optional: true has_many :targets, dependent: :delete_all has_many :diffs, dependent: :delete_all + attribute :reference, :compressed_text validates :url, presence: true @@ -59,7 +60,7 @@ class Site < ApplicationRecord } end end - self.diffs.create! content: diffs + self.diffs.create! content: diffs, created_at: date end self.last_error = nil rescue => e diff --git a/config/application.rb b/config/application.rb index db87c68..9f0f298 100644 --- a/config/application.rb +++ b/config/application.rb @@ -28,6 +28,8 @@ module Webmon # Middleware like session, flash, cookies can be added back manually. # Skip views, helpers and assets when generating a new resource. # config.api_only = true + # + config.autoload_paths += %w(app/models/attributes) config.assets.precompile << Proc.new { |_, fn| fn =~ /vendor\/assets\/images/ } end diff --git a/config/initializers/types.rb b/config/initializers/types.rb new file mode 100644 index 0000000..42c12f2 --- /dev/null +++ b/config/initializers/types.rb @@ -0,0 +1 @@ +ActiveRecord::Type.register :compressed_text, CompressedText, adapter: :postgresql diff --git a/db/migrate/20180510000002_create_sites.rb b/db/migrate/20180510000002_create_sites.rb index bfa57fe..47f6da0 100644 --- a/db/migrate/20180510000002_create_sites.rb +++ b/db/migrate/20180510000002_create_sites.rb @@ -4,7 +4,7 @@ class CreateSites < ActiveRecord::Migration[5.1] t.string :url, null: false t.string :name, index: true - t.text :reference + t.binary :reference t.belongs_to :group, index: true, foreign_key: true t.belongs_to :template, index: true, foreign_key: true diff --git a/db/migrate/20181127204747_create_diffs.rb b/db/migrate/20181127204747_create_diffs.rb new file mode 100644 index 0000000..8d3777c --- /dev/null +++ b/db/migrate/20181127204747_create_diffs.rb @@ -0,0 +1,10 @@ +class CreateDiffs < ActiveRecord::Migration[5.2] + def change + create_table :diffs do |t| + t.json :content, null: false + + t.belongs_to :site, index: true, foreign_key: true, null: false + t.datetime :created_at, null: false + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 5eddd6a..3f102fb 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -32,7 +32,7 @@ ActiveRecord::Schema.define(version: 2018_11_27_204747) do create_table "sites", force: :cascade do |t| t.string "url", null: false t.string "name" - t.text "reference" + t.binary "reference" t.bigint "group_id" t.bigint "template_id" t.string "last_error"