Browse Source

Compress content

master
aeris 4 years ago
parent
commit
14d983ec93
  1. 7
      app/lib/http.rb
  2. 19
      app/models/attributes/compressed_text.rb
  3. 3
      app/models/site.rb
  4. 2
      config/application.rb
  5. 1
      config/initializers/types.rb
  6. 2
      db/migrate/20180510000002_create_sites.rb
  7. 10
      db/migrate/20181127204747_create_diffs.rb
  8. 2
      db/schema.rb

7
app/lib/http.rb

@ -59,16 +59,17 @@ class Http
prefix = self.class.prefix @url
body = response.body
last = Dir[File.join dir, "#{prefix}_*"].sort.last
last = Dir[File.join dir, "#{prefix}_*.xz"].sort.last
if last
old = Digest::SHA256.file(last).hexdigest
last = self.class.cache last
old = Digest::SHA256.hexdigest last
new = Digest::SHA256.hexdigest body
return if old == new
end
time = Time.now.strftime DATE_FORMAT
file = prefix + '_' + time + '.xz'
file = File.join dir, file
file = File.join HTTP_CACHE_DIR, file
body = XZ.compress body, level: 9
File.binwrite file, body
end

19
app/models/attributes/compressed_text.rb

@ -0,0 +1,19 @@
require 'xz'
require 'active_record/connection_adapters/postgresql_adapter'
class CompressedText < ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea
def type
:compressed_text
end
def serialize(value)
return if value.nil?
value = XZ.compress value, level: 9
super value
end
def deserialize(value)
return if value.nil?
XZ.decompress super
end
end

3
app/models/site.rb

@ -3,6 +3,7 @@ class Site < ApplicationRecord
belongs_to :template, optional: true
has_many :targets, dependent: :delete_all
has_many :diffs, dependent: :delete_all
attribute :reference, :compressed_text
validates :url, presence: true
@ -59,7 +60,7 @@ class Site < ApplicationRecord
}
end
end
self.diffs.create! content: diffs
self.diffs.create! content: diffs, created_at: date
end
self.last_error = nil
rescue => e

2
config/application.rb

@ -28,6 +28,8 @@ module Webmon
# Middleware like session, flash, cookies can be added back manually.
# Skip views, helpers and assets when generating a new resource.
# config.api_only = true
#
config.autoload_paths += %w(app/models/attributes)
config.assets.precompile << Proc.new { |_, fn| fn =~ /vendor\/assets\/images/ }
end

1
config/initializers/types.rb

@ -0,0 +1 @@
ActiveRecord::Type.register :compressed_text, CompressedText, adapter: :postgresql

2
db/migrate/20180510000002_create_sites.rb

@ -4,7 +4,7 @@ class CreateSites < ActiveRecord::Migration[5.1]
t.string :url, null: false
t.string :name, index: true
t.text :reference
t.binary :reference
t.belongs_to :group, index: true, foreign_key: true
t.belongs_to :template, index: true, foreign_key: true

10
db/migrate/20181127204747_create_diffs.rb

@ -0,0 +1,10 @@
class CreateDiffs < ActiveRecord::Migration[5.2]
def change
create_table :diffs do |t|
t.json :content, null: false
t.belongs_to :site, index: true, foreign_key: true, null: false
t.datetime :created_at, null: false
end
end
end

2
db/schema.rb

@ -32,7 +32,7 @@ ActiveRecord::Schema.define(version: 2018_11_27_204747) do
create_table "sites", force: :cascade do |t|
t.string "url", null: false
t.string "name"
t.text "reference"
t.binary "reference"
t.bigint "group_id"
t.bigint "template_id"
t.string "last_error"

Loading…
Cancel
Save