Browse Source

Compress content

master
aeris 2 years ago
parent
commit
14d983ec93
8 changed files with 40 additions and 6 deletions
  1. +4
    -3
      app/lib/http.rb
  2. +19
    -0
      app/models/attributes/compressed_text.rb
  3. +2
    -1
      app/models/site.rb
  4. +2
    -0
      config/application.rb
  5. +1
    -0
      config/initializers/types.rb
  6. +1
    -1
      db/migrate/20180510000002_create_sites.rb
  7. +10
    -0
      db/migrate/20181127204747_create_diffs.rb
  8. +1
    -1
      db/schema.rb

+ 4
- 3
app/lib/http.rb View File

@@ -59,16 +59,17 @@ class Http
prefix = self.class.prefix @url

body = response.body
last = Dir[File.join dir, "#{prefix}_*"].sort.last
last = Dir[File.join dir, "#{prefix}_*.xz"].sort.last
if last
old = Digest::SHA256.file(last).hexdigest
last = self.class.cache last
old = Digest::SHA256.hexdigest last
new = Digest::SHA256.hexdigest body
return if old == new
end

time = Time.now.strftime DATE_FORMAT
file = prefix + '_' + time + '.xz'
file = File.join dir, file
file = File.join HTTP_CACHE_DIR, file
body = XZ.compress body, level: 9
File.binwrite file, body
end


+ 19
- 0
app/models/attributes/compressed_text.rb View File

@@ -0,0 +1,19 @@
require 'xz'
require 'active_record/connection_adapters/postgresql_adapter'

class CompressedText < ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea
def type
:compressed_text
end

def serialize(value)
return if value.nil?
value = XZ.compress value, level: 9
super value
end

def deserialize(value)
return if value.nil?
XZ.decompress super
end
end

+ 2
- 1
app/models/site.rb View File

@@ -3,6 +3,7 @@ class Site < ApplicationRecord
belongs_to :template, optional: true
has_many :targets, dependent: :delete_all
has_many :diffs, dependent: :delete_all
attribute :reference, :compressed_text

validates :url, presence: true

@@ -59,7 +60,7 @@ class Site < ApplicationRecord
}
end
end
self.diffs.create! content: diffs
self.diffs.create! content: diffs, created_at: date
end
self.last_error = nil
rescue => e


+ 2
- 0
config/application.rb View File

@@ -28,6 +28,8 @@ module Webmon
# Middleware like session, flash, cookies can be added back manually.
# Skip views, helpers and assets when generating a new resource.
# config.api_only = true
#
config.autoload_paths += %w(app/models/attributes)

config.assets.precompile << Proc.new { |_, fn| fn =~ /vendor\/assets\/images/ }
end


+ 1
- 0
config/initializers/types.rb View File

@@ -0,0 +1 @@
ActiveRecord::Type.register :compressed_text, CompressedText, adapter: :postgresql

+ 1
- 1
db/migrate/20180510000002_create_sites.rb View File

@@ -4,7 +4,7 @@ class CreateSites < ActiveRecord::Migration[5.1]
t.string :url, null: false
t.string :name, index: true

t.text :reference
t.binary :reference

t.belongs_to :group, index: true, foreign_key: true
t.belongs_to :template, index: true, foreign_key: true


+ 10
- 0
db/migrate/20181127204747_create_diffs.rb View File

@@ -0,0 +1,10 @@
class CreateDiffs < ActiveRecord::Migration[5.2]
def change
create_table :diffs do |t|
t.json :content, null: false

t.belongs_to :site, index: true, foreign_key: true, null: false
t.datetime :created_at, null: false
end
end
end

+ 1
- 1
db/schema.rb View File

@@ -32,7 +32,7 @@ ActiveRecord::Schema.define(version: 2018_11_27_204747) do
create_table "sites", force: :cascade do |t|
t.string "url", null: false
t.string "name"
t.text "reference"
t.binary "reference"
t.bigint "group_id"
t.bigint "template_id"
t.string "last_error"


Loading…
Cancel
Save