fetch.rb

require 'typhoeus'
require 'date'
require 'json'
require 'nokogiri'

URL = "https://stoerticker.de/inc/loadarchiv.php"
FILTER = "7" # 7 = MVG
LID = "227904"

def get_messages(lid, filter=[])
  request = Typhoeus::Request.new(URL, params: {lid: lid, evu: filter})

  request.on_complete do |res|
    json = {
      appconnect_time: res.appconnect_time,
      connect_time: res.connect_time,
      headers: res.headers,
      httpauth_avail: res.httpauth_avail,
      namelookup_time: res.namelookup_time,
      pretransfer_time: res.pretransfer_time,
      primary_ip: res.primary_ip,
      redirect_count: res.redirect_count,
      redirect_url: res.redirect_url,
      request_size: res.request_size,
      request_url: URL,
      response_code: res.response_code,
      return_code: res.return_code,
      return_message: res.return_message,
      size_download: res.size_download,
      size_upload: res.size_upload,
      starttransfer_time: res.starttransfer_time,
      total_time: res.total_time
    }
    
    unless res.response_code == 200
      puts("Got respone code #{res.response_code} at lid `#{lid}`")
      exit 1 
    end

    File.write("data/#{lid}_meta.json", JSON.pretty_generate(json))
    File.write("data/#{lid}_body.html", res.body)
  
    document = Nokogiri::HTML(res.body)
    new_lid = document.search('.card').last.at('.clearfix')['href'].split('=')[1]
    date = Date.parse(document.search('.card').last.at('span').text.split(' ')[1])

    puts "Fetched lid `#{lid}` at date #{date} in #{res.total_time.to_i}s"
  
    get_messages(new_lid, filter)
  end
  request.run
end

get_messages(LID, FILTER)