(* Copyright (C) 2008 Mauricio Fernandez <mfp@acm.org> http//eigenclass.org *)
open Printf
open Num
module H = Hashtbl

let re = Pcre.regexp "^/ongoing/When/\\d\\d\\dx/\\d\\d\\d\\d/\\d\\d/\\d\\d/[^ .]+$"
let self_ref_re = Pcre.regexp "^\"http://www.tbray.org/ongoing/"
let (@@) f x = f x

let u_hits, u_bytes, s404s, clients, refs =
  H.create 8192, H.create 8192, H.create 8192, H.create 1384, H.create 16384

let incr_count h k = try incr (H.find h k) with Not_found -> H.add h k (ref 1)

let add_to_num_count h k n =
  try let r = H.find h k in r := !r +/ n with Not_found -> H.add h k (ref n)

let record client u bytes ref =
  if bytes <> 0 then add_to_num_count u_bytes u (num_of_int bytes);
  if Pcre.pmatch ~rex:re u then begin
    incr_count u_hits u;
    incr_count clients client;
    if ref <> "\"-\"" && not (Pcre.pmatch ~rex:self_ref_re ref) then
      incr_count refs (String.sub ref 1 (String.length ref - 2))
  end

let report ?(n = 10) compf print_f label hash =
  let keep n x ((elms, set) as t) =
    if elms = 0 then (1, PMap.add x true set)
    else let min, _ = PMap.min_binding set in
      if compf x min <= 0 then t
      else if elms < n then (elms + 1, PMap.add x true set)
      else (elms, PMap.add x true (PMap.remove_min_binding set)) in

  let elements (_, set) = List.rev (PMap.foldi (fun k _ l -> k :: l) set []) in
    printf "Top %s:\n" label;
    List.iter print_f @@ List.rev @@ elements @@
    H.fold (fun k v s -> keep n (!v, k) s) hash (0, PMap.create compf);
    printf "\n"

let process_line = let rex = Pcre.regexp "\\s+" in fun str ->
  let f = Pcre.asplit ~rex str in
    if Array.length f >= 11 && f.(5) = "\"GET" then
      let client = f.(0) and u = f.(6) and status = f.(8) and ref = f.(10) in
        match status with
            "200" -> record client u (int_of_string f.(9)) ref
          | "304" -> record client u 0 ref
          | "404" -> incr_count s404s u
          | _ -> ()

let () =
  let compare_pairs compare (a1, b1) (a2, b2) =
    match compare a1 a2 with 0 -> String.compare b2 b1 (* reverse *) | r -> r in
  let shrink s = if String.length s > 60 then String.sub s 0 60 ^ "..." else s in
  let r1 ?n = report ?n (compare_pairs compare)
                (fun (n, s) -> printf " %10d: %s\n" n (shrink s)) in
  let mega = Int 1024 */ Int 1024 in
  let r2 ?n =
    report ?n (compare_pairs Num.compare_num)
      (fun (n, s) -> printf " %9.1fM: %s\n" (float_of_num (n // mega)) (shrink s)) in
  Pcre.foreach_line process_line;
  printf "%d resources, %d 404s, %d clients\n\n"
    (H.length u_hits) (H.length s404s) (H.length clients);
  r1 "URIs by hit" u_hits;
  r2 "URIs by bytes" u_bytes;
  r1 "404s" s404s;
  r1 "client addresses" clients;
  r1 "referrers" refs;