Skip to content

Commit

Permalink
[fix] Bring back last sstats field
Browse files Browse the repository at this point in the history
  • Loading branch information
@ptaoussanis
ptaoussanis committed Sep 1, 2023
1 parent 6b36ce1 commit 05ba8a4
Show file tree
Hide file tree
Showing 2 changed files with 26 additions and 21 deletions.
35 changes: 20 additions & 15 deletions src/taoensso/tufte/stats.cljc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,7 @@
^double xsum
^double xmin
^double xmax
^double xlast
^double p25
^double p50
^double p75
Expand All @@ -260,15 +261,16 @@
(assert (pos? nx))
(with-meta
{:n nx
:sum (fin (.-xsum ss))
:min (fin (.-xmin ss))
:max (fin (.-xmax ss))
:p25 (fin (.-p25 ss))
:p50 (fin (.-p50 ss))
:p75 (fin (.-p75 ss))
:p90 (fin (.-p90 ss))
:p95 (fin (.-p95 ss))
:p99 (fin (.-p99 ss))
:sum (fin (.-xsum ss))
:min (fin (.-xmin ss))
:max (fin (.-xmax ss))
:last (fin (.-xlast ss))
:p25 (fin (.-p25 ss))
:p50 (fin (.-p50 ss))
:p75 (fin (.-p75 ss))
:p90 (fin (.-p90 ss))
:p95 (fin (.-p95 ss))
:p99 (fin (.-p99 ss))

:mean (/ (.-xsum ss) nx)
:var (/ (.-xvar-sum ss) nx) ; Currently w/o bessel-correction
Expand Down Expand Up @@ -296,7 +298,7 @@
(enc/cond
(summary-stats? x) x
(map? x)
(let [{:keys [n sum min max p25 p50 p75 p90 p95 p99
(let [{:keys [n sum min max last p25 p50 p75 p90 p95 p99
#_mean #_var #_mad var-sum mad-sum]} x

floats?
Expand All @@ -306,7 +308,7 @@
:else (float-num? sum))]

(SummaryStats. floats?
n sum min max p25 p50 p75 p90 p95 p99 var-sum mad-sum))
n sum min max last p25 p50 p75 p90 p95 p99 var-sum mad-sum))

:else
(let [snums
Expand All @@ -320,16 +322,18 @@

(when (pos? nx)
(let [[xmin p25 p50 p75 p90 p95 p99 xmax] (percentiles snums)
xsum (double (reduce rf-sum 0.0 snums))
xbar (/ xsum nx)
xsum (double (reduce rf-sum 0.0 snums))
xbar (/ xsum nx)
xlast (nth snums (dec nx))
[^double xvar-sum ^double xmad-sum]
(enc/reduce-multi
(partial rf-sum-variance xbar) 0.0
(partial rf-sum-abs-deviation xbar) 0.0
snums)]

(SummaryStats. (sorted-doubles? snums)
nx xsum xmin xmax p25 p50 p75 p90 p95 p99 xvar-sum xmad-sum))))))))
nx xsum xmin xmax xlast p25 p50 p75 p90 p95 p99
xvar-sum xmad-sum))))))))

(comment @(summary-stats [1 2 3]))

Expand Down Expand Up @@ -373,6 +377,7 @@
xsum2 (.-xsum ss2)
xmin2 (.-xmin ss2)
xmax2 (.-xmax ss2)
xlast2 (.-xlast ss2)
p25-2 (.-p25 ss2)
p50-2 (.-p50 ss2)
p75-2 (.-p75 ss2)
Expand Down Expand Up @@ -409,7 +414,7 @@
p99-3 (+ (* nx1-ratio p99-1) (* nx2-ratio p99-2))]

(SummaryStats. xfloats3?
nx3 xsum3 xmin3 xmax3 p25-3 p50-3 p75-3 p90-3 p95-3 p99-3
nx3 xsum3 xmin3 xmax3 xlast2 p25-3 p50-3 p75-3 p90-3 p95-3 p99-3
xvar-sum3 xmad-sum3))
ss1)
ss2)))
Expand Down
12 changes: 6 additions & 6 deletions test/taoensso/stats_tests.cljc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@
(let [key-idx
(into {}
(map-indexed (fn [n k] [k n])
[:n :sum :min :max :p25 :p50 :p75 :p90 :p95 :p99
[:n :sum :min :max :last :p25 :p50 :p75 :p90 :p95 :p99
:mean :var :mad :var-sum :mad-sum :meta]))

comparator (fn [k1 k2] (< (long (get key-idx k1 -1)) (long (get key-idx k2 -1))))]
Expand All @@ -105,12 +105,12 @@

(is
(= (sorted-sstats @(stats/summary-stats (range 1 1001)))
{:n 1000, :sum 500500, :min 1, :max 1000, :p25 251, :p50 501, :p75 750, :p90 900, :p95 950, :p99 990,
{:n 1000, :sum 500500, :min 1, :max 1000, :last 1000, :p25 251, :p50 501, :p75 750, :p90 900, :p95 950, :p99 990,
:mean 500.5, :var 83333.25, :mad 250.0, :var-sum 8.333325E7, :mad-sum 250000.0, :meta {:floats? false}}))

(is
(= (sorted-sstats @(stats/summary-stats (range 0.5 1000)))
{:n 1000, :sum 500000.0, :min 0.5, :max 999.5, :p25 250.5, :p50 500.5, :p75 749.5, :p90 899.5, :p95 949.5, :p99 989.5,
{:n 1000, :sum 500000.0, :min 0.5, :max 999.5, :last 999.5, :p25 250.5, :p50 500.5, :p75 749.5, :p90 899.5, :p95 949.5, :p99 989.5,
:mean 500.0, :var 83333.25, :mad 250.0, :var-sum 8.333325E7, :mad-sum 250000.0, :meta {:floats? true}}))

(is
Expand All @@ -119,7 +119,7 @@
(stats/summary-stats (range 0 900))
(stats/summary-stats (range 200 500))))

{:n 1200, :sum 509400, :min 0, :max 899, :p25 238, :p50 425, :p75 612, :p90 724, :p95 762, :p99 792,
{:n 1200, :sum 509400, :min 0, :max 899, :last 499, :p25 238, :p50 425, :p75 612, :p90 724, :p95 762, :p99 792,
:mean 424.5, :var 52499.916666666664, :mad 187.5, :var-sum 6.29999E7, :mad-sum 225000.0, :meta {:floats? false}}))

(is
Expand All @@ -128,7 +128,7 @@
(stats/summary-stats (range 0.5 900))
(stats/summary-stats (range 200.5 500))))

{:n 1200, :sum 510000.0, :min 0.5, :max 899.5, :p25 238.0, :p50 425.5, :p75 612.0, :p90 724.5, :p95 762.0, :p99 792.0,
{:n 1200, :sum 510000.0, :min 0.5, :max 899.5, :last 499.5, :p25 238.0, :p50 425.5, :p75 612.0, :p90 724.5, :p95 762.0, :p99 792.0,
:mean 425.0, :var 52499.916666666664, :mad 187.5, :var-sum 6.29999E7, :mad-sum 225000.0, :meta {:floats? true}}))

(is
Expand All @@ -138,7 +138,7 @@
(stats/summary-stats (range 0 900))
(stats/summary-stats (range 200.5 500))))

{:n 1200, :sum 509550.0, :min 0.0, :max 899.0, :p25 237.625, :p50 425.125, :p75 611.625, :p90 724.125, :p95 761.625, :p99 791.625,
{:n 1200, :sum 509550.0, :min 0.0, :max 899.0, :last 499.5, :p25 237.625, :p50 425.125, :p75 611.625, :p90 724.125, :p95 761.625, :p99 791.625,
:mean 424.625, :var 52499.916666666664, :mad 187.5, :var-sum 6.29999E7, :mad-sum 225000.0, :meta {:floats? true}}))

(is (nil? (ss-merging-error 10 100 10)))
Expand Down

0 comments on commit 05ba8a4

Please sign in to comment.