revising examples

examples/computation_graph_01.ml

@@ -4,7 +4,6 @@
 *)
 
 
-open Owl
 module G = Owl_computation_cpu_engine.Make (Owl_algodiff_primal_ops.D)
 include Owl_algodiff_generic.Make (G)
 
@@ -18,7 +17,7 @@ let visualise_01 () =
  let z = f x y in
  let inputs = [| unpack_elt x |> G.elt_to_node; unpack_elt y |> G.elt_to_node |] in
  let outputs = [| unpack_elt z |> G.elt_to_node |] in
- let graph = G.make_graph inputs outputs "graph" in
+ let graph = G.make_graph ~input:inputs ~output:outputs "graph" in
  let s = G.graph_to_dot graph in
  Owl_io.write_file "cgraph_01.dot" s;
  Sys.command "dot -Tpdf cgraph_01.dot -o cgraph_01.pdf" |> ignore
@@ -30,7 +29,7 @@ let visualise_02 () =
  let z = (grad (f x)) y in
  let inputs = [| unpack_elt x |> G.elt_to_node; unpack_elt y |> G.elt_to_node |] in
  let outputs = [| unpack_elt z |> G.elt_to_node |] in
- let s = G.make_graph inputs outputs "graph" |> G.graph_to_dot in
+ let s = G.make_graph ~input:inputs ~output:outputs "graph" |> G.graph_to_dot in
  Owl_io.write_file "cgraph_02.dot" s;
  Sys.command "dot -Tpdf cgraph_02.dot -o cgraph_02.pdf" |> ignore
 
@@ -42,18 +41,18 @@ let visualise_03 () =
  let z = f x y in
  let i0 = [| unpack_arr x |> G.arr_to_node; unpack_arr y |> G.arr_to_node |] in
  let o0 = [| primal z |> unpack_elt |> G.elt_to_node |] in
- let s0 = G.make_graph i0 o0 "graph" |> G.graph_to_dot in
+ let s0 = G.make_graph ~input:i0 ~output:o0 "graph" |> G.graph_to_dot in
  Owl_io.write_file "cgraph_03_forward.dot" s0;
  Sys.command "dot -Tpdf cgraph_03_forward.dot -o cgraph_03_forward.pdf" |> ignore;
 
  reverse_prop (pack_flt 1.) z;
  let x' = adjval x |> unpack_arr |> G.arr_to_node in
  let y' = adjval y |> unpack_arr |> G.arr_to_node in
  let i1 = [| unpack_arr x |> G.arr_to_node |] in
- let s1 = G.make_graph i1 [| x' |] "graph" |> G.graph_to_dot in
+ let s1 = G.make_graph ~input:i1 ~output:[| x' |] "graph" |> G.graph_to_dot in
  let i2 = [| unpack_arr y |> G.arr_to_node |] in
- let s2 = G.make_graph i2 [| y' |] "graph" |> G.graph_to_dot in
- let s3 = G.make_graph i0 [| x'; y' |] "graph" |> G.graph_to_dot in
+ let s2 = G.make_graph ~input:i2 ~output:[| y' |] "graph" |> G.graph_to_dot in
+ let s3 = G.make_graph ~input:i0 ~output:[| x'; y' |] "graph" |> G.graph_to_dot in
  Owl_io.write_file "cgraph_03_backward_x.dot" s1;
  Sys.command "dot -Tpdf cgraph_03_backward_x.dot -o cgraph_03_backward_x.pdf" |> ignore;
  Owl_io.write_file "cgraph_03_backward_y.dot" s2;

examples/countmin_distributed.ml

@@ -1,4 +1,3 @@
-#!/usr/bin/env owl
 (* This example demonstrates the use of distributed count-min sketches. It
  * fills a single count-min sketch using the news.txt corpus at 
  * https://github.com/ryanrhymes/owl_dataset, then initializes two new empty

examples/countmin_texts.ml

@@ -1,4 +1,3 @@
-#!/usr/bin/env owl
 (* This example demonstrates the use of the HeavyHitters sketch,
  * which is based on the Count-Min sketch in Owl_base library. 
  * This example finds the words which appear with relative frequency

examples/dataframe.ml

@@ -1,11 +1,7 @@
-#!/usr/bin/env owl
 (* This example demonstrates the dataframe module in Owl. *)
 
 open Owl
 
-#zoo "3de010940ab340e3d2bfb564ecd7d6ba"
-
-
 let example_01 gist_path =
  let fname = gist_path ^ "funding.csv" in
  let types = [|"s";"s";"f";"s";"s";"s";"s";"f";"s";"s"|] in
@@ -19,44 +15,45 @@ let example_02 gist_path =
  let df = Dataframe.of_csv ~sep:',' ~types fname in
  Owl_pretty.pp_dataframe Format.std_formatter df
 
-
-let example_03 gist_path =
- let fname = gist_path ^ "insurance.csv" in
- let types = [|"i";"s";"s";"f";"f";"f";"f";"f";"f";"f";"f";"f";"f";"f";"f";"s";"s";"i"|] in
- let d0 = Dataframe.of_csv ~sep:',' ~types fname in
- let d1 = Dataframe.(filter_row (fun row ->
- (unpack_string row.(15)) = "Commercial" && (unpack_float row.(3)) > 0.
- ) d0) in
- Owl_pretty.pp_dataframe Format.std_formatter d1
-
-
 (* examples for automatic infer types and separators *)
 
-let example_04 gist_path =
+let example_03 gist_path =
  let fname = gist_path ^ "estate.csv" in
  let df = Dataframe.of_csv fname in
  Owl_pretty.pp_dataframe Format.std_formatter df
 
 
-let example_05 gist_path =
+let example_04 gist_path =
  let fname = gist_path ^ "estate.csv" in
  let d0 = Dataframe.of_csv fname in
- let d1 = Dataframe.(d0.?(fun row -> (unpack_string row.(7)) = "Condo")) in
+ let _d1 = Dataframe.(d0.?(fun row -> (unpack_string row.(7)) = "Condo")) in
  let d2 = Dataframe.(d0.?(fun row -> (unpack_string row.(7)) = "Condo").?(fun row -> (unpack_int row.(4)) = 2)) in
  Owl_pretty.pp_dataframe Format.std_formatter d2
 
 
-let example_06 gist_path =
+let example_05 gist_path =
  let fname = gist_path ^ "funding.csv" in
  let df = Dataframe.of_csv fname in
  Owl_pretty.pp_dataframe Format.std_formatter df
 
 
+(* The "insurance.csv" data can be downloaded by the Dataset module. *)
+
+(*
+let example_03 gist_path =
+ let fname = gist_path ^ "insurance.csv" in
+ let types = [|"i";"s";"s";"f";"f";"f";"f";"f";"f";"f";"f";"f";"f";"f";"f";"s";"s";"i"|] in
+ let d0 = Dataframe.of_csv ~sep:',' ~types fname in
+ let d1 = Dataframe.(filter_row (fun row ->
+ (unpack_string row.(15)) = "Commercial" && (unpack_float row.(3)) > 0.
+ ) d0) in
+ Owl_pretty.pp_dataframe Format.std_formatter d1 *)
+
 let _ =
- let gist_path = Owl_zoo_cmd.query_path "3de010940ab340e3d2bfb564ecd7d6ba" in
+ let gist_path = "data/" in
  example_01 gist_path;
  example_02 gist_path;
  example_03 gist_path;
  example_04 gist_path;
- example_05 gist_path;
- example_06 gist_path
+ example_05 gist_path
+ (* example_06 (Sys.getenv "HOME") ^ "/.owl/dataset/" *)

examples/dune

@@ -1,6 +1,30 @@
 (executables
  (names 
  backprop
- custom_algodiff_op 
+ checkpoint
+ cifar10_vgg
+ computation_graph_01
+ computation_graph_02
+ countmin_distributed
+ countmin_texts
+ custom_algodiff_op
+ dataframe
+ feedforward
+ gradient_descent
+ inception_v3
+ kmeans
+ lazy_cifar10
+ lazy_eval
+ lazy_lstm
+ lazy_mnist
+ linear_algebra
+ lstm
+ mnist_cnn
+ mnist_lenet
+ newton_method
+ squeezenet
+ test_log
+ tfidf
+ vgg16
  )
  (libraries owl))

examples/feedforward.ml

@@ -160,7 +160,7 @@ let conv2d ?name ?(padding = SAME) ?(init_typ=Init.Tanh) ?act_typ kernel stride
  |> add_layer ?act_typ nn;
  nn
 
-let conv3d ?name ?(padding = SAME) ?(init_typ=Init.Tanh) ?act_typ kernel_width kernel stride nn =
+let conv3d ?name ?(padding = SAME) ?(init_typ=Init.Tanh) ?act_typ _kernel_width kernel stride nn =
  Conv3D (Conv3D.create padding kernel stride init_typ)
  |> make_layer ?name nn
  |> add_layer ?act_typ nn;
@@ -252,7 +252,7 @@ let train ?state ?params ?(init_model=true) nn x y =
  Optimise.minimise_network ?state p (forward nn) (backward nn) (update nn) (save nn) (Arr x) (Arr y)
 
 let test_model nn x y =
- Dense.Matrix.S.iter2_rows (fun u v ->
+ Dense.Matrix.S.iter2_rows (fun u _v ->
  Owl_dataset.print_mnist_image u;
  let p = run (Arr u) nn |> unpack_arr in
  Owl_dense_matrix_generic.print p;

examples/lazy_cifar10.ml

@@ -7,7 +7,6 @@ module CGCompiler = Owl_neural_compiler.Make (CPU_Engine)
 
 open CGCompiler.Neural
 open CGCompiler.Neural.Graph
-open CGCompiler.Neural.Algodiff
 
 
 let make_network input_shape =

examples/lazy_eval.ml

@@ -38,7 +38,7 @@ let print_graph input =
  print_endline "visualise computation graph to dot file ...";
  let inputs = [|M.arr_to_node x; M.arr_to_node y|] in
  let outputs = [|M.arr_to_node z|] in
- let graph = M.make_graph inputs outputs "graph" in
+ let graph = M.make_graph ~input:inputs ~output:outputs "graph" in
  M.assign_arr x input;
  M.assign_arr y input;
  M.graph_to_dot graph |> Owl_io.write_file "plot_lazy.dot";

examples/lazy_lstm.ml

@@ -1,27 +1,26 @@
 (* This example demonstrates using lazy functor to train a LSTM network. *)
 
-#zoo "217ef87bc36845c4e78e398d52bc4c5b"
-
 open Owl
 
 module CPU_Engine = Owl_computation_cpu_engine.Make (Owl_algodiff_primal_ops.S)
 module CGCompiler = Owl_neural_compiler.Make (CPU_Engine)
 
 open CGCompiler.Neural
 open CGCompiler.Neural.Graph
-open CGCompiler.Neural.Algodiff
 
 
 let prepare window step =
  Owl_log.info "build vocabulary and tokenise ...";
- let chars = load_file ~gist:"217ef87bc36845c4e78e398d52bc4c5b" "wonderland.txt" |> String.lowercase_ascii in
+ let chars = Owl_io.read_file ~trim:true (Sys.getenv "HOME" ^ "/.owl/dataset/wonderland.txt") |> Array.to_list in
+ let chars = String.concat "" chars |> String.lowercase_ascii in
  let vocab = Nlp.Vocabulary.build_from_string ~alphabet:true chars in
  let t_arr = Nlp.Vocabulary.tokenise vocab chars |> Array.map float_of_int in
  let tokens = Dense.Ndarray.S.of_array t_arr [| Array.length t_arr |] in
 
  Owl_log.info "construct x (sliding) and y (one-hot) ...";
  let x = Dense.Ndarray.S.slide ~step:1 ~window tokens in
- let y = Dense.Ndarray.S.(one_hot (Nlp.Vocabulary.length vocab) tokens.${[[window;-1]]}) in
+ let l = Dense.Ndarray.S.get_slice [[window; -1]] tokens in
+ let y = Dense.Ndarray.S.(one_hot (Nlp.Vocabulary.length vocab) l) in
 
  Owl_log.info "chars:%i, symbols:%i, wndsz:%i, stepsz:%i"
  (String.length chars) (Nlp.Vocabulary.length vocab) window step;

examples/linear_algebra.ml

@@ -14,7 +14,7 @@ let qr_fact x =
 
 (* LU factorisation *)
 let lu_fact x =
- let l, u, a = Linalg.D.lu x in
+ let l, u, _a = Linalg.D.lu x in
  (* TODO: permutation *)
  Mat.(l *@ u)
 
@@ -46,10 +46,10 @@ let det_mat x =
 
 let _ =
  let x0 = Mat.uniform 8 8 in
- let x1 = Mat.uniform 8 10 in
+ let _x1 = Mat.uniform 8 10 in
+ let _ = lu_fact x0 in
  lq_fact x0;
  qr_fact x0;
- lu_fact x0;
  svd_fact x0;
  inv_mat x0;
  pinv_mat x0;

examples/lstm.ml

@@ -1,24 +1,23 @@
 (* Text Generation using LSTM.
  The book used is Alice’s Adventures in Wonderland by Lewis Carroll.
  *)
-
-#zoo "217ef87bc36845c4e78e398d52bc4c5b"
-
 open Owl
 open Neural.S
 open Neural.S.Graph
 
 
 let prepare window step =
  Owl_log.info "build vocabulary and tokenise ...";
- let chars = load_file ~gist:"217ef87bc36845c4e78e398d52bc4c5b" "wonderland.txt" |> String.lowercase_ascii in
+ let chars = Owl_io.read_file ~trim:true (Sys.getenv "HOME" ^ "/.owl/dataset/wonderland.txt") |> Array.to_list in
+ let chars = String.concat "" chars |> String.lowercase_ascii in
  let vocab = Nlp.Vocabulary.build_from_string ~alphabet:true chars in
  let t_arr = Nlp.Vocabulary.tokenise vocab chars |> Array.map float_of_int in
  let tokens = Dense.Ndarray.S.of_array t_arr [| Array.length t_arr |] in
 
  Owl_log.info "construct x (sliding) and y (one-hot) ...";
  let x = Dense.Ndarray.S.slide ~step:1 ~window tokens in
- let y = Dense.Ndarray.S.(one_hot (Nlp.Vocabulary.length vocab) tokens.${[[window;-1]]}) in
+ let l = Dense.Ndarray.S.get_slice [[window; -1]] tokens in
+ let y = Dense.Ndarray.S.(one_hot (Nlp.Vocabulary.length vocab) l) in
 
  Owl_log.info "chars:%i, symbols:%i, wndsz:%i, stepsz:%i"
  (String.length chars) (Nlp.Vocabulary.length vocab) window step;
@@ -42,7 +41,8 @@ let sample nn vocab wndsz tlen x =
  let next_i = Stats.(choose (Array.sub (argsort ~inc:false yt) 0 3) 1).(0) in
  Dense.Matrix.S.set all_char 0 (wndsz + i) (float_of_int next_i);
  done;
- Dense.Matrix.S.(to_array all_char.${ [[];[wndsz;-1]] })
+ Dense.Matrix.S.get_slice [[];[wndsz;-1]] all_char
+ |> Dense.Matrix.S.to_array
  |> Array.fold_left (fun a i -> a ^ Nlp.Vocabulary.index2word vocab (int_of_float i)) ""
  |> Printf.printf "generated text: %s\n"
  |> flush_all

examples/regression.ml

@@ -1,6 +1,7 @@
-#require "owl-plplot"
 (* Test Regression module *)
 
+(* You need to run `#require "owl-plplot"` first *)
+
 open Owl
 
 let generate_data () =
@@ -67,4 +68,4 @@ let _ =
  Owl_log.info "test lasso"; test_lasso (); flush_all ();
  Owl_log.info "test ridge"; test_ridge (); flush_all ();
  Owl_log.info "test elastic"; test_elastic (); flush_all ();
- Owl_log.info "test poly"; test_poly (); flush_all ()
+ Owl_log.info "test poly"; let _ = test_poly () in flush_all ()

examples/svm.ml

@@ -1,5 +1,6 @@
-#require "owl-plplot"
-(* This example demonstrates SVM regression. *)
+(* This example demonstrates SVM regression. 
+ You need to first execute `#require "owl-plplot;;"`
+*)
 
 open Owl
 open Owl_plplot

examples/test_log.ml

@@ -17,8 +17,8 @@ let generate_data () =
  let y2 = create c 1 ( 0.)in
  let x = concat_vertical x1 x2 in
  let y = concat_vertical y1 y2 in
- let _ = save_txt x1 "test_log.data1.tmp" in
- let _ = save_txt x2 "test_log.data2.tmp" in
+ let _ = save_txt x1 ~out:"test_log.data1.tmp" in
+ let _ = save_txt x2 ~out:"test_log.data2.tmp" in
  x, y
 
 

examples/tfidf.ml

@@ -5,7 +5,7 @@ open Owl_nlp
 
 let _ =
  let stopwords = Dataset.load_stopwords () in
- let data_path = Sys.getenv "HOME" ^ "/.owl/dataset/news.train" in
+ let data_path = Sys.getenv "HOME" ^ "/.owl/dataset/nips.train" in
  let x = Corpus.build ~stopwords data_path in
  let m = Tfidf.build ~norm:true x in
  Tfidf.save m "news.tfidf"

src/owl/misc/owl_dataset.ml

@@ -37,6 +37,7 @@ let download_all () =
  ; "enron.train.gz"
  ; "nips.test.gz"
  ; "nips.train.gz"
+ ; "insurance.csv.gz"
  ; "mnist-test-images.gz"
  ; "mnist-test-labels.gz"
  ; "mnist-test-lblvec.gz"