Report formats support, misc column used by default (#135)

CHANGELOG.md

@@ -5,6 +5,29 @@ This change log follows the conventions of [keepachangelog.com](http://keepachan
 
 ## [Unreleased]
 
+## [0.45.0] - 2023-03-11
+
+Release **breaks backward compatibility** by adding mandatory `Misc`
+column to the reports. See `Changed` sections for more details.
+
+### Changed
+- `Misc` column no longer depends on the verbosity level and is always
+  shown. For `stdout` reports (default format) visibility of the
+  column can be suppresed via custom `formatter` (e.g. `%s %s %s` to
+  show only first three columns)
+- Default `--formatter` option spans 4 columns (`Dependency`, `License
+  name`, `License type`, `Misc`) and equals to `%-35s %-55s %-20s
+  %-40s`.
+- `--totals` formatting assumes that the first two columns delimited
+  with the same separator; the first separator is used (by default a
+  single space)
+
+### Added
+- Report output format option `--report-format` to support `stdout`
+  (default tabular report printed to the standard output), `json`,
+  `json-pretty` and `csv` formats
+  ([#90](https://github.com/pilosus/pip-license-checker/issues/90))
+
 ## [0.44.0] - 2023-02-25
 
 ### Fixed
@@ -413,7 +436,8 @@ weak copyleft types.
 ### Added
 - Structure for Leiningen app project
 
-[Unreleased]: https://github.com/pilosus/pip-license-checker/compare/0.44.0...HEAD
+[Unreleased]: https://github.com/pilosus/pip-license-checker/compare/0.45.0...HEAD
+[0.45.0]: https://github.com/pilosus/pip-license-checker/compare/0.44.0...0.45.0
 [0.44.0]: https://github.com/pilosus/pip-license-checker/compare/0.43.0...0.44.0
 [0.43.0]: https://github.com/pilosus/pip-license-checker/compare/0.42.1...0.43.0
 [0.42.1]: https://github.com/pilosus/pip-license-checker/compare/0.42.0...0.42.1

README.md

@@ -169,7 +169,8 @@ Description:
   -x, --external FILE_NAME                 []                                      File containing package names and license names
       --external-format FILE_FORMAT        csv                                     External file format: csv, cocoapods, gradle
       --external-options OPTS_EDN_STRING   {:skip-header true, :skip-footer true}  String of options map in EDN format
-      --formatter PRINTF_FMT               %-35s %-55s %-20s                       Printf-style formatter string for report formatting
+      --report-format FORMAT               stdout                                  Report format: stdout, json, json-pretty, csv
+      --formatter PRINTF_FMT               %-35s %-55s %-20s                       Printf-style formatter string for stdout report formatting
   -f, --fail LICENSE_TYPE                  #{}                                     Return non-zero exit code if license type is found
   -e, --exclude REGEX                                                              PCRE to exclude packages with matching names
       --exclude-license REGEX                                                      PCRE to exclude packages with matching license names

project.clj

@@ -1,4 +1,4 @@
-(defproject org.pilosus/pip-license-checker "0.44.0"
+(defproject org.pilosus/pip-license-checker "0.45.0"
   :description "License compliance tool to identify dependencies license names and types: permissive, copyleft, proprietory, etc."
   :url "https://github.com/pilosus/pip-license-checker"
   :license {:name "Eclipse Public License 2.0 OR GNU GPL v2+ with Classpath exception"

src/pip_license_checker/core.clj

@@ -166,7 +166,10 @@
    [nil "--external-options OPTS_EDN_STRING" "String of options map in EDN format"
     :default external/default-options
     :parse-fn external/opts-str->map]
-   [nil "--formatter PRINTF_FMT" "Printf-style formatter string for report formatting"
+   [nil "--report-format FORMAT" "Report format: stdout, json, json-pretty, csv"
+    :default report/format-stdout
+    :validate [report/valid-format? report/invalid-format]]
+   [nil "--formatter PRINTF_FMT" "Printf-style formatter string for stdout report formatting"
     :default report/report-formatter
     :validate [report/valid-formatter? report/invalid-formatter]]
    ["-f" "--fail LICENSE_TYPE" "Return non-zero exit code if license type is found"
@@ -260,5 +263,5 @@
     (-> arguments
         get-deps
         (get-report options)
-        (report/print-report options)
+        (report/format-report options)
         (shutdown options))))

src/pip_license_checker/report.clj

@@ -17,6 +17,8 @@
   "Formatting and printing a report"
   (:gen-class)
   (:require
+   [cheshire.core :as json]
+   [clojure.data.csv :as csv]
    [clojure.string :as str]
    [pip-license-checker.data :as d]))
 
@@ -27,8 +29,28 @@
    {:items items-header
     :totals totals-header}))
 
-(def report-formatter "%-35s %-55s %-20s")
-(def verbose-formatter "%-40s")
+(def report-formatter "%-35s %-55s %-20s %-40s")
+(def printf-specifier-regex #"\%[0 #+-]?[0-9*]*\.?\d*[hl]{0,2}[jztL]?[diuoxXeEfgGaAcpsSn%]")
+(def format-stdout "stdout")
+(def format-json "json")
+(def format-json-pretty "json-pretty")
+(def format-csv "csv")
+
+(def formats
+  (sorted-set
+   format-stdout
+   format-json
+   format-json-pretty
+   format-csv))
+
+(def invalid-format
+  (format "Invalid external format. Use one of: %s"
+          (str/join ", " formats)))
+
+(defn valid-format?
+  "Return true if format string is valid, false otherwise"
+  [format]
+  (contains? formats format))
 
 (defn valid-formatter?
   "Check if printf-style formatter string is valid"
@@ -50,19 +72,25 @@
   ([] (get-totals-fmt report-formatter 2))
   ([s] (get-totals-fmt s 2))
   ([s n]
-   (let [parts (str/split s #"\s+")
-         fmt (->> parts (take n) (str/join " "))]
+   (let [delim
+         (-> s
+             (str/split printf-specifier-regex)
+             ;; first is the empty string
+             rest
+             ;; assume all specifiers separated with the same delimiter
+             first)
+         split-pattern (re-pattern delim)
+         parts (str/split s split-pattern)
+         fmt (->> parts (take n) (str/join delim))]
      fmt)))
 
 (defn get-fmt
   "Get printf-style format string for given options and entity (:totals or :items)"
   [options entity]
-  (let [{:keys [formatter] :or {formatter report-formatter}} options
-        fmt (if (pos? (get options :verbose 0))
-              (format "%s %s" formatter verbose-formatter)
-              formatter)
-        fmt' (if (= entity :totals) (get-totals-fmt fmt) fmt)]
-    fmt'))
+  (let [{:keys [formatter] :or {formatter report-formatter}} options]
+    (if (= entity :totals)
+      (get-totals-fmt formatter)
+      formatter)))
 
 (defn get-items
   "Get a list of dependency fields ready printing"
@@ -79,7 +107,7 @@
   (println (apply format formatter items)))
 
 (defn print-report
-  "Print report to standard output"
+  "Default report printer to standard output"
   [report options]
   (let [{headers-opt :headers
          totals-opt :totals
@@ -106,3 +134,56 @@
 
     ;; return report for pipe to work properly
     report))
+
+(defn print-line-csv
+  [items]
+  (csv/write-csv *out* items :quote? (constantly true))
+  (flush))
+
+(defn print-csv
+  "CSV report printer to standard output"
+  [report options]
+  (let [{headers-opt :headers
+         totals-opt :totals
+         totals-only-opt :totals-only} options
+        {:keys [items totals headers]} report
+        show-totals (or totals-opt totals-only-opt)]
+
+    (when (not totals-only-opt)
+      (when headers-opt
+        (print-line-csv [(:items headers)]))
+
+      (print-line-csv (map get-items items))
+
+      (when totals-opt
+        (print-line-csv [[]])))
+
+    (when show-totals
+      (when headers-opt
+        (print-line-csv [(:totals headers)]))
+
+      (print-line-csv (vec totals)))
+
+    ;; return report for pipe to work properly
+    report))
+
+(defmulti format-report
+  "Format report and print to stdout"
+  (fn [_ options]
+    (get options :report-format)))
+
+(defmethod format-report :default [report options]
+  (print-report report options))
+
+(defmethod format-report "json" [report _]
+  (let [result (json/generate-string report)]
+    (println result)
+    result))
+
+(defmethod format-report "json-pretty" [report _]
+  (let [result (json/generate-string report {:pretty true})]
+    (println result)
+    result))
+
+(defmethod format-report "csv" [report options]
+  (print-csv report options))

test/pip_license_checker/core_test.clj

@@ -33,6 +33,7 @@
                :pre false
                :external-format "csv"
                :external-options external/default-options
+               :report-format report/format-stdout
                :formatter report/report-formatter
                :totals false
                :totals-only false
@@ -53,6 +54,7 @@
                :pre false
                :external-format "csv"
                :external-options external/default-options
+               :report-format report/format-stdout
                :formatter report/report-formatter
                :totals false
                :totals-only false
@@ -73,6 +75,7 @@
                :pre false
                :external-format "csv"
                :external-options external/default-options
+               :report-format report/format-stdout
                :formatter report/report-formatter
                :totals false
                :totals-only false
@@ -99,6 +102,7 @@
                :pre false
                :external-format "csv"
                :external-options external/default-options
+               :report-format report/format-stdout
                :formatter report/report-formatter
                :totals false
                :totals-only false
@@ -121,6 +125,7 @@
                :pre false
                :external-format "csv"
                :external-options external/default-options
+               :report-format report/format-stdout
                :formatter report/report-formatter
                :totals false
                :totals-only false
@@ -145,6 +150,7 @@
                :pre false
                :external-format "csv"
                :external-options external/default-options
+               :report-format report/format-stdout
                :formatter report/report-formatter
                :totals false
                :totals-only false
@@ -170,6 +176,7 @@
                :pre false
                :external-format "cocoapods"
                :external-options {:skip-header false :skip-footer true :int-opt 42 :str-opt "str-val"}
+               :report-format report/format-stdout
                :formatter report/report-formatter
                :totals true
                :totals-only false
@@ -194,6 +201,7 @@
                :pre false
                :external-format "cocoapods"
                :external-options {:skip-header true, :skip-footer true}
+               :report-format report/format-stdout
                :formatter "%-50s %-50s %-30s"
                :totals false
                :totals-only false
@@ -204,6 +212,31 @@
                :exit true
                :rate-limits {:requests 120 :millis 60000}}}
     "Formatter string"]
+   [["--external"
+     "resources/external.cocoapods"
+     "--external-format"
+     "cocoapods"
+     "--report-format"
+     "json-pretty"]
+    {:requirements []
+     :external ["resources/external.cocoapods"]
+     :packages []
+     :options {:verbose 0
+               :fail #{}
+               :pre false
+               :external-format "cocoapods"
+               :external-options {:skip-header true, :skip-footer true}
+               :report-format report/format-json-pretty
+               :formatter report/report-formatter
+               :totals false
+               :totals-only false
+               :headers false
+               :fails-only false
+               :github-token nil
+               :parallel true
+               :exit true
+               :rate-limits {:requests 120 :millis 60000}}}
+    "Report format"]
    [["-v"
      "--external"
      "resources/external.cocoapods"
@@ -219,6 +252,7 @@
                :pre false
                :external-format "cocoapods"
                :external-options {:skip-header true, :skip-footer true}
+               :report-format report/format-stdout
                :formatter "%-50s %-50s %-30s"
                :totals false
                :totals-only false
@@ -244,6 +278,7 @@
                :pre false
                :external-format "cocoapods"
                :external-options {:skip-header true, :skip-footer true}
+               :report-format report/format-stdout
                :formatter "%-50s %-50s %-30s"
                :totals false
                :totals-only false
@@ -262,7 +297,7 @@
     "No packages, no requirements, no external files"]
    [["-r" "--resources/requirements.txt"
      "--formatter" "%s %s %s %s %s %d"]
-    {:exit-message "The following errors occurred while parsing command arguments:\nFailed to validate \"-r --resources/requirements.txt\": Requirements file does not exist\nFailed to validate \"--formatter %s %s %s %s %s %d\": Invalid formatter string. Expected a printf-style formatter to cover 4 columns of string data, e.g. '%-35s %-55s %-20s'"}
+    {:exit-message "The following errors occurred while parsing command arguments:\nFailed to validate \"-r --resources/requirements.txt\": Requirements file does not exist\nFailed to validate \"--formatter %s %s %s %s %s %d\": Invalid formatter string. Expected a printf-style formatter to cover 4 columns of string data, e.g. '%-35s %-55s %-20s %-40s'"}
     "Invalid option"]])
 
 (deftest ^:cli ^:default
@@ -328,7 +363,7 @@
      "--no-headers"
      "--no-parallel"
      "--no-exit"]
-    (str (format report/report-formatter "test:3.7.2" "MIT License" "Permissive") "\n")
+    (str (format report/report-formatter "test:3.7.2" "MIT License" "Permissive" "") "\n")
     "No headers"]
    [[{:ok? true,
       :requirement {:name "test", :version "3.7.2"},
@@ -342,8 +377,8 @@
      "--no-parallel"
      "--no-exit"]
     (str/join
-     [(str (format report/report-formatter "Dependency" "License Name" "License Type") "\n")
-      (str (format report/report-formatter "test:3.7.2" "MIT License" "Permissive") "\n")])
+     [(str (format report/report-formatter "Dependency" "License Name" "License Type" "Misc") "\n")
+      (str (format report/report-formatter "test:3.7.2" "MIT License" "Permissive" "") "\n")])
     "With headers"]
    [[{:ok? true,
       :requirement {:name "test", :version "3.7.2"},
@@ -357,8 +392,8 @@
      "--no-parallel"
      "--no-exit"]
     (str/join
-     [(str (format report/report-formatter "Dependency" "License Name" "License Type") "\n")
-      (str (format report/report-formatter "test:3.7.2" "MIT License" "Permissive") "\n")
+     [(str (format report/report-formatter "Dependency" "License Name" "License Type" "Misc") "\n")
+      (str (format report/report-formatter "test:3.7.2" "MIT License" "Permissive" "") "\n")
       "\n"
       (str (format (report/get-totals-fmt) "License Type" "Found") "\n")
       (str (format (report/get-totals-fmt) "Permissive" 1) "\n")])
@@ -391,8 +426,8 @@
      "--no-parallel"
      "--no-exit"]
     (str/join
-     [(str (format report/report-formatter "test:3.7.2" "MIT License" "Permissive") "\n")
-      (str (format report/report-formatter "another:0.1.2" "BSD License" "Permissive") "\n")])
+     [(str (format report/report-formatter "test:3.7.2" "MIT License" "Permissive" "") "\n")
+      (str (format report/report-formatter "another:0.1.2" "BSD License" "Permissive" "") "\n")])
     "Requirements and external file"]
    [[{:ok? true,
       :requirement {:name "test", :version "3.7.2"},