diff --git a/CHANGELOG.md b/CHANGELOG.md index cb0c091..6c7027f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,29 @@ This change log follows the conventions of [keepachangelog.com](http://keepachan ## [Unreleased] +## [0.45.0] - 2023-03-11 + +Release **breaks backward compatibility** by adding mandatory `Misc` +column to the reports. See `Changed` sections for more details. + +### Changed +- `Misc` column no longer depends on the verbosity level and is always + shown. For `stdout` reports (default format) visibility of the + column can be suppresed via custom `formatter` (e.g. `%s %s %s` to + show only first three columns) +- Default `--formatter` option spans 4 columns (`Dependency`, `License + name`, `License type`, `Misc`) and equals to `%-35s %-55s %-20s + %-40s`. +- `--totals` formatting assumes that the first two columns delimited + with the same separator; the first separator is used (by default a + single space) + +### Added +- Report output format option `--report-format` to support `stdout` + (default tabular report printed to the standard output), `json`, + `json-pretty` and `csv` formats + ([#90](https://github.com/pilosus/pip-license-checker/issues/90)) + ## [0.44.0] - 2023-02-25 ### Fixed @@ -413,7 +436,8 @@ weak copyleft types. ### Added - Structure for Leiningen app project -[Unreleased]: https://github.com/pilosus/pip-license-checker/compare/0.44.0...HEAD +[Unreleased]: https://github.com/pilosus/pip-license-checker/compare/0.45.0...HEAD +[0.45.0]: https://github.com/pilosus/pip-license-checker/compare/0.44.0...0.45.0 [0.44.0]: https://github.com/pilosus/pip-license-checker/compare/0.43.0...0.44.0 [0.43.0]: https://github.com/pilosus/pip-license-checker/compare/0.42.1...0.43.0 [0.42.1]: https://github.com/pilosus/pip-license-checker/compare/0.42.0...0.42.1 diff --git a/README.md b/README.md index 6a3ce20..3bbaa99 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,8 @@ Description: -x, --external FILE_NAME [] File containing package names and license names --external-format FILE_FORMAT csv External file format: csv, cocoapods, gradle --external-options OPTS_EDN_STRING {:skip-header true, :skip-footer true} String of options map in EDN format - --formatter PRINTF_FMT %-35s %-55s %-20s Printf-style formatter string for report formatting + --report-format FORMAT stdout Report format: stdout, json, json-pretty, csv + --formatter PRINTF_FMT %-35s %-55s %-20s Printf-style formatter string for stdout report formatting -f, --fail LICENSE_TYPE #{} Return non-zero exit code if license type is found -e, --exclude REGEX PCRE to exclude packages with matching names --exclude-license REGEX PCRE to exclude packages with matching license names diff --git a/project.clj b/project.clj index f7b2119..0b40090 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.pilosus/pip-license-checker "0.44.0" +(defproject org.pilosus/pip-license-checker "0.45.0" :description "License compliance tool to identify dependencies license names and types: permissive, copyleft, proprietory, etc." :url "https://github.com/pilosus/pip-license-checker" :license {:name "Eclipse Public License 2.0 OR GNU GPL v2+ with Classpath exception" diff --git a/src/pip_license_checker/core.clj b/src/pip_license_checker/core.clj index a7b7032..6461148 100644 --- a/src/pip_license_checker/core.clj +++ b/src/pip_license_checker/core.clj @@ -166,7 +166,10 @@ [nil "--external-options OPTS_EDN_STRING" "String of options map in EDN format" :default external/default-options :parse-fn external/opts-str->map] - [nil "--formatter PRINTF_FMT" "Printf-style formatter string for report formatting" + [nil "--report-format FORMAT" "Report format: stdout, json, json-pretty, csv" + :default report/format-stdout + :validate [report/valid-format? report/invalid-format]] + [nil "--formatter PRINTF_FMT" "Printf-style formatter string for stdout report formatting" :default report/report-formatter :validate [report/valid-formatter? report/invalid-formatter]] ["-f" "--fail LICENSE_TYPE" "Return non-zero exit code if license type is found" @@ -260,5 +263,5 @@ (-> arguments get-deps (get-report options) - (report/print-report options) + (report/format-report options) (shutdown options)))) diff --git a/src/pip_license_checker/report.clj b/src/pip_license_checker/report.clj index d455b77..1eb06d7 100644 --- a/src/pip_license_checker/report.clj +++ b/src/pip_license_checker/report.clj @@ -17,6 +17,8 @@ "Formatting and printing a report" (:gen-class) (:require + [cheshire.core :as json] + [clojure.data.csv :as csv] [clojure.string :as str] [pip-license-checker.data :as d])) @@ -27,8 +29,28 @@ {:items items-header :totals totals-header})) -(def report-formatter "%-35s %-55s %-20s") -(def verbose-formatter "%-40s") +(def report-formatter "%-35s %-55s %-20s %-40s") +(def printf-specifier-regex #"\%[0 #+-]?[0-9*]*\.?\d*[hl]{0,2}[jztL]?[diuoxXeEfgGaAcpsSn%]") +(def format-stdout "stdout") +(def format-json "json") +(def format-json-pretty "json-pretty") +(def format-csv "csv") + +(def formats + (sorted-set + format-stdout + format-json + format-json-pretty + format-csv)) + +(def invalid-format + (format "Invalid external format. Use one of: %s" + (str/join ", " formats))) + +(defn valid-format? + "Return true if format string is valid, false otherwise" + [format] + (contains? formats format)) (defn valid-formatter? "Check if printf-style formatter string is valid" @@ -50,19 +72,25 @@ ([] (get-totals-fmt report-formatter 2)) ([s] (get-totals-fmt s 2)) ([s n] - (let [parts (str/split s #"\s+") - fmt (->> parts (take n) (str/join " "))] + (let [delim + (-> s + (str/split printf-specifier-regex) + ;; first is the empty string + rest + ;; assume all specifiers separated with the same delimiter + first) + split-pattern (re-pattern delim) + parts (str/split s split-pattern) + fmt (->> parts (take n) (str/join delim))] fmt))) (defn get-fmt "Get printf-style format string for given options and entity (:totals or :items)" [options entity] - (let [{:keys [formatter] :or {formatter report-formatter}} options - fmt (if (pos? (get options :verbose 0)) - (format "%s %s" formatter verbose-formatter) - formatter) - fmt' (if (= entity :totals) (get-totals-fmt fmt) fmt)] - fmt')) + (let [{:keys [formatter] :or {formatter report-formatter}} options] + (if (= entity :totals) + (get-totals-fmt formatter) + formatter))) (defn get-items "Get a list of dependency fields ready printing" @@ -79,7 +107,7 @@ (println (apply format formatter items))) (defn print-report - "Print report to standard output" + "Default report printer to standard output" [report options] (let [{headers-opt :headers totals-opt :totals @@ -106,3 +134,56 @@ ;; return report for pipe to work properly report)) + +(defn print-line-csv + [items] + (csv/write-csv *out* items :quote? (constantly true)) + (flush)) + +(defn print-csv + "CSV report printer to standard output" + [report options] + (let [{headers-opt :headers + totals-opt :totals + totals-only-opt :totals-only} options + {:keys [items totals headers]} report + show-totals (or totals-opt totals-only-opt)] + + (when (not totals-only-opt) + (when headers-opt + (print-line-csv [(:items headers)])) + + (print-line-csv (map get-items items)) + + (when totals-opt + (print-line-csv [[]]))) + + (when show-totals + (when headers-opt + (print-line-csv [(:totals headers)])) + + (print-line-csv (vec totals))) + + ;; return report for pipe to work properly + report)) + +(defmulti format-report + "Format report and print to stdout" + (fn [_ options] + (get options :report-format))) + +(defmethod format-report :default [report options] + (print-report report options)) + +(defmethod format-report "json" [report _] + (let [result (json/generate-string report)] + (println result) + result)) + +(defmethod format-report "json-pretty" [report _] + (let [result (json/generate-string report {:pretty true})] + (println result) + result)) + +(defmethod format-report "csv" [report options] + (print-csv report options)) diff --git a/test/pip_license_checker/core_test.clj b/test/pip_license_checker/core_test.clj index 128a9de..ee4af15 100644 --- a/test/pip_license_checker/core_test.clj +++ b/test/pip_license_checker/core_test.clj @@ -33,6 +33,7 @@ :pre false :external-format "csv" :external-options external/default-options + :report-format report/format-stdout :formatter report/report-formatter :totals false :totals-only false @@ -53,6 +54,7 @@ :pre false :external-format "csv" :external-options external/default-options + :report-format report/format-stdout :formatter report/report-formatter :totals false :totals-only false @@ -73,6 +75,7 @@ :pre false :external-format "csv" :external-options external/default-options + :report-format report/format-stdout :formatter report/report-formatter :totals false :totals-only false @@ -99,6 +102,7 @@ :pre false :external-format "csv" :external-options external/default-options + :report-format report/format-stdout :formatter report/report-formatter :totals false :totals-only false @@ -121,6 +125,7 @@ :pre false :external-format "csv" :external-options external/default-options + :report-format report/format-stdout :formatter report/report-formatter :totals false :totals-only false @@ -145,6 +150,7 @@ :pre false :external-format "csv" :external-options external/default-options + :report-format report/format-stdout :formatter report/report-formatter :totals false :totals-only false @@ -170,6 +176,7 @@ :pre false :external-format "cocoapods" :external-options {:skip-header false :skip-footer true :int-opt 42 :str-opt "str-val"} + :report-format report/format-stdout :formatter report/report-formatter :totals true :totals-only false @@ -194,6 +201,7 @@ :pre false :external-format "cocoapods" :external-options {:skip-header true, :skip-footer true} + :report-format report/format-stdout :formatter "%-50s %-50s %-30s" :totals false :totals-only false @@ -204,6 +212,31 @@ :exit true :rate-limits {:requests 120 :millis 60000}}} "Formatter string"] + [["--external" + "resources/external.cocoapods" + "--external-format" + "cocoapods" + "--report-format" + "json-pretty"] + {:requirements [] + :external ["resources/external.cocoapods"] + :packages [] + :options {:verbose 0 + :fail #{} + :pre false + :external-format "cocoapods" + :external-options {:skip-header true, :skip-footer true} + :report-format report/format-json-pretty + :formatter report/report-formatter + :totals false + :totals-only false + :headers false + :fails-only false + :github-token nil + :parallel true + :exit true + :rate-limits {:requests 120 :millis 60000}}} + "Report format"] [["-v" "--external" "resources/external.cocoapods" @@ -219,6 +252,7 @@ :pre false :external-format "cocoapods" :external-options {:skip-header true, :skip-footer true} + :report-format report/format-stdout :formatter "%-50s %-50s %-30s" :totals false :totals-only false @@ -244,6 +278,7 @@ :pre false :external-format "cocoapods" :external-options {:skip-header true, :skip-footer true} + :report-format report/format-stdout :formatter "%-50s %-50s %-30s" :totals false :totals-only false @@ -262,7 +297,7 @@ "No packages, no requirements, no external files"] [["-r" "--resources/requirements.txt" "--formatter" "%s %s %s %s %s %d"] - {:exit-message "The following errors occurred while parsing command arguments:\nFailed to validate \"-r --resources/requirements.txt\": Requirements file does not exist\nFailed to validate \"--formatter %s %s %s %s %s %d\": Invalid formatter string. Expected a printf-style formatter to cover 4 columns of string data, e.g. '%-35s %-55s %-20s'"} + {:exit-message "The following errors occurred while parsing command arguments:\nFailed to validate \"-r --resources/requirements.txt\": Requirements file does not exist\nFailed to validate \"--formatter %s %s %s %s %s %d\": Invalid formatter string. Expected a printf-style formatter to cover 4 columns of string data, e.g. '%-35s %-55s %-20s %-40s'"} "Invalid option"]]) (deftest ^:cli ^:default @@ -328,7 +363,7 @@ "--no-headers" "--no-parallel" "--no-exit"] - (str (format report/report-formatter "test:3.7.2" "MIT License" "Permissive") "\n") + (str (format report/report-formatter "test:3.7.2" "MIT License" "Permissive" "") "\n") "No headers"] [[{:ok? true, :requirement {:name "test", :version "3.7.2"}, @@ -342,8 +377,8 @@ "--no-parallel" "--no-exit"] (str/join - [(str (format report/report-formatter "Dependency" "License Name" "License Type") "\n") - (str (format report/report-formatter "test:3.7.2" "MIT License" "Permissive") "\n")]) + [(str (format report/report-formatter "Dependency" "License Name" "License Type" "Misc") "\n") + (str (format report/report-formatter "test:3.7.2" "MIT License" "Permissive" "") "\n")]) "With headers"] [[{:ok? true, :requirement {:name "test", :version "3.7.2"}, @@ -357,8 +392,8 @@ "--no-parallel" "--no-exit"] (str/join - [(str (format report/report-formatter "Dependency" "License Name" "License Type") "\n") - (str (format report/report-formatter "test:3.7.2" "MIT License" "Permissive") "\n") + [(str (format report/report-formatter "Dependency" "License Name" "License Type" "Misc") "\n") + (str (format report/report-formatter "test:3.7.2" "MIT License" "Permissive" "") "\n") "\n" (str (format (report/get-totals-fmt) "License Type" "Found") "\n") (str (format (report/get-totals-fmt) "Permissive" 1) "\n")]) @@ -391,8 +426,8 @@ "--no-parallel" "--no-exit"] (str/join - [(str (format report/report-formatter "test:3.7.2" "MIT License" "Permissive") "\n") - (str (format report/report-formatter "another:0.1.2" "BSD License" "Permissive") "\n")]) + [(str (format report/report-formatter "test:3.7.2" "MIT License" "Permissive" "") "\n") + (str (format report/report-formatter "another:0.1.2" "BSD License" "Permissive" "") "\n")]) "Requirements and external file"] [[{:ok? true, :requirement {:name "test", :version "3.7.2"}, diff --git a/test/pip_license_checker/report_test.clj b/test/pip_license_checker/report_test.clj index c5f62f8..adfd0fe 100644 --- a/test/pip_license_checker/report_test.clj +++ b/test/pip_license_checker/report_test.clj @@ -44,16 +44,16 @@ (is (= expected (apply report/get-totals-fmt args))))))) (def params-get-fmt - [[{} :items "%-35s %-55s %-20s" + [[{} :items "%-35s %-55s %-20s %-40s" "items, no options"] [{:verbose 1} :items "%-35s %-55s %-20s %-40s" "items, verbose"] - [{:verbose 1 :formatter "%s %s %s"} :items "%s %s %s %-40s" - "items, verbose, customer formatter"] + [{:verbose 1 :formatter "%s %s %s"} :items "%s %s %s" + "items, verbose, custom formatter"] [{:verbose 1 :formatter "%s %s %s"} :totals "%s %s" "totals, verbose, customer formatter"] [{:verbose 0 :formatter "%s %s %s"} :totals "%s %s" - "totals, non-verbose, customer formatter"] + "totals, non-verbose, custom formatter"] [{:verbose 0} :totals "%-35s %-55s" "totals, non-verbose, default formatter"]]) @@ -122,40 +122,69 @@ (def params-report [[report - {:verbose 0 + {:report-format "stdout" + :verbose 0 :totals false :headers false :formatter "%s %s %s"} "aiohttp:3.7.2 Apache Software License Permissive\n" "Non-verbose, no headers, no-totals"] [report - {:verbose 0 + {:report-format "json" + :verbose 0 + :totals false + :headers false + :formatter "%s %s %s"} + "{\"headers\":{\"items\":[\"Dependency\",\"License Name\",\"License Type\",\"Misc\"],\"totals\":[\"License Type\",\"Found\"]},\"items\":[{\"dependency\":{\"name\":\"aiohttp\",\"version\":\"3.7.2\"},\"license\":{\"name\":\"Apache Software License\",\"type\":\"Permissive\"},\"misc\":\"Too many requests\"}],\"totals\":{\"Permissive\":1},\"fails\":null}\n" + "JSON, Non-verbose, no headers, no-totals"] + [report + {:report-format "json-pretty" + :verbose 0 + :totals false + :headers false + :formatter "%s %s %s"} + "{\n \"headers\" : {\n \"items\" : [ \"Dependency\", \"License Name\", \"License Type\", \"Misc\" ],\n \"totals\" : [ \"License Type\", \"Found\" ]\n },\n \"items\" : [ {\n \"dependency\" : {\n \"name\" : \"aiohttp\",\n \"version\" : \"3.7.2\"\n },\n \"license\" : {\n \"name\" : \"Apache Software License\",\n \"type\" : \"Permissive\"\n },\n \"misc\" : \"Too many requests\"\n } ],\n \"totals\" : {\n \"Permissive\" : 1\n },\n \"fails\" : null\n}\n" + "JSON pretty, Non-verbose, no headers, no-totals"] + [report + {:report-format "stdout" + :verbose 0 :totals false :headers true :formatter "%s %s %s"} "Dependency License Name License Type\naiohttp:3.7.2 Apache Software License Permissive\n" "Non-verbose, with headers, no-totals"] [report - {:verbose 0 + {:report-format "stdout" + :verbose 0 :totals true :headers true :formatter "%s %s %s"} "Dependency License Name License Type\naiohttp:3.7.2 Apache Software License Permissive\n\nLicense Type Found\nPermissive 1\n" "Non-verbose, with headers, with totals"] [report - {:verbose 1 + {:report-format "stdout" + :verbose 1 :totals true :headers true :formatter "%s %s %s"} - "Dependency License Name License Type Misc \naiohttp:3.7.2 Apache Software License Permissive Too many requests \n\nLicense Type Found\nPermissive 1\n" + "Dependency License Name License Type\naiohttp:3.7.2 Apache Software License Permissive\n\nLicense Type Found\nPermissive 1\n" "Verbose, with headers, with totals"] + [report + {:report-format "csv" + :verbose 1 + :totals true + :headers true + :formatter "%s %s %s"} + "\"Dependency\",\"License Name\",\"License Type\",\"Misc\"\n\"aiohttp:3.7.2\",\"Apache Software License\",\"Permissive\",\"Too many requests\"\n\n\"License Type\",\"Found\"\n\"Permissive\",\"1\"\n" + "CSV, verbose, with headers, with totals"] [{:headers {:items ["Dependency" "License Name" "License Type" "Misc"] :totals ["License Type" "Found"]} :items [] :totals {} :fails nil} - {:verbose 0 + {:report-format "stdout" + :verbose 0 :totals true :headers true :formatter "%s %s %s"} @@ -167,7 +196,8 @@ :items [] :totals {} :fails nil} - {:verbose 0 + {:report-format "stdout" + :verbose 0 :totals true :headers false :formatter "%s %s %s"} @@ -179,15 +209,16 @@ :items [] :totals {} :fails nil} - {:verbose 0 + {:report-format "stdout" + :verbose 0 :totals false :headers false :formatter "%s %s %s"} "" "No items, no headers, no totals"]]) -(deftest test-print-report - (testing "Print report" +(deftest test-format-report + (testing "Print formatted report" (doseq [[report options expected description] params-report] (testing description - (is (= expected (with-out-str (report/print-report report options)))))))) + (is (= expected (with-out-str (report/format-report report options))))))))