From ebfd7e03cebdba303944834a93d85cee31750aaa Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 12 Nov 2019 20:58:43 -0500 Subject: [PATCH] User scripts, bug fixes, docker image --- CMakeLists.txt | 6 +- Docker/Dockerfile | 9 +++ Docker/build.sh | 8 +++ README.md | 37 +++++++++-- schema/mappings.json | 3 + scripting/README.md | 117 +++++++++++++++++++++++++++++++++++ scripting/genre_example.png | Bin 0 -> 26422 bytes src/cli.c | 33 ++++++++-- src/cli.h | 2 + src/index/elastic.c | 50 +++++++++++++-- src/index/elastic.h | 2 +- src/index/static_generated.c | 2 +- src/io/serialize.c | 5 +- src/main.c | 19 ++++-- src/web/static_generated.c | 8 +-- web/css/dark.css | 46 +++++++++++++- web/css/light.css | 21 ++++++- web/js/dom.js | 30 +++++++-- web/js/search.js | 116 ++++++++++++++++++++++++++++------ web/js/util.js | 14 ++++- web/search.html | 22 +++++-- 21 files changed, 488 insertions(+), 62 deletions(-) create mode 100644 Docker/Dockerfile create mode 100755 Docker/build.sh create mode 100644 scripting/README.md create mode 100644 scripting/genre_example.png diff --git a/CMakeLists.txt b/CMakeLists.txt index cd70f5f..5bf1d36 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -122,10 +122,10 @@ if (WITH_SIST2) target_compile_options(sist2 PRIVATE - -Ofast +# -Ofast # -march=native - -fno-stack-protector - -fomit-frame-pointer +# -fno-stack-protector +# -fomit-frame-pointer ) TARGET_LINK_LIBRARIES( diff --git a/Docker/Dockerfile b/Docker/Dockerfile new file mode 100644 index 0000000..d58f4f9 --- /dev/null +++ b/Docker/Dockerfile @@ -0,0 +1,9 @@ +FROM ubuntu:19.10 +MAINTAINER simon987 + +RUN apt update +RUN apt install -y libglib2.0-0 libcurl4 libmagic1 libharfbuzz-bin libopenjp2-7 + +ADD sist2 /root/sist2 + +ENTRYPOINT ["/root/sist2"] \ No newline at end of file diff --git a/Docker/build.sh b/Docker/build.sh new file mode 100755 index 0000000..255f931 --- /dev/null +++ b/Docker/build.sh @@ -0,0 +1,8 @@ +cp ../sist2 . + +version=$(./sist2 --version) + +echo "Version ${version}" +docker build . -t simon987/sist2:${version} -t simon987/sist2:latest +docker push simon987/sist2:${version} +docker push simon987/sist2:latest diff --git a/README.md b/README.md index 9baa3b3..37479a4 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ sist2 (Simple incremental search tool) * Extracts text from common file types\* * Generates thumbnails\* * Incremental scanning +* Automatic tagging from file attributes via [user scripts](scripting/README.md) \* See [format support](#format-support) @@ -21,11 +22,13 @@ sist2 (Simple incremental search tool) ## Getting Started 1. Have an [Elasticsearch](https://www.elastic.co/downloads/elasticsearch) instance running -1. Download the [latest sist2 release](https://github.com/simon987/sist2/releases) +1. + 1. Download the [latest sist2 release](https://github.com/simon987/sist2/releases) * + 1. *(or)* `docker pull simon987/sist2:latest` + -*Windows users*: `sist2` runs under [WSL](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux) - -*Mac users*: See [#1](https://github.com/simon987/sist2/issues/1) +\* *Windows users*: **sist2** runs under [WSL](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux) +\* *Mac users*: See [#1](https://github.com/simon987/sist2/issues/1) ## Example usage @@ -52,6 +55,32 @@ sist2 index --print ./my_idx > raw_documents.ndjson sist2 web --bind 0.0.0.0 --port 4321 ./my_idx1 ./my_idx2 ./my_idx3 ``` +### Use sist2 with docker + +**scan** +```bash +docker run -it \ + -v /path/to/files/:/files \ + -v $PWD/out/:/out \ + simon987/sist2 scan -t 4 /files -o /out/my_idx1 +``` +**index** +```bash +docker run -it --network host\ + -v $PWD/out/:/out \ + simon987/sist2 index /out/my_idx1 +``` + +**web** +```bash +docker run --rm --network host -d --name sist2\ + -v $PWD/out/my_idx:/idx \ + -v $PWD/my/files:/files + simon987/sist2 web --bind 0.0.0.0 /idx +docker stop sist2 +``` + + ## Format support File type | Library | Content | Thumbnail | Metadata diff --git a/schema/mappings.json b/schema/mappings.json index a2b9d42..12e4fd7 100644 --- a/schema/mappings.json +++ b/schema/mappings.json @@ -80,6 +80,9 @@ "analyzer": "my_nGram" } } + }, + "tag": { + "type": "keyword" } } } diff --git a/scripting/README.md b/scripting/README.md new file mode 100644 index 0000000..ad3c2f9 --- /dev/null +++ b/scripting/README.md @@ -0,0 +1,117 @@ +## User scripts + +*This document is under construction, more in-depth guide coming soon* + +During the `index` step, you can use the `--script-file