diff --git a/.ci/seed_es_on_travis.sh b/.ci/seed_es_on_travis.sh
index f4222c9..6734555 100755
--- a/.ci/seed_es_on_travis.sh
+++ b/.ci/seed_es_on_travis.sh
@@ -30,70 +30,50 @@ case "$ES_VERSION" in
export ES_BINARY_URL="${ES1_ARCHIVE}/elasticsearch-$ES_VERSION.deb"
;;
- "1.4.4")
+ "1.7.5")
export MAPPING_FILE=${LEGACY_MAPPING_FILE};
export ES_BINARY_URL="${ES1_ARCHIVE}/elasticsearch-$ES_VERSION.deb"
;;
- "1.7.2")
- export MAPPING_FILE=${LEGACY_MAPPING_FILE};
- export ES_BINARY_URL="${ES1_ARCHIVE}/elasticsearch-$ES_VERSION.deb"
- ;;
-
- "2.0.2")
- export MAPPING_FILE=${LEGACY_MAPPING_FILE};
- export ES_BINARY_URL="${ES2_ARCHIVE}/$ES_VERSION/elasticsearch-$ES_VERSION.deb"
- ;;
-
- "2.1.2")
- export MAPPING_FILE=${LEGACY_MAPPING_FILE};
- export ES_BINARY_URL="${ES2_ARCHIVE}/$ES_VERSION/elasticsearch-$ES_VERSION.deb"
- ;;
-
- "2.2.2")
- export MAPPING_FILE=${LEGACY_MAPPING_FILE};
- export ES_BINARY_URL="${ES2_ARCHIVE}/$ES_VERSION/elasticsearch-$ES_VERSION.deb"
- ;;
-
- "2.3.5")
+ "2.4.2")
export MAPPING_FILE=${LEGACY_MAPPING_FILE};
export ES_BINARY_URL="${ES2_ARCHIVE}/$ES_VERSION/elasticsearch-$ES_VERSION.deb"
;;
- "5.0.2")
+ "5.6.16")
export MAPPING_FILE=${ES5_MAPPING_FILE};
export ES_BINARY_URL="${ES5PLUS_ARCHIVE}/elasticsearch-$ES_VERSION.deb"
;;
- "5.3.3")
- export MAPPING_FILE=${ES5_MAPPING_FILE};
- export ES_BINARY_URL="${ES5PLUS_ARCHIVE}/elasticsearch-$ES_VERSION.deb"
- ;;
-
- "5.4.3")
- export MAPPING_FILE=${ES5_MAPPING_FILE};
+ "6.0.1")
+ export MAPPING_FILE=${ES6_MAPPING_FILE};
export ES_BINARY_URL="${ES5PLUS_ARCHIVE}/elasticsearch-$ES_VERSION.deb"
;;
- "5.6.9")
- export MAPPING_FILE=${ES5_MAPPING_FILE};
+ "6.8.2")
+ export MAPPING_FILE=${ES6_MAPPING_FILE};
export ES_BINARY_URL="${ES5PLUS_ARCHIVE}/elasticsearch-$ES_VERSION.deb"
;;
- "6.0.1")
- export MAPPING_FILE=${ES6_MAPPING_FILE};
- export ES_BINARY_URL="${ES5PLUS_ARCHIVE}/elasticsearch-$ES_VERSION.deb"
+ "7.0.1")
+ export MAPPING_FILE=${ES7_MAPPING_FILE};
+ export ES_BINARY_URL="${ES5PLUS_ARCHIVE}/elasticsearch-$ES_VERSION-amd64.deb"
+ # overwrite SAMPLE_DATA_FILE to use the ES7-compliant data
+ export SAMPLE_DATA_FILE="${ES7_SAMPLE_DATA_FILE}"
;;
- "6.1.4")
- export ES_VERSION=6.1.4;
- export MAPPING_FILE=${ES6_MAPPING_FILE};
- export ES_BINARY_URL="${ES5PLUS_ARCHIVE}/elasticsearch-$ES_VERSION.deb"
+ "7.1.1")
+ export MAPPING_FILE=${ES7_MAPPING_FILE};
+ export ES_BINARY_URL="${ES5PLUS_ARCHIVE}/elasticsearch-$ES_VERSION-amd64.deb"
+ # overwrite SAMPLE_DATA_FILE to use the ES7-compliant data
+ export SAMPLE_DATA_FILE="${ES7_SAMPLE_DATA_FILE}"
;;
- "6.2.4")
- export MAPPING_FILE=${ES6_MAPPING_FILE};
- export ES_BINARY_URL="${ES5PLUS_ARCHIVE}/elasticsearch-$ES_VERSION.deb"
+ "7.2.1")
+ export MAPPING_FILE=${ES7_MAPPING_FILE};
+ export ES_BINARY_URL="${ES5PLUS_ARCHIVE}/elasticsearch-$ES_VERSION-amd64.deb"
+ # overwrite SAMPLE_DATA_FILE to use the ES7-compliant data
+ export SAMPLE_DATA_FILE="${ES7_SAMPLE_DATA_FILE}"
;;
"7.3.1")
diff --git a/.travis.yml b/.travis.yml
index 754e79e..7ce3098 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -13,24 +13,6 @@ before_script:
script:
- .ci/test.sh
-# Manually specifying each build configuration.
-# Would be better to figure out how to build a matrix with two
-# languages but share the ES_VERSION matrix across all builds.
-# - ES_VERSION=1.0.0
-# - ES_VERSION=1.4.4
-# - ES_VERSION=1.7.2
-# - ES_VERSION=2.0.2
-# - ES_VERSION=2.1.2
-# - ES_VERSION=2.2.2
-# - ES_VERSION=2.3.5
-# - ES_VERSION=5.0.2
-# - ES_VERSION=5.3.3
-# - ES_VERSION=5.4.3
-# - ES_VERSION=5.6.9
-# - ES_VERSION=6.0.1
-# - ES_VERSION=6.1.4
-# - ES_VERSION=6.2.4
-# - ES_VERSION=7.3.1
matrix:
include:
############
@@ -46,79 +28,49 @@ matrix:
warnings_are_errors: true
cache: packages
env:
- - ES_VERSION=1.4.4
+ - ES_VERSION=1.7.5
- TASK=rpkg
- language: r
warnings_are_errors: true
cache: packages
env:
- - ES_VERSION=1.7.2
+ - ES_VERSION=2.4.2
- TASK=rpkg
- language: r
warnings_are_errors: true
cache: packages
env:
- - ES_VERSION=2.0.2
+ - ES_VERSION=5.6.16
- TASK=rpkg
- language: r
warnings_are_errors: true
cache: packages
env:
- - ES_VERSION=2.1.2
- - TASK=rpkg
- - language: r
- warnings_are_errors: true
- cache: packages
- env:
- - ES_VERSION=2.2.2
- - TASK=rpkg
- - language: r
- warnings_are_errors: true
- cache: packages
- env:
- - ES_VERSION=2.3.5
- - TASK=rpkg
- - language: r
- warnings_are_errors: true
- cache: packages
- env:
- - ES_VERSION=5.0.2
- - TASK=rpkg
- - language: r
- warnings_are_errors: true
- cache: packages
- env:
- - ES_VERSION=5.3.3
- - TASK=rpkg
- - language: r
- warnings_are_errors: true
- cache: packages
- env:
- - ES_VERSION=5.4.3
+ - ES_VERSION=6.0.1
- TASK=rpkg
- language: r
warnings_are_errors: true
cache: packages
env:
- - ES_VERSION=5.6.9
+ - ES_VERSION=6.8.2
- TASK=rpkg
- language: r
warnings_are_errors: true
cache: packages
env:
- - ES_VERSION=6.0.1
+ - ES_VERSION=7.0.1
- TASK=rpkg
- language: r
warnings_are_errors: true
cache: packages
env:
- - ES_VERSION=6.1.4
+ - ES_VERSION=7.1.1
- TASK=rpkg
- language: r
warnings_are_errors: true
cache: packages
env:
- - ES_VERSION=6.2.4
+ - ES_VERSION=7.2.1
- TASK=rpkg
- language: r
warnings_are_errors: true
@@ -139,67 +91,42 @@ matrix:
- language: python
python: 3.5
env:
- - ES_VERSION=1.4.4
- - TASK=pypkg
- - language: python
- python: 3.5
- env:
- - ES_VERSION=1.7.2
- - TASK=pypkg
- - language: python
- python: 3.5
- env:
- - ES_VERSION=2.0.2
- - TASK=pypkg
- - language: python
- python: 3.5
- env:
- - ES_VERSION=2.1.2
- - TASK=pypkg
- - language: python
- python: 3.5
- env:
- - ES_VERSION=2.2.2
+ - ES_VERSION=1.7.5
- TASK=pypkg
- language: python
python: 3.5
env:
- - ES_VERSION=2.3.5
+ - ES_VERSION=2.4.2
- TASK=pypkg
- language: python
python: 3.5
env:
- - ES_VERSION=5.0.2
+ - ES_VERSION=5.6.16
- TASK=pypkg
- language: python
python: 3.5
env:
- - ES_VERSION=5.3.3
- - TASK=pypkg
- - language: python
- python: 3.5
- env:
- - ES_VERSION=5.4.3
+ - ES_VERSION=6.0.1
- TASK=pypkg
- language: python
python: 3.5
env:
- - ES_VERSION=5.6.9
+ - ES_VERSION=6.8.2
- TASK=pypkg
- language: python
python: 3.5
env:
- - ES_VERSION=6.0.1
+ - ES_VERSION=7.0.1
- TASK=pypkg
- language: python
python: 3.5
env:
- - ES_VERSION=6.1.4
+ - ES_VERSION=7.1.1
- TASK=pypkg
- language: python
python: 3.5
env:
- - ES_VERSION=6.2.4
+ - ES_VERSION=7.2.1
- TASK=pypkg
- language: python
python: 3.5
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 87fd787..4ffae73 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -6,7 +6,9 @@ The primary goal of this guide is to help you contribute to `uptasticsearch` as
* [Creating an Issue](#issues)
* [Submitting a Pull Request](#prs)
-* [Running Tests Locally](#testing)
+* [Testing Strategy](#testing)
+ * [Travis CI](#travis)
+ * [Running tests locally](#testing-local)
* [Releasing to CRAN (for maintainer)](#cran)
## Creating an Issue
@@ -61,7 +63,47 @@ To submit a PR, please follow these steps:
We will try to review PRs promptly and get back to you within a few days.
-## Running Tests Locally
+## Testing Strategy
+
+### Travis CI
+
+This project uses [Travis CI](https://travis-ci.org/help) to run a variety of tests on every build.
+
+Each Travis build actually runs many sub-builds. Those sub-builds run once for each combination of:
+
+* programming language
+* Elasticsearch version
+
+As of this writing, this project has clients in two programming languages: [R](./r-pkg) and [Python](./py-pkg).
+
+The set of Elasticsearch versions this project tests against changes regularly as [new Elasticsearch versions are released](https://www.elastic.co/downloads/past-releases#elasticsearch). The strategy in this project is to test against the following Elasticsearch versions:
+
+> `uptasticsearch` is tested against Elasticsearch 1.0.0
+
+> `uptasticsearch` is tested against the most recent release in every major release stream from 1.x onwards
+
+> `uptasticsearch` is tested against the most recent maintenance release of the first and last minor releases on the prior stable version
+
+> `uptasticsearch` is tested against the most recent maintenance release on every minor release in the release stream of the current stable version
+
+> `uptasticsearch` may be tested against specific additional intermediate versions if bugs are found in the interaction between `uptasticsearch` and those versions
+
+So, for example, as of September 2019 that meant we tested against:
+
+* 1.0.0
+* 1.7.5
+* 2.4.2
+* 5.6.16
+* 6.0.1
+* 6.8.2
+* 7.0.1
+* 7.1.1
+* 7.2.1
+* 7.3.1
+
+You may notice that this strategy means that `uptasticsearch` is tested for backwards compatibility with Elasticsearch versions which have already reached [End-of-Life](https://www.elastic.co/support/eol). For example, support for Elasticsearch 1.7.x officially ended in January 2017. We test these old versions because we know of users whose companies still run those versions, and for whom Elasticsearch upgrades are prohibitively expensive. In general, upgrades across major versions pre-6.x [require a full cluster restart](https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html).
+
+### Running Tests Locally
When developing on this package, you may want to run Elasticsearch locally to speed up the testing cycle. We've provided some gross bash scripts at the root of this repo to help!
diff --git a/README.md b/README.md
index 15693e8..859ced3 100644
--- a/README.md
+++ b/README.md
@@ -6,20 +6,17 @@
## Introduction
-This project tackles the issue of getting data out of Elasticsearch and into a tabular format in R.
+`uptasticsearch` tackles the issue of getting data out of Elasticsearch and into a tabular format in R and Python. It should work for all versions of Elasticsearch from 1.0.0 onwards, but [is not regularly tested against all of them](./CONTRIBUTING.md#travis). If you run into a problem, please [open an issue](https://github.com/uptake/uptasticsearch/issues).
# Table of contents
-1. [How it Works](#howitworks)
-2. [Installation](#installation)
- 1. [R](#rinstallation)
- 2. [Python](#pythoninstallation)
-3. [Usage Examples](#examples)
- 1. [Get a Batch of Documents](#example1)
- 2. [Aggregation Results](#example2)
-4. [Next Steps](#nextsteps)
- 1. [Auth Support](#authsupport)
-5. [Running Tests Locally](#local-tests)
-6. [Regenerating the Documentation Site](#the-site)
+
+* [How it Works](#howitworks)
+* [Installation](#installation)
+ * [R](#rinstallation)
+ * [Python](#pythoninstallation)
+* [Usage Examples](#examples)
+ * [Get a Batch of Documents](#example1)
+ * [Aggregation Results](#example2)
## How it Works
@@ -32,13 +29,19 @@ The core functionality of this package is the `es_search` function. This returns
Releases of this package can be installed from CRAN:
```
-install.packages('uptasticsearch')
+install.packages(
+ 'uptasticsearch'
+ , repos = "http://cran.rstudio.com"
+)
```
To use the development version of the package, which has the newest changes, you can install directly from GitHub
```
-devtools::install_github("uptake/uptasticsearch", subdir = "r-pkg")
+devtools::install_github(
+ "uptake/uptasticsearch"
+ , subdir = "r-pkg"
+)
```
### Python
diff --git a/setup_local.sh b/setup_local.sh
index d22be8f..1fba2b1 100755
--- a/setup_local.sh
+++ b/setup_local.sh
@@ -63,7 +63,7 @@ case "${MAJOR_VERSION}" in
;;
5.6) docker run -d -p 9200:9200 \
-e "xpack.security.enabled=false" \
- docker.elastic.co/elasticsearch/elasticsearch:5.6.9
+ docker.elastic.co/elasticsearch/elasticsearch:5.6.16
MAPPING_FILE=$(pwd)/test_data/es5_shakespeare_mapping.json
;;
6.0) docker run -d -p 9200:9200 \
@@ -84,6 +84,12 @@ case "${MAJOR_VERSION}" in
docker.elastic.co/elasticsearch/elasticsearch:6.2.4
MAPPING_FILE=$(pwd)/test_data/es6_shakespeare_mapping.json
;;
+6.8) docker run -d -p 9200:9200 \
+ -e "discovery.type=single-node" \
+ -e "xpack.security.enabled=false" \
+ docker.elastic.co/elasticsearch/elasticsearch:6.8.2
+ MAPPING_FILE=$(pwd)/test_data/es6_shakespeare_mapping.json
+ ;;
7.3) docker run -d -p 9200:9200 \
-e "discovery.type=single-node" \
-e "xpack.security.enabled=false" \