Skip to content

Commit

Permalink
chore(deps): updated elasticsearch client
Browse files Browse the repository at this point in the history
  • Loading branch information
@tompahoward
tompahoward committed Jun 2, 2021
1 parent c0d7f14 commit 53395a4
Show file tree
Hide file tree
Showing 8 changed files with 387 additions and 409 deletions.
10 changes: 10 additions & 0 deletions TODO.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# To Do

- [ ] switch to waycharter
- [ ] switch to github actions
- [ ] auto deploy server to GCP
- [ ] setup automatic job to load gnaf
- [ ] fix code coverage
- [ ] HEAD handler
- [ ] Switch to GDA2020 - https://www.icsm.gov.au/gda2020/what-changing-and-why
- [ ] etags
210 changes: 112 additions & 98 deletions client/elasticsearch.js
View file
Original file line number Diff line number Diff line change
@@ -1,38 +1,39 @@
const waitPort = require('wait-port');
const elasticsearch = require('elasticsearch');
import debug from 'debug';
const logger = debug('api');
const error = debug('error');
const waitPort = require('wait-port')
const elasticsearch = require('@elastic/elasticsearch')
import debug from 'debug'
const logger = debug('api')
const error = debug('error')

const ES_INDEX_NAME = process.env.ES_INDEX_NAME || 'addressr';
export const ELASTIC_PORT = Number.parseInt(process.env.ELASTIC_PORT || '9200');
const ELASTIC_HOST = process.env.ELASTIC_HOST || '127.0.0.1';
const ELASTIC_USERNAME = process.env.ELASTIC_USERNAME || undefined;
const ELASTIC_PASSWORD = process.env.ELASTIC_PASSWORD || undefined;
const ELASTIC_PROTOCOL = process.env.ELASTIC_PROTOCOL || 'http';
const ES_INDEX_NAME = process.env.ES_INDEX_NAME || 'addressr'
export const ELASTIC_PORT = Number.parseInt(process.env.ELASTIC_PORT || '9200')
const ELASTIC_HOST = process.env.ELASTIC_HOST || '127.0.0.1'
const ELASTIC_USERNAME = process.env.ELASTIC_USERNAME || undefined
const ELASTIC_PASSWORD = process.env.ELASTIC_PASSWORD || undefined
const ELASTIC_PROTOCOL = process.env.ELASTIC_PROTOCOL || 'http'

export async function dropIndex(esClient) {
export async function dropIndex (esClient) {
// eslint-disable-next-line security/detect-non-literal-fs-filename
if (await esClient.indices.exists({ index: ES_INDEX_NAME })) {
await esClient.indices.delete({ index: ES_INDEX_NAME });
let exists = await esClient.indices.exists({ index: ES_INDEX_NAME })
if (exists.body) {
const deleteIndexResult = await esClient.indices.delete({
index: ES_INDEX_NAME
})
logger({ deleteIndexResult })
}
logger('checking if index exists');
logger('checking if index exists')
// eslint-disable-next-line security/detect-non-literal-fs-filename
const exists = await esClient.indices.exists({ index: ES_INDEX_NAME });
logger('index exists:', exists);
exists = await esClient.indices.exists({ index: ES_INDEX_NAME })
logger('index exists:', exists)
}

export async function initIndex(esClient, clear, synonyms) {
// eslint-disable-next-line security/detect-non-literal-fs-filename
if (await esClient.indices.exists({ index: ES_INDEX_NAME })) {
if (clear) {
await esClient.indices.delete({ index: ES_INDEX_NAME });
}
export async function initIndex (esClient, clear, synonyms) {
if (clear) {
await dropIndex(esClient)
}
logger('checking if index exists');
logger('checking if index exists')
// eslint-disable-next-line security/detect-non-literal-fs-filename
const exists = await esClient.indices.exists({ index: ES_INDEX_NAME });
logger('index exists:', exists);
const exists = await esClient.indices.exists({ index: ES_INDEX_NAME })
logger('index exists:', exists.body)
const indexBody = {
settings: {
index: {
Expand All @@ -41,13 +42,13 @@ export async function initIndex(esClient, clear, synonyms) {
my_synonym_filter: {
type: 'synonym',
lenient: true,
synonyms,
synonyms
},
comma_stripper: {
type: 'pattern_replace',
pattern: ',',
replacement: '',
},
replacement: ''
}
},
analyzer: {
// default: {
Expand All @@ -61,64 +62,73 @@ export async function initIndex(esClient, clear, synonyms) {
'asciifolding',
'my_synonym_filter',
'comma_stripper',
'trim',
],
},
'trim'
]
}
},
tokenizer: {
whitecomma: {
type: 'pattern',
pattern: '[\\W,]+',
lowercase: false,
},
},
},
},
lowercase: false
}
}
}
}
},
aliases: {},
mappings: {
properties: {
structured: {
type: 'object',
enabled: false,
enabled: false
},
sla: {
type: 'text',
analyzer: 'my_analyzer',
fielddata: true,
fielddata: true
},
ssla: {
type: 'text',
analyzer: 'my_analyzer',
analyzer: 'my_analyzer'
},
confidence: { type: 'integer' },
},
},
};
confidence: { type: 'integer' }
}
}
}

if (!exists) {
await esClient.indices.create({
if (!exists.body) {
logger(`creating index: ${ES_INDEX_NAME}`)
const indexCreateResult = await esClient.indices.create({
index: ES_INDEX_NAME,
body: indexBody,
});
body: indexBody
})
logger({ indexCreateResult })
} else {
// update the index
await esClient.indices.close({
index: ES_INDEX_NAME,
});
await esClient.indices.putSettings({
const indexCloseResult = await esClient.indices.close({
index: ES_INDEX_NAME
})
logger({ indexCloseResult })
const indexPutSettingsResult = await esClient.indices.putSettings({
index: ES_INDEX_NAME,
body: indexBody,
});
body: indexBody
})
logger({ indexPutSettingsResult })
// eslint-disable-next-line security/detect-non-literal-fs-filename
await esClient.indices.open({
index: ES_INDEX_NAME,
});
const indexOpenResult = await esClient.indices.open({
index: ES_INDEX_NAME
})
logger({ indexOpenResult })
}
await esClient.indices.get({ index: ES_INDEX_NAME, includeDefaults: true });
const indexGetResult = await esClient.indices.get({
index: ES_INDEX_NAME,
includeDefaults: true
})
logger({ indexGetResult })
}

export async function esConnect(
export async function esConnect (
esport = ELASTIC_PORT,
eshost = ELASTIC_HOST,
interval = 1000,
Expand All @@ -127,66 +137,70 @@ export async function esConnect(
// we keep trying to connect, no matter what
// eslint-disable-next-line no-constant-condition
while (true) {
console.log(`trying to reach elastic search on ${eshost}:${esport}...`);
console.log(`trying to reach elastic search on ${eshost}:${esport}...`)
try {
const open = await waitPort({
host: eshost,
port: esport,
interval,
timeout,
});
timeout
})
if (open) {
logger(`...${eshost}:${esport} is reachable`);
logger(`...${eshost}:${esport} is reachable`)

// eslint-disable-next-line no-constant-condition
while (true) {
try {
const esClientOptions = {
host: [
{
host: eshost,
...(ELASTIC_USERNAME &&
ELASTIC_PASSWORD && {
auth: `${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}`,
}),
protocol: ELASTIC_PROTOCOL,
port: esport,
},
],
log: 'info',
};
const esClient = new elasticsearch.Client(esClientOptions);
logger(
`connecting elastic search client on ${eshost}:${esport}...`
);
await esClient.ping({
requestTimeout: interval,
maxRetries: 0,
});
logger(`...connected to ${eshost}:${esport}`);
global.esClient = esClient;
return esClient;
node: `${ELASTIC_PROTOCOL}://${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}@${eshost}:${esport}`
// ...(ELASTIC_USERNAME &&
// ELASTIC_PASSWORD && {
// cloud: {
// username: ELASTIC_USERNAME,
// password: ELASTIC_PASSWORD
// }
// })

// host: [
// {
// host: eshost,
// ...(ELASTIC_USERNAME &&
// ELASTIC_PASSWORD && {
// auth: `${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}`,
// }),
// protocol: ELASTIC_PROTOCOL,
// port: esport,
// },
// ],
// log: 'info',
}
const esClient = new elasticsearch.Client(esClientOptions)
logger(`connecting elastic search client on ${eshost}:${esport}...`)
await esClient.ping()
logger(`...connected to ${eshost}:${esport}`)
global.esClient = esClient
return esClient
} catch (error_) {
error(
`An error occured while trying to connect the elastic search client on ${eshost}:${esport}`,
`An error occurred while trying to connect the elastic search client on ${eshost}:${esport}`,
error_
);
await new Promise((resolve) => {
setTimeout(() => resolve(), interval);
});
logger('retrying...');
)
await new Promise(resolve => {
setTimeout(() => resolve(), interval)
})
logger('retrying...')
}
}
}
} catch (error_) {
error(
`An error occured while waiting to reach elastic search on ${eshost}:${esport}`,
error_
);
await new Promise((resolve) => {
setTimeout(() => resolve(), interval);
});
logger('retrying...');
)
await new Promise(resolve => {
setTimeout(() => resolve(), interval)
})
logger('retrying...')
}
}
}
Loading

0 comments on commit 53395a4

Please sign in to comment.