Skip to content

Instantly share code, notes, and snippets.

@EmmanuelDemey

EmmanuelDemey/JavaScript

Last active February 2, 2024 14:41
Show Gist options
  • Save EmmanuelDemey/00437521dc21c1900266b22e2b2966aa to your computer and use it in GitHub Desktop.
Save EmmanuelDemey/00437521dc21c1900266b22e2b2966aa to your computer and use it in GitHub Desktop.
Download ZIP
Raw
JavaScript
import { Client } from "@elastic/elasticsearch"
import { readFileSync } from "fs"
(async function(){
const pipeLineName = "asn-demo-node-js";
const documents: any = [{
path: "./dummy.pdf",
name: "Manu"
}]
const client = new Client({
node: 'xxx',
auth: {
username: 'xx',
password: 'xx'
}
})
client.indices.putSettings({
index: 'asn-demo-nodejs-file',
settings: {
refresh_interval: '-1'
}
})
for(let d of documents){
console.log(d.path)
const body = readFileSync(d.path).toString("base64")
const documentToIngestToElasticsearch = {
body: body,
path: d.path,
name: d.name
}
const { result } = await client.index({
index: 'asn-demo-nodejs-file',
pipeline: pipeLineName,
body: documentToIngestToElasticsearch
})
}
client.indices.putSettings({
index: 'asn-demo-nodejs-file',
settings: {
refresh_interval: '1s'
}
})
client.indices.refresh({
index: 'asn-demo-nodejs-file'
})
const {hits } = await client.search({
index: "asn-demo-nodejs-file",
_source: {
excludes: ["attachment.content"]
},
body: {
query: {
match: {
"attachment.content": "file"
}
},
highlight: {
fields: {
"attachment.content": {}
}
}
},
})
console.log(hits.total)
console.log(hits.hits[0]._source)
console.log(hits.hits[0].highlight)
//const response = await client.cluster.health();
//console.log(response)
})()
Raw
TP Backup
PUT _slm/policy/asn-snapshots-ede
{
"schedule": "0 30 * * * ?",
"name": "<ede-asn-snap-{now/d}>",
"repository": "found-snapshots",
"config": {
"indices": "person-ede-*",
"include_global_state": true
},
"retention": {
"expire_after": "30d",
"min_count": 5,
"max_count": 50
}
}
PUT _slm/policy/asn-snapshots-ede/_execute
DELETE person-ede-v5
POST /_snapshot/found-snapshots/ede-asn-snap-2024.02.02-njzsgjv0s0sthcr8-ujj9a/_restore?wait_for_completion=true
{
"indices": "person-ede-v5"
}
GET /person-ede-v5/_search
Raw
tp pipeline
POST _ingest/pipeline/_simulate
{
"pipeline": {
"description" : "...",
"processors": [
{
"grok": {
"field": "balance",
"patterns": ["%{DEVISE:devise}%{GREEDYDATA:convertedBalance}"],
"pattern_definitions": {
"DEVISE": "\\$"
}
}
},
{
"set": {
"field": "devise",
"value": "dollars",
"if": "ctx.devise == '$'"
}
},
{
"gsub": {
"field": "convertedBalance",
"pattern": ",",
"replacement": ""
}
},
{
"convert": {
"field": "convertedBalance",
"type": "float"
}
},
{
"remove": {
"field": "balance"
}
}
]
},
"docs":[
{
"_source": {
"balance": "$3,758.05"
}
}
]
}
PUT _ingest/pipeline/asn-ede-pipeline
{
"description" : "...",
"processors": [
{
"grok": {
"field": "balance",
"patterns": ["%{DEVISE:devise}%{GREEDYDATA:convertedBalance}"],
"pattern_definitions": {
"DEVISE": "\\$"
}
}
},
{
"set": {
"field": "devise",
"value": "dollars",
"if": "ctx.devise == '$'"
}
},
{
"gsub": {
"field": "convertedBalance",
"pattern": ",",
"replacement": ""
}
},
{
"convert": {
"field": "convertedBalance",
"type": "float"
}
},
{
"remove": {
"field": "balance"
}
}
]
}
GET /_ingest/pipeline
GET _cat/templates?v
GET /_index_template/person-ede-template
PUT /_index_template/person-ede-template
{
"index_patterns": [
"person-ede-*"
],
"template": {
"settings": {
"index": {
"number_of_shards": "1",
"default_pipeline": "asn-ede-pipeline"
}
},
"mappings": {
"properties": {
"address": {
"type": "text"
},
"gender": {
"type": "keyword"
},
"about": {
"type": "text"
},
"registered": {
"format": [
"yyyy-MM-dd"
],
"type": "date"
},
"isActive": {
"type": "boolean"
},
"friends": {
"properties": {
"name": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"id": {
"type": "long"
}
}
},
"picture": {
"index": false,
"type": "text"
},
"tags": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"balance": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"eyeColor": {
"type": "keyword"
},
"phone": {
"type": "keyword"
},
"name": {
"type": "text"
},
"company": {
"type": "text"
},
"location": {
"type": "geo_point"
},
"age": {
"type": "long"
},
"email": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
}
}
},
"aliases": {
"mydata": {}
}
},
"composed_of": []
}
GET /_index_template/person-ede-template
GET _cat/indices
POST _reindex
{
"source": {
"index": "person-ede-v4"
},
"dest": {
"index": "person-ede-v5"
}
}
POST /person-ede-v5/_search
{
"query": {
"range": {
"convertedBalance": {
"gte": 1000,
"lte": 2000
}
}
}
}
Raw
tp securité
POST /_security/role/person-ede-ro
{
"indices": [
{
"names": [
"person-ede-*"
],
"privileges": [
"read"
],
"field_security": {
"grant": [
"*"
],
"except": [
"age"
]
},
"query": {
"term": {
"gender": {
"value": "female"
}
}
}
}
]
}
POST /_security/user/asn-manu
{
"password" : "userpassword",
"full_name" : "ASN Manu",
"email" : "john.doe@anony.mous",
"roles" : [ "person-ede-ro" ]
}
Raw
tp1
POST /person-ede-v1/_doc
{
"isActive": true,
"balance": "$3,758.05",
"picture": "http://placehold.it/32x32",
"age": 26,
"eyeColor": "green",
"name": "Alexandra Frost",
"gender": "female",
"company": "GLUKGLUK",
"email": "alexandrafrost@glukgluk.com",
"phone": "+1 (918) 547-3593",
"address": "921 Emerson Place, Norfolk, Georgia, 6723",
"about": "Eiusmod nostrud eiusmod ut voluptate do do tempor voluptate. Aliqua voluptate veniam reprehenderit sunt est laborum occaecat laboris eiusmod elit cillum ut laboris. Aliqua quis voluptate consectetur tempor cupidatat elit qui sint ipsum. Nisi ipsum minim culpa dolor id sint Lorem aliquip pariatur voluptate reprehenderit consequat est ullamco. Pariatur deserunt consequat esse anim nostrud ut laboris enim elit.\r\n",
"registered": "2022-09-02",
"location": {
"lat": 75.810554,
"lon": 169.492938
},
"tags": [
"minim",
"tempor",
"dolor",
"commodo",
"excepteur",
"sint",
"quis"
],
"friends": [
{
"id": 0,
"name": "Harrison Wong"
},
{
"id": 1,
"name": "Chavez Benton"
},
{
"id": 2,
"name": "Aimee Horne"
}
]
}
GET /person-ede-v1/_doc/I_gwX40BZ4wdy-XFNee7
POST /person-ede-v1/_update/I_gwX40BZ4wdy-XFNee7/
{
"doc": {
"age": 15
}
}
PUT /person-ede-v1/_doc/I_gwX40BZ4wdy-XFNee7
{
"isActive": true,
"balance": "$3,758.05",
"picture": "http://placehold.it/32x32",
"age": 26,
"eyeColor": "green",
"name": "Alexandra Frost",
"gender": "female",
"company": "GLUKGLUK",
"email": "alexandrafrost@glukgluk.com",
"phone": "+1 (918) 547-3593",
"address": "921 Emerson Place, Norfolk, Georgia, 6723",
"about": "Eiusmod nostrud eiusmod ut voluptate do do tempor voluptate. Aliqua voluptate veniam reprehenderit sunt est laborum occaecat laboris eiusmod elit cillum ut laboris. Aliqua quis voluptate consectetur tempor cupidatat elit qui sint ipsum. Nisi ipsum minim culpa dolor id sint Lorem aliquip pariatur voluptate reprehenderit consequat est ullamco. Pariatur deserunt consequat esse anim nostrud ut laboris enim elit.\r\n",
"registered": "2022-09-02",
"location": {
"lat": 75.810554,
"lon": 169.492938
},
"tags": [
"minim",
"tempor",
"dolor",
"commodo",
"excepteur",
"sint",
"quis"
],
"friends": [
]
}
DELETE /person-ede-v1/_doc/I_gwX40BZ4wdy-XFNee7
POST _bulk
{ "index": {"_index": "person-ede-v3"}}
{"isActive":true,"balance":"$3,758.05","picture":"http://placehold.it/32x32","age":26,"eyeColor":"green","name":"Alexandra Frost","gender":"female","company":"GLUKGLUK","email":"alexandrafrost@glukgluk.com","phone":"+1 (918) 547-3593","address":"921 Emerson Place, Norfolk, Georgia, 6723","about":"Eiusmod nostrud eiusmod ut voluptate do do tempor voluptate. Aliqua voluptate veniam reprehenderit sunt est laborum occaecat laboris eiusmod elit cillum ut laboris. Aliqua quis voluptate consectetur tempor cupidatat elit qui sint ipsum. Nisi ipsum minim culpa dolor id sint Lorem aliquip pariatur voluptate reprehenderit consequat est ullamco. Pariatur deserunt consequat esse anim nostrud ut laboris enim elit.\r\n","registered":"2022-09-02","location":{"lat":75.810554,"lon":169.492938},"tags":["minim","tempor","dolor","commodo","excepteur","sint","quis"],"friends":[{"id":0,"name":"Harrison Wong"},{"id":1,"name":"Chavez Benton"},{"id":2,"name":"Aimee Horne"}]}
{ "index": {"_index": "person-ede-v3"}}
{"isActive":false,"balance":"$2,614.81","picture":"http://placehold.it/32x32","age":30,"eyeColor":"brown","name":"Holland Cherry","gender":"male","company":"EPLODE","email":"hollandcherry@eplode.com","phone":"+1 (982) 523-3980","address":"127 Knickerbocker Avenue, Clara, Maryland, 9340","about":"Anim enim nisi occaecat veniam culpa elit aliqua cillum quis eu. Mollit ipsum ullamco officia sunt sunt consectetur irure aliquip. Qui ipsum eu irure eiusmod ut nostrud ad anim reprehenderit sit eiusmod est consectetur aute. Occaecat enim Lorem pariatur id consectetur consectetur ea nostrud ut sunt.\r\n","registered":"2019-11-18","location":{"lat":51.142573,"lon":78.029858},"tags":["deserunt","proident","ea","nulla","veniam","incididunt","ea"],"friends":[{"id":0,"name":"Leona Pickett"},{"id":1,"name":"Britney Mccall"},{"id":2,"name":"Estella Sutton"}]}
{ "index": {"_index": "person-ede-v3"}}
{"isActive":true,"balance":"$2,635.57","picture":"http://placehold.it/32x32","age":23,"eyeColor":"brown","name":"Patrice Carroll","gender":"female","company":"PARAGONIA","email":"patricecarroll@paragonia.com","phone":"+1 (900) 552-2158","address":"484 Front Street, Dixie, Guam, 8584","about":"Non officia qui id magna ea sit ipsum ea ipsum sit adipisicing. Eiusmod non consequat laborum tempor excepteur ea laboris excepteur est ullamco velit sint laborum cupidatat. Eiusmod est et ex occaecat do pariatur eiusmod deserunt laborum irure tempor ut. Commodo do ad officia sit ex qui minim consectetur. Officia eiusmod ullamco fugiat id nostrud commodo.\r\n","registered":"2015-05-19","location":{"lat":60.915164,"lon":44.468252},"tags":["sint","laborum","enim","ea","nostrud","ullamco","anim"],"friends":[{"id":0,"name":"Trisha Porter"},{"id":1,"name":"Silvia Delacruz"},{"id":2,"name":"Rose Nash"}]}
POST /person-ede-v3/_search
POST /_aliases
{
"actions": [
{
"add": {
"index": "person-ede-v1",
"alias": "person-ede"
}
}
]
}
POST /person-ede/_search
POST /_aliases
{
"actions": [
{
"remove": {
"index": "person-ede-v1",
"alias": "person-ede"
}
},
{
"add": {
"index": "person-ede-v3",
"alias": "person-ede"
}
}
]
}
POST /person-ede/_search
POST /_aliases
{
"actions": [
{
"add": {
"index": "person-ede-v3",
"alias": "person-ede-male",
"filter": {
"term": {
"gender": "male"
}
}
}
},
{
"add": {
"index": "person-ede-v3",
"alias": "person-ede-female",
"filter": {
"term": {
"gender": "female"
}
}
}
}
]
}
POST /person-ede/_search
POST /person-ede-female/_search
POST /person-ede-male/_search
POST /person-ede/_search?q=maryland
GET /_cluster/health
GET person-ede-v3
Raw
tp2
GET person-ede-v3/_mapping
PUT _index_template/person-ede-template
{
"index_patterns": [
"person-ede-*"
],
"template": {
"settings": {
"number_of_shards": 1
},
"mappings": {
"properties": {
"about": {
"type": "text"
},
"address": {
"type": "text"
},
"age": {
"type": "long"
},
"balance": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"company": {
"type": "text"
},
"email": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"eyeColor": {
"type": "keyword"
},
"friends": {
"properties": {
"id": {
"type": "long"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"gender": {
"type": "keyword"
},
"isActive": {
"type": "boolean"
},
"location": {
"type": "geo_point"
},
"name": {
"type": "text"
},
"phone": {
"type": "keyword"
},
"picture": {
"type": "text",
"index": false
},
"registered": {
"type": "date",
"format": [
"yyyy-MM-dd"
]
},
"tags": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"aliases": {
"mydata": {}
}
}
}
GET /_cat/indices?v
POST _reindex
{
"source": {
"index": "person-ede-v3"
},
"dest": {
"index": "person-ede-v4"
}
}
GET person-ede-v4
GET /_cat/indices?v
GET person-ede/_search
POST _aliases
{
"actions": [
{
"remove": {
"index": "person-ede-v3",
"alias": "person-ede"
}
},
{
"add": {
"index": "person-ede-v4",
"alias": "person-ede"
}
}
]
}
GET person-ede/_search
Raw
tp3
POST person-ede/_search
{
"query": {
"match": {
"gender": "female"
}
}
}
POST person-ede/_search
{
"query": {
"term": {
"gender": {
"value": "female"
}
}
}
}
POST person-ede/_search
{
"query": {
"range": {
"age": {
"gt": 20
}
}
}
}
POST person-ede/_search
{
"_source": false,
"query": {
"bool": {
"filter": [
{
"range": {
"age": {
"gt": 20
}
}
},
{
"term": {
"gender": "male"
}
}
]
}
}
}
POST person-ede/_search
{
"_source": {
"includes": ["balance"]
},
"query": {
"bool": {
"filter": [
{
"range": {
"age": {
"gt": 20
}
}
},
{
"range": {
"balance": {
"gte": 1000,
"lte": 2000
}
}
}
]
}
}
}
POST /person-ede/_search
{
"query": {
"geo_distance": {
"distance": "10km",
"location": {
"lat": 75.810554,
"lon": 169.492938
}
}
}
}
Raw
tp4
POST person-ede/_search
{
"size": 0,
"aggs": {
"avg_age": {
"avg": {
"field": "age"
}
},
"par_gender": {
"terms": {
"field": "gender",
"size": 2
},
"aggs": {
"per_eyeColor": {
"terms": {
"field": "eyeColor",
"size": 10
}
},
"per_registered": {
"date_histogram": {
"field": "registered",
"calendar_interval": "year"
}
}
}
},
"per_registered": {
"date_histogram": {
"field": "registered",
"calendar_interval": "year"
},
"aggs": {
"per_gender": {
"terms": {
"field": "gender"
}
},
"avg_age": {
"avg": {
"field": "age"
}
},
"more-30_bucket_filter": {
"bucket_selector": {
"buckets_path": {
"avgAge": "avg_age"
},
"script": "params.avgAge > 28"
}
},
"oldPerson": {
"top_hits": {
"size": 1,
"sort": ["age"],
"_source": {
"includes": ["name", "age"]
}
}
}
}
},
"max_avg_age_per_year": {
"max_bucket": {
"buckets_path": "per_registered>avg_age"
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment