Count number of docs indexed in certain interval (e.g., last 15 min)
GET /logstash-<DATE>/_search?filter_path=hits.total
{
"query": {
"bool": {
"filter": [
{
"range": {
FROM docker.elastic.co/elasticsearch/elasticsearch:5.6.4 | |
USER root | |
RUN sed -i '/^exclude/d' /etc/yum.conf && yum update -y && yum install -y java-1.8.0-openjdk-devel | |
USER elasticsearch |
Count number of docs indexed in certain interval (e.g., last 15 min)
GET /logstash-<DATE>/_search?filter_path=hits.total
{
"query": {
"bool": {
"filter": [
{
"range": {
PUT test | |
{ | |
"settings": { | |
"analysis": { | |
"analyzer": { | |
"ReverseIt": { | |
"type": "custom", | |
"tokenizer": "keyword", | |
"filter": [ | |
"reverse" |
Reindexing with Logstash can be done with the following Logstash configuration:
input {
# We read from the "old" index
elasticsearch {
hosts => ["http://<host>:<port>"]
index => "<old_index>"
size => 500
scroll => "5m"
worker_processes 1; | |
error_log /var/log/nginx/error.log; | |
events { | |
worker_connections 1024; | |
} | |
http { | |
log_format es '$remote_addr - $remote_user [$time_local] ' |
#!/usr/bin/env bash | |
ES_VERSION="1.7.2" | |
ES_URL="https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-$ES_VERSION.tar.gz" | |
curl -s -L -o - $ES_URL | tar -xz -C /opt \ | |
&& ln -s /opt/elasticsearch-$ES_VERSION /opt/elasticsearch \ | |
&& mkdir /opt/elasticsearch/{data,logs,plugins} | |
chown -R vagrant:vagrant /opt/elasticsearch-${ES_VERSION} |
"trigger": { | |
"schedule": { | |
"interval": "10m" | |
} | |
}, | |
"input": { | |
"search": { | |
"request": { | |
"search_type": "count", | |
"indices": [ |
This is just some quick notes for importing the Backblaze Hard Drive Test Data into Elasticsearch. Of the archives that Backblaze has provided, you only need to download the 2013 and 2014 data-sets and unpack them to a temporary location.
After you've unpacked the data, you'll need to convert the CSV to JSON. I use the csvjson
tool from Csvkit for this. In the directory containing the CSV files, run this bash loop:
for csv in *.csv; do name=$(basename $csv .csv); csvjson "${name}.csv" > "${name}.json"; done
#!/usr/bin/env bash | |
mail_to=root | |
ignorefile=/etc/rpm-dispatch-conf.ignore | |
newfiles=$(find / -noleaf -ignore_readdir_race -xdev -name \*.rpmsave -or -name \*.rpmnew 2>/dev/null) | |
for f in $newfiles; do | |
newfile=$f | |
oldfile=$f |
LUSTRE_OBJECT %{WORD}(-%{WORD}){1,3}
LUSTRE_LNET %{IP}@%{WORD}
LUSTRE_SOURCECODE (%{USERNAME}.c:%{INT})
LUSTRE_ERRCODE rc (=)? (%{INT:error_code}|%{INT}/%{INT})
LUSTRE_LOGPREFIX1 (Lustre|LustreError|LNetError): (%{WORD}-%{WORD}: )?%{LUSTRE_OBJECT:lustre_object}:
LUSTRE_LOGPREFIX2 (Lustre|LustreError|LNet|LNetError):%{SPACE}?%{WORD}:%{WORD}:\(%{LUSTRE_SOURCECODE:lustre_source}:%{USERNAME:lustre_function}\(\)\)
LUSTRE_LOGPREFIX3 (Lustre|LustreError|LNet|LNetError):