Skip to content

Instantly share code, notes, and snippets.

@CesarCapillas
Last active April 23, 2024 21:14
Show Gist options
  • Save CesarCapillas/a796c0e7cba10ac02213c7f3485d6e90 to your computer and use it in GitHub Desktop.
Save CesarCapillas/a796c0e7cba10ac02213c7f3485d6e90 to your computer and use it in GitHub Desktop.
SOLR bash recipes for creating, deleting or truncating collections, monitoring and searching.
#!/bin/bash
COLLECTION=${2:-zylk}
SERVER=${3:-localhost}
PORT=${4:-8983}
if [ -z "$1" ]; then
# Usage
echo 'Usage: add-by-id.sh <id> [<collection> <solr-server=localhost> <port=8383>]'
else
curl -X POST "http://${SERVER}:${PORT}/solr/${COLLECTION}/update?commit=true" -H "Content-Type: text/xml" --data-binary "<add><doc><field name='id'>$1</field><field name='url'>$1</field></doc></add>"
fi
SERVER=${1:-localhost}
PORT=${2:-8983}
NUMSERVERS=${3:-1}
if [ "$PORT" = "443" ]; then
PROTOCOL="https"
else
PROTOCOL="http"
fi
ENDPOINT="$PROTOCOL://${SERVER}:${PORT}/solr/admin/collections?action=clusterstatus&wt=json"
if [[ "$1" == "" ]]; then
echo "Usage:"
echo " check_solr.sh <SOLRSERVER=localhost> [<PORT=8983> <NUMSERVERS=1>]"
exit
fi
CURL=`curl --silent -X GET ${ENDPOINT}`
CHCK=`echo $CURL | grep "live_nodes"`
if [[ "$CHCK" == "" ]]; then
CHECK="Failed"
else
CHECK="OK"
#SOLR_RES=`echo $CURL | jq ".cluster.live_nodes" | tr -d '\r\n'`
SOLR_RES=`echo $CURL | jshon -e cluster | jshon -e live_nodes | tr -d '\r\n'`
# tricky
#SOLR_RES=`echo $CURL | jq ".cluster.live_nodes" | wc -w`
SOLR_NUM=`echo $CURL | jshon -e cluster | jshon -e live_nodes | wc -w`
SOLR_VAR=`expr $SOLR_NUM - 2`
fi
if [[ "$CHECK" == "OK" ]]; then
if (($SOLR_VAR < $NUMSERVERS));then
echo "CRITICAL: SOLR ($SOLR_VAR live nodes) = $SOLR_RES (<$NUMSERVERS)"
exit 2
fi
echo "INFO: SOLR ($SOLR_VAR live nodes) = $SOLR_RES "
exit 0
elif [[ "$CHECK" == "Failed" ]]; then
echo "CRITICAL: ${SERVER}"
exit 2
else
echo "Check failed."
exit 3
fi
#!/bin/bash
ALIAS=${1:-zylk-alias}
COLLECTION=${3:-zylk}
SERVER=${3:-localhost}
PORT=${4:-8983}
if [ -z "$1" ]; then
# Usage
echo 'Usage: create-alias.sh <alias-name> <collection-name-list> [<solr-server=localhost> <port=8983>]'
#curl -s "http://${SERVER}:${PORT}/solr/admin/collections?action=LISTALIASES" | xmllint --format -
else
echo "Creating SOLR alias collection $ALIAS for ${COLLECTION}"
curl -s "http://${SERVER}:${PORT}/solr/admin/collections?action=CREATEALIAS&name=${ALIAS}&collections=${COLLECTION}" | tidy -q -xml -i -
fi
curl -s "http://${SERVER}:${PORT}/solr/admin/collections?action=LISTALIASES" | tidy -q -xml -i -
#!/bin/bash
COLLECTION=${1:-zylk}
MYDATE=`/bin/date +%Y%m%d`
MYBACK="${COLLECTION}-${MYDATE}-backup"
BACKUPNAME=${2:-${MYBACK}}
BACKUPDIR=${3:-/opt/solr6/backups}
SERVER=${4:-localhost}
PORT=${5:-8983}
if [ -z "$1" ]; then
# Usage
echo 'Usage: create-backup.sh <collection-name> <backupname> <local-path> [<solr-server=localhost> <port=8983>]'
else
echo "Creating SOLR backup collection for ${COLLECTION}"
curl -s "http://${SERVER}:${PORT}/solr/admin/collections?action=BACKUP&name=${BACKUPNAME}&collection=${COLLECTION}&location=${BACKUPDIR}"
fi
#!/bin/bash
#./bin/solr create -c zylk -shards 1 -replicationFactor 2 -p 8983 -d server/solr/configsets/nutchconfig-base
SOLRHOME=/opt/solr6/solr-6.6.0
CONFIGSET=server/solr/configsets/nutchconfig-base
COLLECTION=${1:-zylk}
PORT=${2:-8983}
if [ -z "$1" ]; then
# Usage
echo 'Usage: create-collection.sh <collection-name> [ <port=8080> ]'
else
echo "Creating SOLR collection ${COLLECTION}"
(cd ${SOLRHOME} && ./bin/solr create -c ${COLLECTION} -shards 1 -replicationFactor 2 -p ${PORT} -d ${CONFIGSET})
fi
#!/bin/bash
COLLECTION=${2:-zylk}
SERVER=${3:-localhost}
PORT=${4:-8983}
if [ -z "$1" ]; then
# Usage
echo 'Usage: delete-by-id.sh <id> [<collection> <solr-server=localhost> <port=8983>]'
else
curl -X POST "http://${SERVER}:${PORT}/solr/${COLLECTION}/update?commit=true" -H "Content-Type: text/xml" --data-binary "<delete><id>$1</id></delete>"
fi
#!/bin/bash
SOLR_DIR=/opt/solr6/solr-6.6.0/
COLLECTION=${2:-gettingstarted}
PORT=${3:-8983}
if [ -z "$1" ]; then
# Usage
echo 'Usage: delete-by-id2.sh <id> [<collection>]'
else
cd SOLR_DIR
./bin/post -p $PORT -c $COLLECTION -d "<delete><id>$1</id></delete>"
fi
#!/bin/bash
COLLECTION=${2:-gettingstarted}
SERVER=${3:-localhost}
PORT=${4:-8983}
if [ -z "$1" ]; then
# Usage
echo 'Usage: delete-by-id3.sh <id> [<collection> <solr-server=localhost> <port=8383>]'
else
curl -X POST "http://${SERVER}:${PORT}/solr/${COLLECTION}/update?commit=true" -H "Content-Type: application/json" --data-binary "{\"delete\": {\"id\":\"$1\"}}"
fi
#! /bin/bash
###
### Script for do SOLR querys from command line via REST API
###
urlencode() {
# urlencode <string>
old_lc_collate=$LC_COLLATE
LC_COLLATE=C
local length="${#1}"
for (( i = 0; i < length; i++ )); do
local c="${1:i:1}"
case $c in
[a-zA-Z0-9.~_-]) printf "$c" ;;
*) printf '%%%02X' "'$c" ;;
esac
done
LC_COLLATE=$old_lc_collate
}
urldecode() {
# urldecode <string>
local url_encoded="${1//+/ }"
printf '%b' "${url_encoded//%/\\x}"
}
# Usage functions
usage() { echo "Usage: $0 [-t <term>] [-c <collection-name>] [-l <field-list-fl>] [-q <field-query-fq>] [-r <query-type-qt>] [-h <solr-server>] [-p <solr-port>]" 1>&2; exit 1; }
# Command line options
while getopts "t:c:l:q:r:h:p" o; do
case "${o}" in
t)
TERM=${OPTARG}
;;
l)
FL=${OPTARG}
;;
q)
FQ=${OPTARG}
;;
r)
QT=${OPTARG}
;;
c)
COLLECTION=${OPTARG}
;;
h)
SERVER=${OPTARG}
;;
p)
PORT=${OPTARG}
;;
\?)
echo "Invalid Option: -$OPTARG" 1>&2
exit 1
;;
*)
usage
;;
esac
done
shift $((OPTIND-1))
# Needs at least COLLECTION and TERM as parameters
if [ -z "${COLLECTION}" ] || [ -z "${TERM}" ] ; then
usage
fi
# Exports SERVER,PORT
#source ./exportENVARS.sh
# Default parameters
# Encode parameters (as blank spaces)
TERM=`urlencode "$TERM"`
#COLLECTION=${COLLECTION:-zylk}
FL=${FL:-"url,title,lang,score"}
FL=`urlencode $FL`
FQ=${FQ:-"*"}
QT=${QT:-select}
SERVER=${SERVER:-localhost}
PORT=${PORT:-8983}
# Main
NUMROWS=`curl -s "http://${SERVER}:${PORT}/solr/${COLLECTION}/${QT}?fq=$FQ&indent=on&q=${TERM}&wt=json" | jq '.response.numFound'`
#echo "Number of results for search: $1 --> $NUMROWS"
ROWS=10000
COUNT=0
START=0
while [ $START -lt $NUMROWS ]; do
# Only 10 results finally
#curl -s "http://${SERVER}:${PORT}/solr/${COLLECTION}/${QT}?fl=$FL&indent=on&q=$TERM&rows=$ROWS&start=$START&wt=json" | jq '.response.docs[]'
echo "curl -s \"http://${SERVER}:${PORT}/solr/${COLLECTION}/${QT}?fq=$FQ&fl=$FL&indent=on&q=$TERM&rows=10&start=$START&wt=json\""
curl -s "http://${SERVER}:${PORT}/solr/${COLLECTION}/${QT}?fq=$FQ&fl=$FL&indent=on&q=$TERM&rows=10&start=$START&wt=json" | jq '.response.docs[]'
COUNT=$((COUNT+1))
START=$((ROWS*COUNT))
done
echo "Number of results for search: $1 --> $NUMROWS"
#! /bin/bash
###
### Script for getting indexed urls from SOLR
###
# Usage functions
usage() { echo "Usage: $0 [-c <collection-name>] [-h <solr-host>] [-p <solr-port>]" 1>&2; exit 1; }
# Command line options
while getopts "c:h:p:" o; do
case "${o}" in
c)
COLLECTION=${OPTARG}
;;
h)
SERVER=${OPTARG}
;;
p)
PORT=${OPTARG}
;;
\?)
echo "Invalid Option: -$OPTARG" 1>&2
exit 1
;;
*)
usage
;;
esac
done
shift $((OPTIND-1))
# Needs at least COLLECTION as parameter
if [ -z "${COLLECTION}" ]; then
usage
fi
# Exports SERVER, PORT ?
#source ./exportENVARS.sh
# Default parameters
#COLLECTION=${COLLECTION:-garbiker}
SERVER=${SERVER:-localhost}
PORT=${PORT:-8983}
# Main
NUMROWS=`curl -s "http://${SERVER}:${PORT}/solr/${COLLECTION}/select?indent=on&q=*:*&wt=json" | jq '.response.numFound'`
echo "Number of indexed urls in $1 : $NUMROWS"
ROWS=10000
COUNT=0
START=0
while [ $START -lt $NUMROWS ]; do
curl -s "http://${SERVER}:${PORT}/solr/${COLLECTION}/select?fl=url&indent=on&q=*:*&rows=$ROWS&start=$START&sort=url%20desc&wt=json" | jq '.response.docs[].url'
COUNT=$((COUNT+1))
START=$((ROWS*COUNT))
done
#! /bin/bash
###
### Script for reloading SOLR collection via REST API
###
# Usage functions
usage() { echo "Usage: $0 [-c <collection-name>] [-h <solr-host>] [-p <solr-port>]" 1>&2; exit 1; }
# Command line options
while getopts "c:h:p:" o; do
case "${o}" in
c)
COLLECTION=${OPTARG}
;;
h)
SERVER=${OPTARG}
;;
p)
PORT=${OPTARG}
;;
\?)
echo "Invalid Option: -$OPTARG" 1>&2
exit 1
;;
*)
usage
;;
esac
done
shift $((OPTIND-1))
# Needs at least COLLECTION as parameter
if [ -z "${COLLECTION}" ]; then
usage
fi
# Exports SERVER, PORT ?
#source ./exportENVARS.sh
# Default parameters
#COLLECTION=${COLLECTION:-turismoberria}
SERVER=${SERVER:-localhost}
PORT=${PORT:-8983}
# Reload
curl "http://${SERVER}:${PORT}/solr/admin/collections?action=RELOAD&name=${COLLECTION}"
#!/bin/bash
ALIAS=${1:-zylk-alias}
SERVER=${2:-localhost}
PORT=${3:-8983}
if [ -z "$1" ]; then
# Usage
echo 'Usage: remove-alias.sh <alias-name> [<solr-server=localhost> <port=8983>]'
#curl -s "http://${SERVER}:${PORT}/solr/admin/collections?action=LISTALIASES" | xmllint --format -
else
echo "Removing SOLR alias collection ${ALIAS}"
curl -s "http://${SERVER}:${PORT}/solr/admin/collections?action=DELETEALIAS&name=${ALIAS}" | tidy -q -xml -i -
fi
curl -s "http://${SERVER}:${PORT}/solr/admin/collections?action=LISTALIASES" | tidy -q -xml -i -
#!/bin/bash
#./bin/solr delete -c zylk
SOLRHOME=/opt/solr6/solr-6.6.0
COLLECTION=${1:-zylk}
if [ -z "$1" ]; then
# Usage
echo 'Usage: remove-collection.sh <collection-name>'
else
echo "Removing SOLR collection ${COLLECTION}"
(cd ${SOLRHOME} && ./bin/solr delete -c ${COLLECTION})
fi
#!/bin/bash
COLLECTION=${1:-zylk-backup}
BACKUPNAME=${2:-zylk-backup-name}
BACKUPDIR=${3:-/opt/solr6/backups}
SERVER=${4:-localhost}
PORT=${5:-8983}
if [ -z "$2" ]; then
# Usage
echo 'Usage: restore-backup.sh <collection-name> <backupname> <local-path> [<solr-server=localhost> <port=8983>]'
else
echo "Restoring SOLR backup collection for ${COLLECTION}"
curl -s "http://${SERVER}:${PORT}/solr/admin/collections?action=RESTORE&name=${BACKUPNAME}&collection=${COLLECTION}&location=${BACKUPDIR}"
fi
#! /bin/bash
###
### Script for truncating SOLR collection via REST API
###
# Usage functions
usage() { echo "Usage: $0 [-c <collection-name>] [-h <solr-host>] [-p <solr-port>]" 1>&2; exit 1; }
# Command line options
while getopts "c:h:p:" o; do
case "${o}" in
c)
COLLECTION=${OPTARG}
;;
h)
SERVER=${OPTARG}
;;
p)
PORT=${OPTARG}
;;
\?)
echo "Invalid Option: -$OPTARG" 1>&2
exit 1
;;
*)
usage
;;
esac
done
shift $((OPTIND-1))
# Needs at least COLLECTION as parameter
if [ -z "${COLLECTION}" ]; then
usage
fi
# Exports SERVER, PORT ?
#source ./exportENVARS.sh
# Default parameters
#COLLECTION=${COLLECTION:-turismoberria}
SERVER=${SERVER:-localhost}
PORT=${PORT:-8983}
# Truncate
curl -X POST http://${SERVER}:${PORT}/solr/${COLLECTION}/update?commit=true -H "Content-Type: text/xml" --data-binary '<delete><query>*:*</query></delete>'
# Reload
curl "http://${SERVER}:${PORT}/solr/admin/collections?action=RELOAD&name=${COLLECTION}"
#!/bin/bash
json=$(cat <<EOF
[
{
"id": "$1",
"url": "$1"
}
]
EOF
)
COLLECTION=${2:-zylk}
SERVER=${3:-localhost}
PORT=${4:-8983}
if [ -z "$1" ]; then
# Usage
echo 'Usage: update-json.sh <id> [<collection> <solr-server=localhost> <port=8983>]'
else
curl -X POST "http://${SERVER}:${PORT}/solr/${COLLECTION}/update?boost=0&commit=true" -H "Content-Type: application/json" --data-binary "$json"
fi
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment