Commandlines and Tricks

Your sync script between two servers, skipping existing files

#!/usr/bin/env bash

DESTHOST=destination
LOGDIR=rsync.log
TS=$(date +%F_%H-%M-%S)

mkdir -p $LOGDIR/$TS

echo "[Info] Sync-ing huggingface cache..."
nohup rsync -varP ~/.cache/huggingface/ ${DESTHOST}:~/.cache/huggingface > $LOGDIR/$TS/log.hf.cache 2>&1

echo "[Info] Sync-ing miniconda ..."
nohup rsync -varP ~/miniconda3/ ${DESTHOST}:~/miniconda3 > $LOGDIR/$TS/log.miniconda3 2>&1

echo "[Info]: Sync Done"

Disable Kernel update

https://vojcik.net/hot-to-disable-automatic-kernel-updates-ubuntu/

Download from public/open files from google drive

https://stackoverflow.com/questions/25010369/wget-curl-large-file-from-google-drive

pip install gdown
gdown https://drive.google.com/uc?id=<file_id>  # for files
gdown --id <file_id>                            # alternative format
gdown --folder https://drive.google.com/drive/folders/<file_id>  # for folders
gdown --folder --id <file_id>                   # this format works for folders too

Alternative:

1. rclone config is super useful for download gdrive, setup is so easy that it provides a link to get the token for setup
2. publicly shared file of gdrive may not be available due to cap on requests (perhaps to avoid ddos), workaround is to make a copy to one's gdrive and rclone sync
3. rclone sync works in parallel for each files, for one of previous experience, 3 files of 30GB each is complete within 30mins.
4. md5sum is also easy and useful to ensure bit correctness of file

Sample alternatives for ninja

sudo update-alternatives --install /usr/bin/ninja ninja /home/vchua/miniconda3/envs/AutoQPrecInit/bin/ninja 1 --force

Run specific parameterized pytest

pytest test_compressed_graph.py::test_synthetic_model_quantization[embedding]

Crawl folder on linux server with wget

https://stackoverflow.com/questions/273743/using-wget-to-recursively-fetch-a-directory-with-arbitrary-files-in-it

Update forked Repo

https://medium.com/@topspinj/how-to-git-rebase-into-a-forked-repo-c9f05e821c8a

Mount a network folder

sudo apt-get install cifs-utils
sudo mkdir -p /mnt/windows/share-name
sudo mount -t cifs //<host.ip/domain>/External/share /mnt/windows/share-name -o username=xxxx,domain=xx.xx.com
# domain can be dropped in some cases

Install Intel GPU drivers

https://dgpu-docs.intel.com/driver/installation.html
use the other proxy setup for intel-graphics repo
any new user must be added to group render
`xpu-smi discovery` will list gpu
OV gpu visibility - follow sd-perf

Install Nvidia drivers

# To reflect pci device id with human-readable label
sudo update-pciids

# To get latest drivers in Ubuntu Xenial
# Skip any driver through apt, go to nvidia official site and use .deb(local)

# To remove outdated drivers
sudo nvidia-uninstall

# https://askubuntu.com/questions/530043/removing-nvidia-cuda-toolkit-and-installing-new-one
sudo apt-get --purge remove "*nvidia*"
sudo apt autoremove
sudo apt autoclean

*IMPORTANT*
If automatic kernel update is enabled, the nvidia driver will be automatically update and most of the time, those drivers failed.
It usually shows "Failed to initialize NVML: Driver/library version mismatch" when nvidia-smi is issued.

# check kernel version, you might see updated kernel
uname -ra 

# check nvidia* package
dpkg -l | grep -i nvidia | less

# follow steps above to reinstall drivers (use nvidia official!)
https://developer.nvidia.com/cuda-downloads

Check system GPU and Memory

# GPU
sudo lshw -c video

# Memory
free -g
sudo dmidecode -t memory | grep -i size | sort | uniq -c
grep MemTotal /proc/meminfo

conda create and remove environment

conda create -n utopia python=3.9.5

conda remove --name utopia --all

Rsync

rsync -varP <src> <dst>

UTF8 Encoding for python

export PYTHONIOENCODING=utf-8

Change number of workspace in ubuntu16.04

gsettings set org.compiz.core:/org/compiz/profiles/unity/plugins/core/ hsize 2
gsettings set org.compiz.core:/org/compiz/profiles/unity/plugins/core/ vsize 4

Prompt showing git branch

parse_git_branch() {
 git branch 2> /dev/null | sed -e '/^[^*]/d' -e 's/* \(.*\)/(\1)/'
}
if [ "$color_prompt" = yes ]; then
 PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\] : \[\033[01;34m\]\w\[\033[01;31m\] $(parse_git_branch)\[\033[00m\]\$ '
else
 PS1='${debian_chroot:+($debian_chroot)}\u@\h : \w $(parse_git_branch)\$ '
fi

bash utility functions

# .bashutils.sh
function vshome {
    export VSHOME=$HOME
    cd $VSHOME
}

function new_conda_env {
    vshome
    mkdir -p $1
    conda create -n $1 python=$2
}

function rm_conda_env {
    conda remove --name $1 --all
    echo "Reminder: you might want to remove ${1} folder"
}

# Vertical/Columnar aligned less on csv
function pretty_csv {
    column -t -s, -n "$@" | less -F -S -X -K
}

# Change group and owner to USER
function owndir {
    sudo chown -R ${USER}:${USER} $@
    ls -l $@
}

function get_gpu {
    echo "[Info]: CUDA_VISIBLE_DEVICES = ${CUDA_VISIBLE_DEVICES}"
}

function set_gpu {
    export CUDA_VISIBLE_DEVICES=$1
    get_gpu
}

function hostnetron {
    netron --host 0.0.0.0 $@
}

function hosttb {
    tensorboard --logdir=./ --host=0.0.0.0
}

function act {
    conda activate $1 bash
}

function deact {
    conda deactivate
}

function ls-env-conda {
    conda info --envs
}

function docker-terminal {
    sudo docker exec -it $1 bash
}

function get-latest-container {
    export latest_container=$(sudo docker ps -n=-1 --format "{{.ID}}")
}

function dt {
    docker-terminal $latest_container
}

function ls-container {
    get-latest-container
    sudo docker ps --format "{{.ID}} | {{.Image}} | {{.Names}} | {{.Status}}"
}

function ns {
    nvidia-smi $@
}

function ns1 {
    nvidia-smi -l 1
}

function gs {
    git status $@
}

function recursive_pylint() {
    find . -iregex '.*\.py' -exec pylint {} +
}

# TODO: Need to figure out a neat way of achiving objective below
# function rm {
#    mkdir -p /tmp/.trash
#    mv $@ /tmp/.trash
# }

get-latest-container
ls-container
ls-env-conda

Compress/Decompress file

# compress to tar.gz
# ------------------
tar -cvzf example.tar.gz {files or folders to tar and compress}  

# tar.gz/.tgz
# ------------
# decompress to a folder
tar -zxvf foo.tar.gz -C bar/

# recursive decompress
for targz in *.tar.gz
do 
  dir=$(basename ${targz} .tar.gz)
  mkdir -p $dir && tar zxvf ${targz} -C $dir
done

# bz2
# ---
bzip2 -d filename.bz2

# To preserve the original archive, add the -k option:
bzip2 -dk filename.bz2

# tar
tar xvf 

Txt file processing in bash

# concatenate every other line of a text file
paste -s -d' \n' input.txt 

Find files

# Find multiple extensions
find . -name "*.caffemodel" -o -name "*.prototxt"

Remove duplicated level of folders

function rm_duplicate_dir() {
  for filepath in "$1"/*
  do
      if [ -d ${filepath} ] ; then
          # echo "current: $filepath"
          dirs=(${filepath//// })

          if [ ${dirs[-1]} == ${dirs[-2]} ] ; then
              echo "[Info]: Duplicated Found => ${dirs[-1]} == ${dirs[-2]}"
              abspath=$(realpath $filepath)
              absdir=$(dirname $abspath)
              mv $abspath/* $absdir/. && rm -rf $abspath
          fi
          rm_duplicate_dir "${filepath}"
      fi
  done
}

$ rm_duplicate_dir ./

CMake

# Undocumented one-liner to generate with CMake (https://cmake.org/pipermail/cmake-developers/2018-January/030520.html)
cmake -H. -Bbuild

# List all targets at the generated directory
cmake --build . --target help

# Build for Debug
cmake -DCMAKE_BUILD_TYPE=Debug ..

VLC

# Run VLC as root 
# For example, in docker, execute the line below to replace geteuid to getppid in the vlc executable, you can run the vlc as usual after that. 
sed -i 's/geteuid/getppid/' /usr/bin/vlc

### Play compressed stream in VLC
vlc --demux h264 test_kendo_mvc.264

# mpeg4 part2
vlc --demux mp4v test1_cam4_2Mbps.mpeg4

### Play raw frames in VLC
vlc --demux rawvideo --rawvid-fps 30 --rawvid-height 768 --rawvid-width 1024 --rawvid-chroma I420 test_dec_ViewId0001.yuv

Convert text encoding

$ file -i docker_run.sh 
docker_run.sh: text/x-shellscript; charset=us-ascii

$ iconv -f us-ascii -t utf-8  docker_run.sh -o docker_run.utf-8.sh

$ file -i docker_run.utf-8.sh
docker_run.utf-8.sh: text/x-shellscript; charset=us-ascii

Update Ubuntu Time

# Update HW Clk from google.com
sudo hwclock --set --date="$(wget -qSO- --max-redirect=0 google.com 2>&1 | grep Date: | cut -d' ' -f5-8)Z"

# [obsolete, just for record] Set the kernel clock, on reboot, kernel will follow hardware clock.
$ sudo date -s "$(wget -qSO- --max-redirect=0 google.com 2>&1 | grep Date: | cut -d' ' -f5-8)Z"

Mount Drive

# check filesystem type of drive
sudo file -s /dev/sdb

# permanent drive mount
# in /etc/fstab, add entry like below
# -----------------------------------------------------------------------------------
# /dev/sdb: Linux rev 1.0 ext4 filesystem data,
# UUID=2e030e52-7f87-4875-8afe-b265cced4ebd,
# volume name "Lambda 2TB HDD" (extents) (large files) (huge files)
UUID=2e030e52-7f87-4875-8afe-b265cced4ebd /data           ext4    defaults 0       2
# -----------------------------------------------------------------------------------

Create & Remove User

# add sudo user
sudo adduser vchua
sudo usermod -aG sudo vchua

# delete user and remove home directory
sudo deluser --remove-home vchua
 
# lock/unlock user
sudo passwd -l <user>
sudo passwd -u <user>

# check lock/unlock status, L is locked, P is unlocked
sudo passwd -S <user>

Bypass password prompt during sudo

# edit sudoers file to execute program in sudo without password
sudo visudo
# Append a line with your USERNAME
vchua ALL=(ALL) NOPASSWD: ALL

Modify ubuntu default text editor

sudo update-alternatives --config editor

vimrc 4-space tab for python

filetype plugin indent on
" show existing tab with 4 spaces width
set tabstop=4
" when indenting with '>', use 4 spaces width
set shiftwidth=4
" On pressing tab, insert 4 spaces
set expandtab

Fork github to gitlab

# Ref: # https://gist.github.com/DavideMontersino/810ebaa170a2aa2d2cad

# 1. clone the github project in your workspace
git clone <github>

# 2. rename the remote
git remote rename origin upstream

# 3. Create a new repo in gitlab website

# 4. Add the new origin to your repo
git remote add origin git@gitlab.extranet.weborama.com:whatever/swiper.git

# 5. push to the private repository (track master) - only the first time
git push -u origin master

# all subsequent push
git push

# bypass certificate verification
export GIT_SSL_NO_VERIFY=1

Webcam setting in linux

# Webcam in Ubuntu

### How to access webcam (can be used for camera validation)
1. $ sudo apt-get install -y cheese; cheese
2. vlc - File > Open Capture Device -> Capture Mode select Direct Show
Reference: https://askubuntu.com/questions/117291/how-to-turn-on-the-integrated-webcam

### Configure webcam
sudo apt-get install guvcview

# Reference: http://www.linux-hardware-guide.com/2013-06-19-microsoft-lifecam-cinema-720p-169