#!/bin/bash

declare -A NIC

	
RPS_CPUS=();
XPS_CPUS=();
XPS_RQS=();
	
NIC[vmx0]=IN
NIC[vmx1]=OUT
NIC[mgmt]=IN


:<< _EOF
What:		/sys/class/net/<iface>/queues/tx-<queue>/xps_cpus
Date:		November 2010
KernelVersion:	2.6.38
Contact:	netdev@vger.kernel.org
Description:
		** Mask of the CPU(s) ** currently enabled to participate into the
		Transmit Packet Steering packet processing flow for this
		network device transmit queue. Possible values depend on the
		number of available CPU(s) in the system.

What:		/sys/class/net/<iface>/queues/tx-<queue>/xps_rxqs
Date:		June 2018
KernelVersion:	4.18.0
Contact:	netdev@vger.kernel.org
Description:
		** Mask of the receive queue(s) ** currently enabled to participate
		into the Transmit Packet Steering packet processing flow for this
		network device transmit queue. Possible values depend on the
		number of available receive queue(s) in the network device.
		Default is disabled.
_EOF

let CPUS=$(nproc)
let CC=$(( CPUS - 1 ))
ALL_CPUS=( `seq 0 $CC` )


NICS=(`ip link | awk -F: '$0 !~ "lo|vir|wl|^[^0-9]"{print $2}' | tr -d ' '`)

systemctl stop irqbalance.service

ORD_CORE_MASK()
{
	local s=$3;
	local l=$4;
	local z=0	
	local i=0;
	declare -a tc;
	local x=0;

	declare -n I="$1" OP="$2"
	tc=( ${I[@]:s:l}  )	
	tc+=( ${I[@]} )
	tc=( ${tc[@]:0:l}  )

	echo "tc ${#tc[@]} ( ${tc[@]} )"
	
	for (( i=0; i < ${#tc[*]}; i++ ))
	do
		let b=$x
		z=$(( 1 << tc[i] ))
		x=$(( b | z ))
#		echo x[$x] b[$b] z[$z]
	done
	
	OP=$x	
#	printf "%02d%032s\n" $O $(echo "obase=2; $O" | bc)
}


GET_CORE_MASK_C()
{
	local z=0
	local s=0;
	local i=0;
	local m=0;
	local x=$3

	declare -n IN="$1" O="$2"
	
	for (( m=0 ; m < ${#IN[*]} ; m++ ))
	do
		ORD_CORE_MASK IN O[$m] $m $x
	done
}

:<< _EOF
XX=`nproc`
PP=( `seq 0 $((XX - 1))` )
declare -a TT
for (( m=0 ; m < ${#PP[*]} ; m++ ))
do
	ORD_CORE_MASK PP TT[$m] $m 3
done

echo ${TT[@]}

exit
_EOF



GET_CORE_MASK()
{
	local z=0
	local s=0;
	local i=0;

	declare -n I="$1" O="$2"
	
	[ ${#I[@]} -le 0 ] && O=( 0 ) && return;
	
	s=$(( I[0] ))
	
	[[ $s -lt 0 ]] && z=$(( ( 1 << `nproc` ) - 1 )) && O=( $z ) && return;
	
	for (( i=0 ; i < ${#I[@]}; i++ ))
	do
		s=$(( I[i] ))
		z=$(( 1 << s ))
		O+=( $z )
	done
}

GET_QUEUE_LIST()
{
	local IFACE=$1
	
	RPS_CPUS=();
	XPS_CPUS=();
	XPS_RQS=();
	
	if [ -d /sys/class/net/$IFACE/queues/ ]; then
		RPS_CPUS=( `find /sys/class/net/$IFACE/queues/ -name rps_cpus | sort -V` )
		XPS_CPUS=( `find /sys/class/net/$IFACE/queues/ -name xps_cpus | sort -V` )
		XPS_RQS=(`find /sys/class/net/$IFACE/ -name "xps_rxqs" | sort -V`)
	fi
}

GET_IRQS()
{
	local IFACE=$1
	local Q=0;
	
	IRQx=();
	IRQi=();
	IRQo=();
	
	IRQx=(`grep -iE "$IFACE-rxtx-" /proc/interrupts | awk -F':' '{print $1}'`) 
	let Q=${#IRQx[*]}
	[ $Q -ne 0 ] && return
	
	local qq=$(basename `readlink -f /sys/class/net/$IFACE/device`)	
	IRQi=(`grep -iE "$qq-input" /proc/interrupts | awk -F':' '{print $1}'`) 
	IRQo=(`grep -iE "$qq-output" /proc/interrupts | awk -F':' '{print $1}'`) 
}

ETHTOOL_SET_RING()
{
	local IFACE=$1
	local Q=$(( 0 ))
	local J=();
	declare -A V=()
	
	# Max Ring parameters for ${IFACE}:
	
	J=(`ethtool -g ${IFACE} | head -n 6 | grep -i "RX:"`); V[RX]=${J[-1]}
	J=(`ethtool -g ${IFACE} | head -n 6 | grep -i "RX Mini:"`); V[RXM]=${J[-1]}
	J=(`ethtool -g ${IFACE} | head -n 6 | grep -i "RX Jumbo:"`); V[RXJ]=${J[-1]}
	J=(`ethtool -g ${IFACE} | head -n 6 | grep -i "TX:"`); V[TX]=${J[-1]}
	
	ethtool -G ${IFACE} rx ${V[RX]} rx-mini ${V[RXM]} rx-jumbo ${V[RXJ]} tx ${V[TX]} # 2> /dev/null

	# Max Channel parameters for ${IFACE}:
	V=();

	J=(`ethtool -l ${IFACE} | head -n 6 | grep -i "RX:"`); V[RX]=${J[-1]}
	J=(`ethtool -l ${IFACE} | head -n 6 | grep -i "TX:"`); V[TX]=${J[-1]}
	J=(`ethtool -l ${IFACE} | head -n 6 | grep -i "Other:"`); V[Other]=${J[-1]}
	J=(`ethtool -l ${IFACE} | head -n 6 | grep -i "Combined:"`); V[Combined]=${J[-1]}

	ethtool -L ${IFACE} rx ${V[RX]} tx ${V[TX]} other ${V[Other]} combined ${V[Combined]}  # 2> /dev/null
}


SET_IRQ_CPUS()
{
	local IFACE=$1
	
	local Q=${#IRQS[*]}
	local CPU_BITMASK=();
	local USE_CORES=();
	local CORE_LIST=();
	local M="IN"
	local i=0;

	
	M="${NIC[${IFACE}]}"
	
	[ "x${M}" == "xOUT" ] && CORE_LIST=( "${OUT_IRQ_CPUS[@]}" ) || CORE_LIST=( "${IN_IRQ_CPUS[@]}" )	
		
	GET_CORE_MASK CORE_LIST USE_CORES
		
	for (( i=0; ${#CPU_BITMASK[*]} < Q ; i++ ))
	do
		CPU_BITMASK+=( ${USE_CORES[*]}  )
	done
	
	_IFS="$IFS"
	IFS=$'\n' CPU_BITMASK=($(sort -n <<<"${CPU_BITMASK[*]}"))
	IFS="$_IFS"

	echo "$FUNCNAME $IFACE: [${CORE_LIST[@]}]:  ${CPU_BITMASK[@]}"
	
	for (( i = 0 ; i < Q  ; i++ )) 
	do
		printf "%08x" "${CPU_BITMASK[$i]}" > /proc/irq/${IRQS[$i]}/smp_affinity
		echo -e "$IFACE \t /proc/irq/${IRQS[$i]}/smp_affinity `</proc/irq/${IRQS[$i]}/smp_affinity`"
	done
	
	return 0;	
}

IRQ_SETUP()
{
	GET_IRQS "$@"
	[ ${#IRQx[@]} -gt 0 ] && IRQS=( "${IRQx[@]}" ) && SET_IRQ_CPUS $@
	[ ${#IRQi[@]} -gt 0 ] && IRQS=( "${IRQi[@]}" ) && SET_IRQ_CPUS $@
	[ ${#IRQo[@]} -gt 0 ] && IRQS=( "${IRQo[@]}" ) && SET_IRQ_CPUS $@
}


RPS_SETUP()
{
	local IFACE=$1
	
	local M="IN"
	local RPS_CPUS=( `find /sys/class/net/$IFACE/queues/ -name rps_cpus | sort -V` )
	local CPU_BITMASK=();
	local USE_CORES=();
	local CORE_LIST=();
	local RPS_FLOW_CNT=0;
	local Q=${#RPS_CPUS[*]}
	local i=0;

	M="${NIC[${IFACE}]}"
	
	[ "x${M}" == "xOUT" ] && CORE_LIST=( "${OUT_RPS_CPUS[@]}" ) || CORE_LIST=( "${IN_RPS_CPUS[@]}" )	
	[ "x${M}" == "xOUT" ] && RPS_FLOW_CNT=$(( OUT_FLOW_COUNT )) || RPS_FLOW_CNT=$(( IN_FLOW_COUNT ))
		
	
	GET_CORE_MASK CORE_LIST USE_CORES
	
		
	for (( i=0; ${#CPU_BITMASK[*]} < Q ; i++ ))
	do
		CPU_BITMASK+=( ${USE_CORES[*]}  )
	done
	
	_IFS="$IFS"
	IFS=$'\n' CPU_BITMASK=($(sort -n <<<"${CPU_BITMASK[*]}"))
	IFS="$_IFS"
		
	echo "$FUNCNAME $IFACE [${CORE_LIST[*]}]:  ${CPU_BITMASK[*]}"
		
	for (( i = 0 ; i < Q  ; i++ )) 
	do
		printf "%08x" ${CPU_BITMASK[$i]}  > ${RPS_CPUS[$i]}
		
		MAX_RPS_FLOW_CNT=$(( RPS_FLOW_CNT + MAX_RPS_FLOW_CNT ))
		D=`dirname ${RPS_CPUS[$i]}`
		echo $(( RPS_FLOW_CNT ))  > ${D}/rps_flow_cnt
		
		echo -e "$IFACE \t ${RPS_CPUS[$i]} `<${RPS_CPUS[$i]}`"
		echo -e "$IFACE \t ${D}/rps_flow_cnt `<${D}/rps_flow_cnt`		"
	done
	return;
}

XPS_SETUP()
{
	local IFACE=$1
	
	local M="IN"
	local XPS_CPUS=( `find /sys/class/net/$IFACE/queues/ -name xps_cpus | sort -V` )
	local CPU_BITMASK=();
	local USE_CORES=();
	local CORE_LIST=();
	local Q=${#XPS_CPUS[*]}
	local i=0;


	M="${NIC[${IFACE}]}"
	
	[ "x${M}" == "xOUT" ] && CORE_LIST=( "${OUT_XPS_CPUS[@]}" ) || CORE_LIST=( "${IN_XPS_CPUS[@]}" )	
	
	
	GET_CORE_MASK CORE_LIST USE_CORES
		
	for (( i=0; ${#CPU_BITMASK[*]} < Q ; i++ ))
	do
		CPU_BITMASK+=( ${USE_CORES[*]}  )
	done

	_IFS="$IFS"
	IFS=$'\n' CPU_BITMASK=($(sort -n <<<"${CPU_BITMASK[*]}"))
	IFS="$_IFS"
	
	echo "$FUNCNAME $IFACE [${CORE_LIST[*]}]:  ${CPU_BITMASK[*]}"
	
	for (( i = 0 ; i < Q  ; i++ )) 
	do
		printf "%08x" ${CPU_BITMASK[$i]}  > ${XPS_CPUS[$i]}				
		echo -e "$IFACE \t ${XPS_CPUS[$i]} `<${XPS_CPUS[$i]}`"
	done
	return;
}


RXQS_SETUP()
{
	local IFACE="$1"
	local QUEUE_BITMASK=();
	local ALL_QUEUES=();
	local USE_QUEUES=();
	local X_QUEUES=();
	local i=0;
	local M="IN"
	
	local XPS_RQS=(`find /sys/class/net/$IFACE/ -name "xps_rxqs" | sort -V`)
	local Q=${#XPS_RQS[@]}
		
	[ $Q -le 0 ] && return;

	M="${NIC[${IFACE}]}"

	[ "x${M}" == "xOUT" ] && ALL_QUEUES=( "${OUT_XPS_RQS[@]}" ) || ALL_QUEUES=( "${IN_XPS_RQS[@]}" )
	
	i=${#XPS_RQS[*]}
	i=$(( i - 1 ))
	
	X_QUEUES=( `seq 0 $i` )

	GET_CORE_MASK ALL_QUEUES USE_QUEUES
#	GET_CORE_MASK_C X_QUEUES USE_QUEUES ${#ALL_QUEUES[*]}

		
	for (( i=0; ${#QUEUE_BITMASK[*]} < Q ; i++ ))
	do
		QUEUE_BITMASK+=( ${USE_QUEUES[*]}  )
	done

	_IFS="$IFS"
	IFS=$'\n' QUEUE_BITMASK=($(sort -n <<<"${QUEUE_BITMASK[*]}"))
	IFS="$_IFS"


	echo -e "$FUNCNAME $IFACE \t QUEUE_BITMASK: [${#QUEUE_BITMASK[*]}]:  ${QUEUE_BITMASK[*]}"


	for (( i = 0 ; i < Q  ; i++ )) 
	do
		printf "%08x" ${QUEUE_BITMASK[$i]} >  ${XPS_RQS[$i]}		
		echo -e "$IFACE \t ${XPS_RQS[$i]} `<${XPS_RQS[$i]}`"
	done
	
}


MAIN()
{
		
	local N=${#NICS[@]}	
	echo "NICs: $N: ${NICS[@]}"
	local IFACE=
	local M="IN"
	
	let MAX_RPS_FLOW_CNT=$(( 0 ))

	systemctl stop irqbalance.service
	echo 0 > /proc/sys/net/core/rps_sock_flow_entries
	
	local i=0;	
	for (( i=0 ; i < N ; i++ ))
	do
		IFACE=${NICS[$i]}
		M="IN"
		M="${NIC[${IFACE}]}"
		echo  "Tuning NICS[$i] ${IFACE} $M"

		ETHTOOL_SET_RING "${IFACE}"

		IRQ_SETUP "${IFACE}"
		RPS_SETUP "${IFACE}" 
		XPS_SETUP "${IFACE}"
		RXQS_SETUP "${IFACE}"
	done
	
	MAX_RPS_FLOW_CNT=0 # $(( 1 << 16 ))
	
	echo $MAX_RPS_FLOW_CNT > /proc/sys/net/core/rps_sock_flow_entries

}

PLANZ()
{
	local T=${#ALL_CPUS[@]} #total cpus available in the host system 
	local R=1 # cpus reserved for operations other than networking
	local I=3 # cpus reserved for incoming packets from clients and servers
	local C=6 # cpus reserved for outgoing packets to clients
	local n=0
	local W=0

	IN_RPS_CPUS=( ${ALL_CPUS[@]:R:I} );
	n=$(( R + I ))
	IN_XPS_CPUS=( ${ALL_CPUS[@]:n:C} ); 
	IN_IRQ_CPUS=( ${IN_RPS_CPUS[@]} ); 
	IN_XPS_RQS=( ${ALL_CPUS[@]:-1:0} ); 
	let IN_FLOW_COUNT=8192

	let n=$(( R + I + C ))
	let W=$(( T - n ))

	OUT_RPS_CPUS=( ${IN_RPS_CPUS[@]} ); 
	OUT_XPS_CPUS=( ${ALL_CPUS[@]:n:W} );
	OUT_IRQ_CPUS=( ${OUT_XPS_CPUS[@]} );
	OUT_XPS_RQS=( ${ALL_CPUS[@]:-1:0} ); 
	let OUT_FLOW_COUNT=8192
	
	return;
}


declare -f PLANZ

PLANZ
MAIN
