diee/prep-node.sh at main · publiusys/diee · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
#!/bin/bash

#set -x

CWD=$(pwd)
GENIUSER=`geni-get user_urn | awk -F+ '{print $4}'`

sudo apt-get update
sudo apt install msr-tools cpufrequtils python3 python3-pip netperf openjdk-8-jdk-headless libopencv-dev python3-opencv uuid-dev autotools-dev automake tcl libtool libreadline-dev libgtop2-dev bison swig scons libevent-dev gengetopt libzmq3-dev libevent-dev -y

# disable HyperThreads
echo off | sudo tee /sys/devices/system/cpu/smt/control

# disable TurboBoost
echo "1" | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo

# disable irq rebalance
sudo killall irqbalance

# set irq affinity - make sure receive/transmit queues are mapped to the same core
ieth=$(ifconfig | grep -B1 10.10.1 | grep -o "^\w*")
sudo $CWD/intel_set_irq_affinity.sh -x all ${ieth}

# sets hostname depending on IP
mip=$(ifconfig | grep -B1 10.10.1 | grep inet | grep -oP 'inet \K(\d+\.\d+\.\d+\.\d+)')
case $mip in
    "10.10.1.1")
	v=$(echo $mip | cut -d. -f4)
	sudo hostname client-$v
	;;

    "10.10.1.2")
	v=$(echo $mip | cut -d. -f4)
	sudo hostname server-$v
	;;

    *)
	v=$(echo $mip | cut -d. -f4)
	sudo hostname agent-$v
	;;
esac

# this is causing firmware issues on c6220 nodes, disable for now
sudo rmmod mlx4_ib
sudo rmmod mlx4_core

# list current status
sudo ufw status

# setup firewall
sudo ufw allow ssh

# allow connections from the following IP
sudo ufw allow from 10.10.1.1
sudo ufw allow from 10.10.1.2
sudo ufw allow from 10.10.1.3
sudo ufw allow from 10.10.1.5
sudo ufw allow from 10.10.1.6
sudo ufw allow from 10.10.1.7
sudo ufw allow from 10.10.1.8
sudo ufw allow from 10.10.1.9
sudo ufw allow from 10.10.1.10

sudo ufw allow 8080
sudo ufw allow 11211

# deny everything else
sudo ufw default allow outgoing
sudo ufw default deny incoming

# enable ufw
echo y | sudo ufw enable
sudo ufw status

# disable redundant logging messages
sudo ufw logging off

# enable MSR to set DVFS statically
sudo modprobe msr
# lets run without sudo
sudo setcap cap_sys_rawio=ep /usr/sbin/rdmsr
sudo setcap cap_sys_rawio=ep /usr/sbin/wrmsr
sudo setcap cap_net_admin+ep /usr/sbin/ethtool

# setup password-less ssh between nodes
sudo $CWD/prep-ssh.sh

# allow msr without sudo
sudo groupadd msr
sudo chgrp msr /dev/cpu/*/msr
sudo ls -l /dev/cpu/*/msr
sudo chmod g+rw /dev/cpu/*/msr
sudo usermod -aG msr $GENIUSER

echo "🔴🔴  Exit and relogin to this node for msr group changes to take effect 🔴🔴"
sudo newgrp msr