$ sudo useradd -m -d /var/lib/munge -s /sbin/nologin -u 5001 munge

$ sudo apt update
$ sudo apt install -y munge libmunge2 libmunge-dev

$ sudo systemctl restart munge.service

$ munge -n | unmunge | grep STATUS
STATUS:           Success (0)
$

$ sudo apt install -y mariadb-server libmariadb-dev

$ sudo diff /etc/mysql/mariadb.conf.d/50-server.cnf_org /etc/mysql/mariadb.conf.d/50-server.cnf
106a107,108
> innodb_buffer_pool_size=4096M
> innodb_lock_wait_timeout=900
$

$ sudo systemctl restart mariadb

$ sudo mysql
Welcome to the MariaDB monitor.  Commands end with ; or \g.
Your MariaDB connection id is 8
Server version: 10.3.39-MariaDB MariaDB Server

Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

MariaDB [(none)]> create database slurm_acct_db;
Query OK, 1 row affected (0.000 sec)

MariaDB [(none)]> create user 'slurm'@'localhost' identified by 'SLURM';
Query OK, 0 rows affected (0.000 sec)

MariaDB [(none)]> set password for slurm@localhost = password('passcord');
Query OK, 0 rows affected (0.000 sec)

MariaDB [(none)]> grant all on slurm_acct_db.* TO 'slurm'@'localhost';
Query OK, 0 rows affected (0.000 sec)

MariaDB [(none)]> FLUSH PRIVILEGES;
Query OK, 0 rows affected (0.000 sec)

MariaDB [(none)]> exit
Bye
$

$ mysql --user=slurm --password=passcord slurm_acct_db -e 'show databases;'
+--------------------+
| Database           |
+--------------------+
| information_schema |
| slurm_acct_db      |
+--------------------+
$

$ sudo apt install -y build-essential gcc g++ make libtool pkg-config
$ mkdir ~/`hostname` && cd ~/`hostname` && wget https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz
$ tar -xvf ./libevent-2.1.12-stable.tar.gz
$ cd libevent-2.1.12-stable && ./configure --prefix=/opt/libevent
$ make -j 16 && sudo make install
$ cd ~/`hostname` && wget https://download.open-mpi.org/release/hwloc/v2.12/hwloc-2.12.2.tar.gz
$ tar -xvf ./hwloc-2.12.2.tar.gz
$ cd hwloc-2.12.2 && ./configure --prefix=/opt/hwloc
$ make -j 16 && sudo make install
$ cd ~/`hostname` && wget https://github.com/openpmix/openpmix/releases/download/v5.0.8/pmix-5.0.8.tar.gz
$ tar -xvf ./pmix-5.0.8.tar.gz
$ cd pmix-5.0.8 && ./configure --prefix=/opt/pmix --with-libevent=/opt/libevent --with-hwloc=/opt/hwloc
$ make -j 16 && sudo make install

$ cd ~/`hostname` && wget https://github.com/openucx/ucx/releases/download/v1.19.0/ucx-1.19.0.tar.gz
$ tar -xvf ./ucx-1.19.0.tar.gz
$ cd ucx-1.19.0 && ./contrib/configure-release --prefix=/opt/ucx
$ make -j 16 && sudo make install

$ sudo apt install -y checkinstall libmariadb-dev libdbus-1-dev libnuma-dev

$ mkdir ~/`hostname` && cd ~/`hostname` && wget https://download.schedmd.com/slurm/slurm-25.05.3.tar.bz2
$ tar -xvf ./slurm-25.05.3.tar.bz2
$ cd slurm-25.05.3 && ./configure --prefix=/opt/slurm --sysconfdir=/opt/slurm/etc --disable-debug --with-pmix=/opt/pmix --with-ucx=/opt/ucx --with-systemdsystemunitdir=/lib/systemd/system
$ make -j 16
$ sudo checkinstall -y --install=no --fstrans=no

$ ls *.deb
slurm_25.05.3-1_amd64.deb
$

$ sudo apt install -y ./slurm_25.05.3-1_amd64.deb

$ sudo useradd -m -d /var/lib/slurm -s /bin/bash -u 5000 slurm
$ sudo mkdir /var/spool/slurmctld && sudo chown slurm:slurm /var/spool/slurmctld
$ sudo mkdir /var/spool/slurmd && sudo chown slurm:slurm /var/spool/slurmd
$ sudo mkdir /var/log/slurm && sudo chown slurm:slurm /var/log/slurm
$ sudo mkdir /opt/slurm/etc && sudo chown slurm:slurm /opt/slurm/etc
$ sudo su - slurm
$ echo "export PATH=/opt/slurm/sbin:/opt/slurm/bin:\$PATH" | tee -a ~/.bash_profile

$ sudo useradd -m -d /var/lib/slurm -s /bin/bash -u 5000 slurm
$ sudo mkdir /var/spool/slurmd && sudo chown slurm:slurm /var/spool/slurmd
$ sudo mkdir /var/log/slurm && sudo chown slurm:slurm /var/log/slurm
$ sudo mkdir /opt/slurm/etc && sudo chown slurm:slurm /opt/slurm/etc

$ sudo useradd -m -d /var/lib/slurm -s /bin/bash -u 5000 slurm
$ sudo mkdir /opt/slurm/etc && sudo chown slurm:slurm /opt/slurm/etc

ClusterName=sltest
SlurmctldHost=slurm-srv-ub
AuthType=auth/munge
PluginDir=/opt/slurm/lib/slurm
SchedulerType=sched/backfill
SlurmUser=slurm
SlurmctldPort=7002
SlurmctldTimeout=300
SlurmdPort=7003
SlurmdSpoolDir=/var/spool/slurmd
SlurmdTimeout=300
SlurmctldLogFile=/var/log/slurm/slurmctld.log
SlurmdLogFile=/var/log/slurm/slurmd.log
SlurmdDebug=3
StateSaveLocation=/var/spool/slurmd
SwitchType=switch/none
AccountingStorageType=accounting_storage/slurmdbd
AccountingStorageHost=slurm-srv-ub
AccountingStoragePort=7004
MpiDefault=pmix
#
# For per-GPU scheduling
AccountingStorageTRES=gres/gpu
DebugFlags=CPU_Bind,gres
GresTypes=gpu
JobAcctGatherType=jobacct_gather/cgroup
SelectType=select/cons_tres
TaskPlugin=task/cgroup,task/affinity
#
# GPU node specifications
NodeName=inst-ao-ub Gres=gpu:nvidia_a100-sxm4-40gb:8 CPUs=64 Boards=1 SocketsPerBoard=2 CoresPerSocket=32 ThreadsPerCore=1 RealMemory=2000000 TmpDisk=10000 State=UNKNOWN
PartitionName=sltest Nodes=ALL DefMemPerGPU=250000 Default=YES MaxTime=INFINITE State=UP

$ srun -p sltest --gres=gpu:nvidia_a100-sxm4-40gb:4 --mem=1500000G ./a.out

ArchiveEvents=yes
ArchiveJobs=yes
ArchiveResvs=yes
ArchiveSteps=no
ArchiveSuspend=no
ArchiveTXN=no
ArchiveUsage=no
AuthType=auth/munge
AuthInfo=/var/run/munge/munge.socket.2
DbdHost=slurm-srv-ub
DbdPort=7004
DebugLevel=info
PurgeEventAfter=1month
PurgeJobAfter=12month
PurgeResvAfter=1month
PurgeStepAfter=1month
PurgeSuspendAfter=1month
PurgeTXNAfter=12month
PurgeUsageAfter=24month
LogFile=/var/log/slurm/slurmdbd.log
PidFile=/var/run/slurmdbd/slurmdbd.pid
SlurmUser=slurm
StorageType=accounting_storage/mysql
StorageUser=slurm
StoragePass=passcord
StorageLoc=slurm_acct_db

PMIxDirectConnEarly=true
PMIxDirectConnUCX=true

ConstrainCores=yes 
ConstrainDevices=yes
ConstrainRAMSpace=yes

Name=gpu Type=nvidia_a100-sxm4-40gb File=/dev/nvidia[0-3] COREs=[0-31]
Name=gpu Type=nvidia_a100-sxm4-40gb File=/dev/nvidia[4-7] COREs=[32-63]

$ sudo systemctl enable --now slurmdbd
$ sudo systemctl enable --now slurmctld

$ sudo systemctl enable --now slurmd

$ sinfo
PARTITION AVAIL  TIMELIMIT  NODES  STATE NODELIST
sltest*      up   infinite      1   idle inst-ao-ub
$

$ srun --mpi=list
MPI plugin types are...
	none
	pmix
	pmi2
	cray_shasta
specific pmix plugin versions available: pmix_v5
$

$ echo "export PATH=/opt/slurm/sbin:/opt/slurm/bin:\$PATH" | tee -a ~/.bashrc

$ srun -p sltest -n 1 --gres=gpu:nvidia_a100-sxm4-40gb:1 nvidia-smi | grep SXM
|   0  NVIDIA A100-SXM4-40GB          On  |   00000000:51:00.0 Off |                    0 |
$ srun -p sltest -n 1 --gres=gpu:nvidia_a100-sxm4-40gb:2 nvidia-smi | grep SXM
|   0  NVIDIA A100-SXM4-40GB          On  |   00000000:51:00.0 Off |                    0 |
|   1  NVIDIA A100-SXM4-40GB          On  |   00000000:54:00.0 Off |                    0 |
$ srun -p sltest -n 1 --gres=gpu:nvidia_a100-sxm4-40gb:4 nvidia-smi | grep SXM
|   0  NVIDIA A100-SXM4-40GB          On  |   00000000:0F:00.0 Off |                    0 |
|   1  NVIDIA A100-SXM4-40GB          On  |   00000000:15:00.0 Off |                    0 |
|   2  NVIDIA A100-SXM4-40GB          On  |   00000000:51:00.0 Off |                    0 |
|   3  NVIDIA A100-SXM4-40GB          On  |   00000000:54:00.0 Off |                    0 |
$ srun -p sltest -n 1 --gres=gpu:nvidia_a100-sxm4-40gb:8 nvidia-smi | grep SXM
|   0  NVIDIA A100-SXM4-40GB          On  |   00000000:0F:00.0 Off |                    0 |
|   1  NVIDIA A100-SXM4-40GB          On  |   00000000:15:00.0 Off |                    0 |
|   2  NVIDIA A100-SXM4-40GB          On  |   00000000:51:00.0 Off |                    0 |
|   3  NVIDIA A100-SXM4-40GB          On  |   00000000:54:00.0 Off |                    0 |
|   4  NVIDIA A100-SXM4-40GB          On  |   00000000:8D:00.0 Off |                    0 |
|   5  NVIDIA A100-SXM4-40GB          On  |   00000000:92:00.0 Off |                    0 |
|   6  NVIDIA A100-SXM4-40GB          On  |   00000000:D6:00.0 Off |                    0 |
|   7  NVIDIA A100-SXM4-40GB          On  |   00000000:DA:00.0 Off |                    0 |
$ srun -p sltest -n 1 --gres=gpu:nvidia_a100-sxm4-40gb:8 nvidia-smi | grep SXM | awk '{print $2, $7}'
0 00000000:0F:00.0
1 00000000:15:00.0
2 00000000:51:00.0
3 00000000:54:00.0
4 00000000:8D:00.0
5 00000000:92:00.0
6 00000000:D6:00.0
7 00000000:DA:00.0
$

$ module load nvhpc openmpi omb
$ srun -p sltest -n 2 --gres=gpu:nvidia_a100-sxm4-40gb:2 osu_latency -x 1000 -i 10000 -m 1:1 -d cuda D D

# OSU MPI-CUDA Latency Test v7.5
# Datatype: MPI_CHAR.
# Size       Avg Latency(us)
1                       2.75
$

$ srun -p sltest -n 2 --gres=gpu:nvidia_a100-sxm4-40gb:2 osu_bw -x 10 -i 10 -m 268435456:268435456 -d cuda D D

# OSU MPI-CUDA Bandwidth Test v7.5
# Datatype: MPI_CHAR.
# Size      Bandwidth (MB/s)
268435456          279575.44
$

$ sacct -u user_name -o JobID,JobName,State | tail -2
89               osu_bw  COMPLETED 
89.0             osu_bw  COMPLETED 
$ sacct -j 89 -o JobID,AllocTRES%50
JobID                                                 AllocTRES 
------------ -------------------------------------------------- 
89               billing=2,cpu=2,gres/gpu=2,mem=2000000M,node=1 
89.0                       cpu=2,gres/gpu=2,mem=2000000M,node=1
$

#!/bin/bash
#SBATCH -p sltest
#SBATCH -n 32
#SBATCH -N 1
#SBATCH --gres=gpu:nvidia_a100-sxm4-40gb:4
#SBATCH -o stdout.%J
#SBATCH -e stderr.%J

sleep 10
srun -n 1 nvidia-smi | grep SXM | awk '{print $2, $7}'
srun bash -c 'echo -n "Rank $SLURM_PROCID Node $SLURM_NODEID Core "; taskset -cp $$ | cut -d" " -f6' | sort -k 2n,2

$ for i in `seq 1 2`; do sbatch gpu_affinity.sh; done
Submitted batch job 301
Submitted batch job 302
$ squeue 
             JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)
               301    sltest gpu_affi    usera  R       0:02      1 inst-ao-ub
               302    sltest gpu_affi    usera  R       0:02      1 inst-ao-ub
$ cat stdout.301
0 00000000:0F:00.0
1 00000000:15:00.0
2 00000000:51:00.0
3 00000000:54:00.0
Rank 0 Node 0 Core 0
Rank 1 Node 0 Core 1
:
:
:
Rank 30 Node 0 Core 30
Rank 31 Node 0 Core 31
$ cat stdout.302
0 00000000:8D:00.0
1 00000000:92:00.0
2 00000000:D6:00.0
3 00000000:DA:00.0
Rank 0 Node 0 Core 32
Rank 1 Node 0 Core 33
:
:
:
Rank 30 Node 0 Core 62
Rank 31 Node 0 Core 63
$

$ srun -p sltest -n 33 --gres=gpu:nvidia_a100-sxm4-40gb:4 bash -c "true"
$ srun -p sltest -n 33 --gres=gpu:nvidia_a100-sxm4-40gb:4 --gres-flags=enforce-binding bash -c "true"
srun: error: Unable to allocate resources: Requested node configuration is not available
$

gdoc_arrow_left_alt Slurmによるリソース管理・ジョブ管理システム構築方法 Slurmによるリソース管理・ジョブ管理システム運用Tips gdoc_arrow_right_alt