$ ssh ubuntu@aaa.bbb.ccc.ddd

$ sudo cp -p /etc/apt/apt.conf.d/20auto-upgrades /tmp/
$ sudo sed -i 's/Upgrade "1"/Upgrade "0"/g' /etc/apt/apt.conf.d/20auto-upgrades
$ sudo sed -i 's/Lists "1"/Lists "0"/g' /etc/apt/apt.conf.d/20auto-upgrades
$ sudo diff /tmp/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades
1,2c1,2
< APT::Periodic::Update-Package-Lists "1";
< APT::Periodic::Unattended-Upgrade "1";
---
> APT::Periodic::Update-Package-Lists "0";
> APT::Periodic::Unattended-Upgrade "0";
$ sudo systemctl disable --now unattended-upgrades
Synchronizing state of unattended-upgrades.service with SysV service script with /usr/lib/systemd/systemd-sysv-install.
Executing: /usr/lib/systemd/systemd-sysv-install disable unattended-upgrades
Removed "/etc/systemd/system/multi-user.target.wants/unattended-upgrades.service".
$

$ sudo iptables -P INPUT ACCEPT
$ sudo iptables -P OUTPUT ACCEPT
$ sudo iptables -P FORWARD ACCEPT
$ sudo iptables -F
$ sudo systemctl disable --now iptables

$ sudo systemctl disable --now apparmor

$ distribution=$(. /etc/os-release;echo $ID$VERSION_ID | sed -e 's/\.//g')
$ mkdir ~/`hostname` && cd ~/`hostname` && wget https://developer.download.nvidia.com/compute/cuda/repos/$distribution/x86_64/cuda-keyring_1.1-1_all.deb
$ sudo dpkg -i cuda-keyring_1.1-1_all.deb
$ sudo apt update
$ sudo apt install -y cuda-drivers-575
$ echo nvidia_peermem | sudo tee -a /etc/modules

$ nvidia-smi

$ sudo apt install -y cuda-toolkit-12-9 cmake unzip

$ sudo apt install -y cuda-drivers-fabricmanager-575
$ sudo systemctl enable nvidia-fabricmanager

$ curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg
$ echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list
$ sudo apt update
$ sudo apt install -y nvhpc-25-7-cuda-multi environment-modules
$ sudo cp -p /opt/nvidia/hpc_sdk/modulefiles/nvhpc/25.7 /usr/share/modules/modulefiles/nvhpc

$ sudo apt install -y libssl-dev autoconf libtool zlib1g-dev

$ cd ~/`hostname` && wget https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz
$ tar -xvf ./libevent-2.1.12-stable.tar.gz
$ cd libevent-2.1.12-stable && ./configure --prefix=/opt/libevent
$ make -j 128 && sudo make install

$ cd ~/`hostname` && wget https://download.open-mpi.org/release/hwloc/v2.12/hwloc-2.12.2.tar.gz
$ tar -xvf ./hwloc-2.12.2.tar.gz
$ cd hwloc-2.12.2 && ./configure --prefix=/opt/hwloc --with-cuda=/usr/local/cuda-12.9
$ make -j 128 && sudo make install

$ cd ~/`hostname` && wget https://github.com/openpmix/openpmix/releases/download/v5.0.8/pmix-5.0.8.tar.gz
$ tar -xvf ./pmix-5.0.8.tar.gz
$ cd pmix-5.0.8 && ./configure --prefix=/opt/pmix --with-libevent=/opt/libevent --with-hwloc=/opt/hwloc
$ make -j 128 && sudo make install

$ cd ~/`hostname` && git clone https://github.com/hpc/xpmem.git
$ cd xpmem && ./autogen.sh && ./configure --prefix=/opt/xpmem
$ make -j 128 && sudo make install

$ sudo install -D -m 644 /opt/xpmem/lib/modules/`uname -r`/kernel/xpmem/xpmem.ko /lib/modules/`uname -r`/extra/xpmem.ko
$ sudo depmod -a
$ echo xpmem | sudo tee /etc/modules-load.d/xpmem.conf
$ sudo modprobe xpmem

$ cd ~/`hostname` && wget https://github.com/NVIDIA/gdrcopy/archive/refs/tags/v2.5.1.tar.gz
$ tar -xvf ./v2.5.1.tar.gz
$ cd gdrcopy-2.5.1 && make -j 128 CUDA=/usr/local/cuda-12.9 all && sudo make prefix=/opt/gdrcopy install
$ sudo sed 's/src\/gdrdrv/\/lib\/modules\/`uname -r`\/extra/g' ./insmod.sh | sudo tee /opt/gdrcopy/bin/insmod.sh && sudo chmod 755 /opt/gdrcopy/bin/insmod.sh
$ sudo install -D -m 644 ~/`hostname`/gdrcopy-2.5.1/src/gdrdrv/gdrdrv.ko /lib/modules/`uname -r`/extra/gdrdrv.ko
$ sudo depmod -a

[Unit]
Description=Start gdrcopy

[Service]
ExecStart=/opt/gdrcopy/bin/insmod.sh
Restart=no
Type=oneshot
RemainAfterExit=yes

[Install]
WantedBy=multi-user.target

$ sudo systemctl daemon-reload
$ sudo systemctl enable --now gdrcopy

$ cd ~/`hostname` && wget https://github.com/openucx/ucx/releases/download/v1.19.0/ucx-1.19.0.tar.gz
$ tar -xvf ./ucx-1.19.0.tar.gz
$ cd ucx-1.19.0 && ./contrib/configure-release --prefix=/opt/ucx --with-xpmem=/opt/xpmem --with-cuda=/usr/local/cuda-12.9 -with-gdrcopy=/opt/gdrcopy
$ make -j 128 && sudo make install

$ cd ~/`hostname` && wget https://github.com/openucx/ucc/archive/refs/tags/v1.5.0.tar.gz
$ tar -xvf ./v1.5.0.tar.gz
$ cd ./ucc-1.5.0/ && ./autogen.sh && ./configure --prefix=/opt/ucc --with-ucx=/opt/ucx --with-cuda=/usr/local/cuda-12.9 --with-nccl=/opt/nvidia/hpc_sdk/Linux_x86_64/25.7/comm_libs/nccl
$ make -j 128 && sudo make install

$ cd ~/`hostname` && wget https://download.open-mpi.org/release/open-mpi/v5.0/openmpi-5.0.8.tar.gz
$ tar -xvf ./openmpi-5.0.8.tar.gz
$ . /etc/profile.d/modules.sh
$ module load nvhpc
$ cd openmpi-5.0.8 && ./configure --prefix=/opt/openmpi --with-libevent=/opt/libevent --with-hwloc=/opt/hwloc --with-pmix=/opt/pmix --with-ucx=/opt/ucx --with-ucc=/opt/ucc --with-slurm --with-cuda=/usr/local/cuda-12.9 CC=nvc CXX=nvc++ FC=nvfortran CFLAGS="-mno-hle"

$ diff libtool_org libtool
10560a10561
> 	  export PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/25.7/compilers/bin:${PATH}
$

$ make -j 128 all && sudo make install

#%Module1.0
##
## OpenMPI for NVIDIA compiler 25.7

proc ModulesHelp { } {
        puts stderr "OpenMPI 5.0.8 for NVIDIA compiler 25.7\n"
}

module-whatis   "OpenMPI 5.0.8 for NVIDIA compiler 25.7"

set pkg_root    /opt/openmpi
set ver         5.0.8

setenv MPI_ROOT $pkg_root
setenv MPICC    mpicc
setenv MPICXX   mpicxx
setenv MPIFC    mpif90

prepend-path PATH               $pkg_root/bin:/opt/ucx/bin
prepend-path LD_LIBRARY_PATH    $pkg_root/lib:/opt/gdrcopy/lib
prepend-path LIBRARY_PATH       $pkg_root/lib:/opt/gdrcopy/lib
prepend-path CPATH              $pkg_root/include
prepend-path C_INCLUDE_PATH     $pkg_root/include
prepend-path CPLUS_INCLUDE_PATH $pkg_root/include
prepend-path MANPATH            $pkg_root/share/man

$ mkdir ~/`hostname` && cd ~/`hostname` && wget https://github.com/NVIDIA/cuda-samples/archive/refs/tags/v12.9.zip
$ unzip v12.9.zip
$ export PATH=/usr/local/cuda-12.9/bin:${PATH}
$ cd cuda-samples-12.9 && mkdir build && cd build && cmake .. && make -j 128

$ ./Samples/1_Utilities/deviceQuery/deviceQuery
:
:
deviceQuery, CUDA Driver = CUDART, CUDA Driver Version = 12.9, CUDA Runtime Version = 12.9, NumDevs = 8
Result = PASS
$

#include <stdio.h>
#define N 1000000000
int array[N];
int main() {
#pragma acc parallel loop copy(array[0:N])
   for(int i = 0; i < N; i++) {
      array[i] = 3.0;
   }
   printf("Success!\n");
}

$ module load nvhpc
$ nvc -acc -gpu=cc80 -o gpu.exe test.c
$ ./gpu.exe & sleep 3; nvidia-smi | tail -3
[1] 18096
Success!
|=========================================================================================|
|    0   N/A  N/A           18096      C   ./gpu.exe                              4234MiB |
+-----------------------------------------------------------------------------------------+
$

program main
  use cublas_v2
  implicit none

  integer, parameter :: n = 2
  real(8) :: alpha, beta
  real(8), dimension(n,n) :: a_host, b_host, c_host
  real(8), dimension(n,n), device :: a_dev, b_dev, c_dev
  integer :: i, j
  type(cublashandle) :: handle
  integer :: cublas_status

  alpha = 1.0d0
  beta = 0.0d0
  do j = 1, n
    do i = 1, n
      a_host(i,j) = i + j
      b_host(i,j) = i - j
    end do
  end do

  cublas_status = cublascreate(handle)
  if (cublas_status /= cublas_status_success) then
    print *, 'Error creating cuBLAS handle!'
    stop
  end if
  a_dev = a_host
  b_dev = b_host
  cublas_status = cublasdgemm(handle, cublas_op_n, cublas_op_n, n, n, n, &
                              alpha, a_dev, n, b_dev, n, beta, c_dev, n)
  if (cublas_status /= cublas_status_success) then
    print *, 'Error executing cublasdgemm!'
    stop
  end if
  c_host = c_dev
  cublas_status = cublasdestroy(handle)
  if (cublas_status /= cublas_status_success) then
    print *, 'Error destroying cuBLAS handle!'
    stop
  end if

  print *, 'Matrix c:'
  do i = 1, n
    print *, (c_host(i,j), j = 1, n)
  end do
end program main

$ module load nvhpc
$ nvfortran -cuda -o gpu_blas.exe test.f90
$ ./gpu_blas.exe 
 Matrix c:
    3.000000000000000        -2.000000000000000     
    4.000000000000000        -3.000000000000000
$

$ cd ~/`hostname` && git clone https://github.com/hoshino-UTokyo/lecture_openacc_mpi.git

$ diff Makefile_org Makefile
8c8
< CFLAGS    = -O3 -acc -Minfo=accel  -ta=tesla,cc80
---
> CFLAGS    = -O3 -acc -Minfo=accel  -gpu=cc80
$

$ cd ~/`hostname`/lecture_openacc_mpi/C/openacc_mpi_basic/04_cuda_aware
$ module purge
$ module load nvhpc openmpi
$ make
mpicc -O3 -acc -Minfo=accel  -gpu=cc80 -c main.c
"main.c", line 57: warning: The independent loop parallelism with no parallelism level is set to seq when inferring the routine parallelism of the enclosing function [independent_loop_type]
  #pragma acc loop independent
  ^

Remark: individual warnings can be suppressed with "--diag_suppress <warning-name>"

main:
     54, Generating create(a[:16777216],b[:16777216]) [if not already present]
         Generating copyout(b[:16777216],a[:16777216]) [if not already present]
     58, Loop is parallelizable
         Generating NVIDIA GPU code
         58, #pragma acc loop gang, vector(128) /* blockIdx.x threadIdx.x */
     72, Generating implicit copy(sum) [if not already present]
         Generating copyin(b[:16777216]) [if not already present]
     76, Loop is parallelizable
         Generating NVIDIA GPU code
         76, #pragma acc loop gang, vector(128) /* blockIdx.x threadIdx.x */
             Generating reduction(+:sum)
mpicc -O3 -acc -Minfo=accel  -gpu=cc80  main.o   -o run
$ mpirun -n 2 -x UCX_NET_DEVICES=mlx5_4:1 ./run
[inst-v6xda-ao-ub24:17765] SET UCX_NET_DEVICES=mlx5_4:1
num of GPUs = 8
Rank 0: hostname = inst-v6xda-ao-ub24, GPU num = 0
Rank 1: hostname = inst-v6xda-ao-ub24, GPU num = 1
mean = 30.00
Time =    0.007 [sec]
$