
#  yum install infiniband-diags
#  yum install libibverbs
#  yum install opa-address-resolution
#  yum install opa-fastfabric opa-libopamgt perftest qperf srp_daemon glusterfs-rdma

#  yum install ibacm ibutils       libibverbs-utils librdmacm-utils mstflint     
#  yum install dapl
#  yum install dapl-utils 

See also /boot/config*  or   rdma-core config of modules *_RXE

#  strings /usr/lib64/libibverbs/librxe-rdmav17.so | grep ibv_

##################################################################################################

00:34 root@HPMC9: ~ #  ls -l /sys/class/infiniband_verbs/uverbs3
lrwxrwxrwx 1 root root 0 Sep  6 00:34 /sys/class/infiniband_verbs/uverbs3 -> ../../devices/pci0000:17/0000:17:00.0/0000:18:00.0/0000:19:03.0/0000:1a:00.0/infiniband_verbs/uverbs3


00:34 root@HPMC9: ~ #  lsmod | grep ib_uverbs
ib_uverbs              64636  2 ib_ucm,rdma_ucm
ib_core               211874  16 rdma_cm,i40iw,ib_cm,iw_cm,rpcrdma,ib_qib,ib_srp,ib_ucm,rdmavt,ib_iser,ib_srpt,ib_umad,ib_uverbs,rdma_ucm,ib_ipoib

00:35 root@HPMC9: ~ #  systemctl list-unit-files    

00:36 root@HPMC9: ~ #  systemctl list-unit-files    | grep open
opensm.service                                disabled

00:36 root@HPMC9: ~ #  systemctl enable opensm
Created symlink from /etc/systemd/system/network.target.wants/opensm.service to /usr/lib/systemd/system/opensm.service.

00:36 root@HPMC9: ~ #  systemctl start opensm 

00:36 root@HPMC9: ~ #  systemctl list-unit-files    | grep open
opensm.service                                enabled 

00:36 root@HPMC9: ~ #  systemctl status opensm                 
● opensm.service - Starts the OpenSM InfiniBand fabric Subnet Manager
   Loaded: loaded (/usr/lib/systemd/system/opensm.service; enabled; vendor preset: disabled)
   Active: active (running) since Tue 2022-09-06 00:36:54 EDT; 14s ago
     Docs: man:opensm
  Process: 30288 ExecStart=/usr/libexec/opensm-launch (code=exited, status=0/SUCCESS)
 Main PID: 30289 (opensm-launch)
    Tasks: 120
   CGroup: /system.slice/opensm.service
           ├─30289 /bin/bash /usr/libexec/opensm-launch
           └─30290 /usr/sbin/opensm

Sep 06 00:36:55 HPMC9 opensm-launch[30288]: -------------------------------------------------
Sep 06 00:36:55 HPMC9 OpenSM[30290]: /var/log/opensm.log log file opened
Sep 06 00:36:55 HPMC9 OpenSM[30290]: OpenSM 3.3.20
Sep 06 00:36:55 HPMC9 opensm-launch[30288]: OpenSM 3.3.20
Sep 06 00:36:55 HPMC9 OpenSM[30290]: Entering DISCOVERING state
Sep 06 00:36:55 HPMC9 opensm-launch[30288]: Entering DISCOVERING state
Sep 06 00:36:55 HPMC9 OpenSM[30290]: Entering STANDBY state
Sep 06 00:36:55 HPMC9 opensm-launch[30288]: Entering STANDBY state
Sep 06 00:36:55 HPMC9 OpenSM[30290]: Entering MASTER state
Sep 06 00:36:55 HPMC9 opensm-launch[30288]: Entering MASTER state


00:37 root@HPMC9: ~ #  ls -l /sys/class/infiniband_verbs/uverbs3
lrwxrwxrwx 1 root root 0 Sep  6 00:34 /sys/class/infiniband_verbs/uverbs3 -> ../../devices/pci0000:17/0000:17:00.0/0000:18:00.0/0000:19:03.0/0000:1a:00.0/infiniband_verbs/uverbs3

00:37 root@HPMC9: ~ #  yum install infiniband-diags

00:38 root@HPMC9: ~ #  yum install libibverbs

 libibverbs                       
 rdma-core-devel         
 ibacm        
 libibverbs-utils    
 librdmacm         
 librdmacm-utils   


00:40 root@HPMC9: ~ #  ibv_devices 
    device                 node GUID
    ------              ----------------
    qib0                00117500006eef26
    qib1                0011750000794fd8
    i40iw0              002590bc3ab90000
    i40iw1              002590bc3ab80000

00:46 root@HPMC9: ~ #  ibv_devinfo
hca_id: qib0
        transport:                      InfiniBand (0)
        fw_ver:                         0.0.0
        node_guid:                      0011:7500:006e:ef26
        sys_image_guid:                 0011:7500:006e:ef26
        vendor_id:                      0x1175
        vendor_part_id:                 29474
        hw_ver:                         0x2
        board_id:                       InfiniPath_QLE7340
        phys_port_cnt:                  1
                port:   1
                        state:                  PORT_ACTIVE (4)
                        max_mtu:                4096 (5)
                        active_mtu:             2048 (4)
                        sm_lid:                 1
                        port_lid:               1
                        port_lmc:               0x00
                        link_layer:             InfiniBand

hca_id: qib1
        transport:                      InfiniBand (0)
        fw_ver:                         0.0.0
        node_guid:                      0011:7500:0079:4fd8
        sys_image_guid:                 0011:7500:006e:ef26
        vendor_id:                      0x1175
        vendor_part_id:                 29474
        hw_ver:                         0x2
        board_id:                       InfiniPath_QLE7340
        phys_port_cnt:                  1
                port:   1
                        state:                  PORT_ACTIVE (4)
                        max_mtu:                4096 (5)
                        active_mtu:             2048 (4)
                        sm_lid:                 1
                        port_lid:               2
                        port_lmc:               0x00
                        link_layer:             InfiniBand

hca_id: i40iw0
        transport:                      iWARP (1)
        fw_ver:                         0.2
        node_guid:                      0025:90bc:3ab9:0000
        sys_image_guid:                 0025:90bc:3ab9:0000
        vendor_id:                      0x8086
        vendor_part_id:                 14290
        hw_ver:                         0x0
        board_id:                       I40IW Board ID
        phys_port_cnt:                  1
                port:   1
                        state:                  PORT_DOWN (1)
                        max_mtu:                4096 (5)
                        active_mtu:             1024 (3)
                        sm_lid:                 0
                        port_lid:               1
                        port_lmc:               0x00
                        link_layer:             Ethernet

hca_id: i40iw1
        transport:                      iWARP (1)
        fw_ver:                         0.2
        node_guid:                      0025:90bc:3ab8:0000
        sys_image_guid:                 0025:90bc:3ab8:0000
        vendor_id:                      0x8086
        vendor_part_id:                 14290
        hw_ver:                         0x0
        board_id:                       I40IW Board ID
        phys_port_cnt:                  1
                port:   1
                        state:                  PORT_ACTIVE (4)
                        max_mtu:                4096 (5)
                        active_mtu:             1024 (3)
                        sm_lid:                 0
                        port_lid:               1
                        port_lmc:               0x00
                        link_layer:             Ethernet

00:48 root@HPMC9: ~ #  yum install libverbs

00:49 root@HPMC9: ~ #  yum install opa-address-resolution

00:51 root@HPMC9: ~ #  yum install opa-fastfabric opa-libopamgt perftest qperf srp_daemon glusterfs-rdma

00:51 root@HPMC9: ~ #  yum install ibacm ibutils iwpmd libibverbs-utils librdmacm-utils mstflint

(1/4): ibutils-1.5.7-13.el7.x86_64.rpm         
(2/4): ibutils-libs-1.5.7-13.el7.x86_64.rpm   
(3/4): iwpmd-17.2-3.el7.x86_64.rpm           
(4/4): mstflint-4.9.0-3.el7.x86_64.rpm      

00:52 root@HPMC9: ~ #  yum install ibacm ibutils  libibverbs-utils librdmacm-utils mstflint     


01:12 root@HPMC9: ~ #  systemctl start mpss

01:13 root@HPMC9: ~ #  yum install dapl-utils

01:24 root@HPMC9: ~ #  rpm -i /OFED/dapl-2.1.9-2.glibc2.12.x86_64.rpm
        package dapl-2.1.9-2.glibc2.12.x86_64 is already installed

01:24 root@HPMC9: ~ #  vi /etc/mpxyd.conf

01:34 root@HPMC9: / #  man dapltest
01:34 root@HPMC9: / #  man dtest
01:34 root@HPMC9: / #  dtest
23108 Running as server - ofa-v2-mlx4_0-1u v2 
23108 Local Address AF_INET6 - fe80::11:7500:6e:ef26 flowinfo(QPN)=0x19, port(LID)=0x1
23108 Server is waiting for client connection to send server info
23108 Server waiting for connect request on port b0c0
23108 Waiting for connect response

23108 CONNECTED!

23108 Send RMR msg to remote: r_key_ctx=0xc0d00,va=0x7f5e41be4000,len=0x400000
23108 remote RMR data arrived!
23108 Received RMR from remote: r_iov: r_key_ctx=121300,va=7fb51e5f4000,len=0x400000

23108 Query EP: LOCAL addr 0.0.0.25 port 0
23108 Query EP: REMOTE addr 0.0.0.33 port 0

 23108 RDMA WRITE DATA with SEND MSG

23108 inbound rdma_write; send message arrived!
23108 Sending RDMA WRITE completion message
23108 Received RMR from remote: r_iov: r_key_ctx=121300,va=7fb51e5f4000,len=0x400000
23108 SERVER: RDMA write buffer contains: client RDMA write data...

 23108 RDMA READ DATA with SEND MSG

23108 Sending RDMA read completion message
23108 Waiting for inbound message....
23108 inbound rdma_read; send message arrived!
23108 Received RMR from remote: r_iov: r_key_ctx=121300,va=7fb51e5f4000,len=0x400000
23108 SERVER: RCV RDMA read buffer contains: client RDMA read data...

 23108 PING DATA with SEND MSG

23108: Message PingPong: (100 x 4194304) Total 218527.79 us: latency 1092.64 us, BW 3838.69 MB/s

23108: RDMA write (bi-direction): Total=442818.16 usec, itime=221.41 us, poll=0, 2000 x 4194304, 18943.69 MB/sec

23108: DAPL Test Complete. PASSED

01:36 root@HPMC9: / #  dtestcm
 Running server on ofa-v2-ib0 with 1000 RC connections
23248 Local IA Address - 192.168.0.132 port 45248
 Server ready on port 45248
23248 Conn server waiting for client, port 15828 
 Accepting...
^C^C^C

