1 Reply Latest reply on Dec 12, 2018 6:47 AM by talatb

    infiniband interfaces are missing after server reboot

      OS- centos 7

       

      100G interfaces are missing after server reboot

       

      [root@ceph-deploy MLNX_OFED_LINUX-4.4-2.0.7.0-rhel7.5-x86_64]# ifconfig

      eno1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500

              inet 172.22.131.174  netmask 255.255.255.0  broadcast 172.22.131.255

              inet6 fe80::8dc0:ff5e:b70:883  prefixlen 64  scopeid 0x20<link>

              ether 28:ac:9e:83:41:aa  txqueuelen 1000  (Ethernet)

              RX packets 1461  bytes 122059 (119.1 KiB)

              RX errors 0  dropped 0  overruns 0  frame 0

              TX packets 1125  bytes 217097 (212.0 KiB)

              TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

       

      eno2: flags=4099<UP,BROADCAST,MULTICAST>  mtu 1500

              ether 28:ac:9e:83:41:ab  txqueuelen 1000  (Ethernet)

              RX packets 0  bytes 0 (0.0 B)

              RX errors 0  dropped 0  overruns 0  frame 0

              TX packets 0  bytes 0 (0.0 B)

              TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

       

      enp216s0f0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500

              ether 90:e2:ba:f6:fb:7c  txqueuelen 1000  (Ethernet)

              RX packets 2  bytes 120 (120.0 B)

              RX errors 0  dropped 0  overruns 0  frame 0

              TX packets 0  bytes 0 (0.0 B)

              TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

       

      enp216s0f1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500

              ether 90:e2:ba:f6:fb:7d  txqueuelen 1000  (Ethernet)

              RX packets 2  bytes 120 (120.0 B)

              RX errors 0  dropped 0  overruns 0  frame 0

              TX packets 0  bytes 0 (0.0 B)

              TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

       

      lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536

              inet 127.0.0.1  netmask 255.0.0.0

              inet6 ::1  prefixlen 128  scopeid 0x10<host>

              loop  txqueuelen 1000  (Local Loopback)

              RX packets 184  bytes 17698 (17.2 KiB)

              RX errors 0  dropped 0  overruns 0  frame 0

              TX packets 184  bytes 17698 (17.2 KiB)

              TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

       

      virbr0: flags=4099<UP,BROADCAST,MULTICAST>  mtu 1500

              inet 192.168.122.1  netmask 255.255.255.0  broadcast 192.168.122.255

              ether 52:54:00:41:2e:ad  txqueuelen 1000  (Ethernet)

              RX packets 0  bytes 0 (0.0 B)

              RX errors 0  dropped 0  overruns 0  frame 0

              TX packets 0  bytes 0 (0.0 B)

              TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

       

      [root@ceph-deploy MLNX_OFED_LINUX-4.4-2.0.7.0-rhel7.5-x86_64]# lspci | grep Infiniband

      [root@ceph-deploy MLNX_OFED_LINUX-4.4-2.0.7.0-rhel7.5-x86_64]# nmcli connection show

      NAME        UUID                                  TYPE        DEVICE

      eno1        f841eea4-e685-4953-ab95-6b41c7c72718  ethernet    eno1  

      virbr0      aa390664-9fa4-4908-9ffc-3e4b6bc9c1d7  bridge      virbr0

      eno2        142c810e-5ad2-41ec-aa7c-c7e13accc83d  ethernet    --    

      enp216s0f0  8a00f6e1-b2bc-47a4-bd71-4c195c9e55ff  ethernet    --    

      enp216s0f1  e8802335-f41d-4d15-ad6c-891b89a324b4  ethernet    --    

      ib0         aebb25e7-519c-4171-b146-2b7562b2975e  infiniband  --    

      ib1         d4917106-71cd-4f8c-9fd8-03e585c526fe  infiniband  --    

      [root@ceph-deploy MLNX_OFED_LINUX-4.4-2.0.7.0-rhel7.5-x86_64]# lsmod | grep mlx

      [root@ceph-deploy MLNX_OFED_LINUX-4.4-2.0.7.0-rhel7.5-x86_64]#

       

       

      [root@ceph-deploy MLNX_OFED_LINUX-4.4-2.0.7.0-rhel7.5-x86_64]# ./mlnxofedinstall --add-kernel-support

      Current operation system is not supported!

       

       

      [root@ceph-deploy network-scripts]# ls

      ifcfg-eno1        ifcfg-ib0  ifdown-bnep  ifdown-isdn    ifdown-sit       ifup          ifup-ippp  ifup-plusb   ifup-sit       ifup-wireless

      ifcfg-eno2        ifcfg-ib1  ifdown-eth   ifdown-post    ifdown-Team      ifup-aliases  ifup-ipv6  ifup-post    ifup-Team      init.ipv6-global

      ifcfg-enp216s0f0  ifcfg-lo   ifdown-ippp  ifdown-ppp     ifdown-TeamPort  ifup-bnep     ifup-isdn  ifup-ppp     ifup-TeamPort  network-functions

      ifcfg-enp216s0f1  ifdown     ifdown-ipv6  ifdown-routes  ifdown-tunnel    ifup-eth      ifup-plip  ifup-routes  ifup-tunnel    network-functions-ipv6

      [root@ceph-deploy network-scripts]#

       

      [root@ceph-deploy ~]# lsmod | grep mlx

      [root@ceph-deploy ~]# lsmod

      Module                  Size  Used by

      mst_pciconf            99312  0

      xt_CHECKSUM            12549  1

      iptable_mangle         12695  1

      ipt_MASQUERADE         12678  3

      nf_nat_masquerade_ipv4    13412  1 ipt_MASQUERADE

      iptable_nat            12875  1

      nf_nat_ipv4            14115  1 iptable_nat

      nf_nat                 26787  2 nf_nat_ipv4,nf_nat_masquerade_ipv4

      nf_conntrack_ipv4      15053  2

      nf_defrag_ipv4         12729  1 nf_conntrack_ipv4

      xt_conntrack           12760  1

      nf_conntrack          133095  5 nf_nat,nf_nat_ipv4,xt_conntrack,nf_nat_masquerade_ipv4,nf_conntrack_ipv4

      ipt_REJECT             12541  2

      nf_reject_ipv4         13373  1 ipt_REJECT

      tun                    31740  1

      bridge                151336  0

      stp                    12976  1 bridge

      llc                    14552  2 stp,bridge

      ebtable_filter         12827  0

      ebtables               35009  1 ebtable_filter

      ip6table_filter        12815  0

      ip6_tables             26912  1 ip6table_filter

      iptable_filter         12810  1

      sunrpc                353352  1

      skx_edac               13313  0

      intel_powerclamp       14419  0

      coretemp               13444  0

      intel_rapl             19542  0

      iosf_mbi               15582  1 intel_rapl

      kvm_intel             183621  0

      kvm                   586948  1 kvm_intel

      irqbypass              13503  1 kvm

      crc32_pclmul           13133  0

      ghash_clmulni_intel    13273  0

      aesni_intel           189414  0

      lrw                    13286  1 aesni_intel

      gf128mul               15139  1 lrw

      glue_helper            13990  1 aesni_intel

      ablk_helper            13597  1 aesni_intel

      cryptd                 21190  3 ghash_clmulni_intel,aesni_intel,ablk_helper

      joydev                 17389  0

      pcspkr                 12718  0

      ses                    17546  0

      enclosure              15699  1 ses

      sg                     40721  0

      ipmi_ssif              29487  0

      ipmi_si                59571  0

      ipmi_devintf           17603  0

      wmi                    21636  0

      ipmi_msghandler        56032  3 ipmi_ssif,ipmi_devintf,ipmi_si

      acpi_power_meter       18104  0

      acpi_pad              116284  0

      mei_me                 32848  0

      lpc_ich                21086  0

      mei                    91150  1 mei_me

      tcp_htcp               12999  13

      knem                   36921  0

      ip_tables              27126  3 iptable_filter,iptable_mangle,iptable_nat

      xfs                   997127  3

      libcrc32c              12644  3 xfs,nf_nat,nf_conntrack

      sd_mod                 46281  3

      crc_t10dif             12912  1 sd_mod

      crct10dif_generic      12647  0

      mgag200                41053  2

      i2c_algo_bit           13413  1 mgag200

      drm_kms_helper        179394  1 mgag200

      syscopyarea            12529  1 drm_kms_helper

      sysfillrect            12701  1 drm_kms_helper

      sysimgblt              12640  1 drm_kms_helper

      crct10dif_pclmul       14307  1

      fb_sys_fops            12703  1 drm_kms_helper

      crct10dif_common       12595  3 crct10dif_pclmul,crct10dif_generic,crc_t10dif

      uas                    22445  0

      crc32c_intel           22094  1

      ttm                   114635  1 mgag200

      drm                   429744  5 ttm,drm_kms_helper,mgag200

      ixgbe                 318969  0

      usb_storage            66718  1 uas

      ahci                   34056  0

      mpt3sas               226872  2

      libahci                31992  1 ahci

      devlink                48345  0

      nvme                   32338  0

      mdio                   14073  1 ixgbe

      libata                243133  2 ahci,libahci

      ptp                    19231  1 ixgbe

      raid_class             13554  1 mpt3sas

      nvme_core              58852  1 nvme

      drm_panel_orientation_quirks    12957  1 drm

      pps_core               19057  1 ptp

      scsi_transport_sas     41224  2 ses,mpt3sas

      dca                    15130  1 ixgbe

      nfit                   55016  1 skx_edac

      libnvdimm             147731  1 nfit

      dm_mirror              22289  0

      dm_region_hash         20813  1 dm_mirror

      dm_log                 18411  2 dm_region_hash,dm_mirror

      dm_mod                124407  11 dm_log,dm_mirror

      [root@ceph-deploy ~]# lsmod | grp mls

       

       

      thanks

      Sudhakar