############ BASELINE Mon Jul 27 03:56:42 PM UTC 2026 ############
### uname
Linux sasquatch 6.17.0-35-generic #35~24.04.1-Ubuntu SMP PREEMPT_DYNAMIC Tue May 26 19:30:42 UTC 2 x86_64 x86_64 x86_64 GNU/Linux

### /proc/cmdline
BOOT_IMAGE=/boot/vmlinuz-6.17.0-35-generic root=UUID=a3f2987d-7650-45d5-bbfa-b321e073dc59 ro quiet splash vt.handoff=7

### nvidia-smi topo -m
	[4mGPU0	GPU1	GPU2	GPU3	GPU4	GPU5	GPU6	GPU7	GPU8	CPU Affinity	NUMA Affinity	GPU NUMA ID[0m
GPU0	 X 	PIX	NODE	NODE	SYS	SYS	SYS	SYS	SYS	0-31,128-159	0		N/A
GPU1	PIX	 X 	NODE	NODE	SYS	SYS	SYS	SYS	SYS	0-31,128-159	0		N/A
GPU2	NODE	NODE	 X 	PIX	SYS	SYS	SYS	SYS	SYS	0-31,128-159	0		N/A
GPU3	NODE	NODE	PIX	 X 	SYS	SYS	SYS	SYS	SYS	0-31,128-159	0		N/A
GPU4	SYS	SYS	SYS	SYS	 X 	PIX	NODE	NODE	NODE	64-95,192-223	2		N/A
GPU5	SYS	SYS	SYS	SYS	PIX	 X 	NODE	NODE	NODE	64-95,192-223	2		N/A
GPU6	SYS	SYS	SYS	SYS	NODE	NODE	 X 	PIX	PIX	64-95,192-223	2		N/A
GPU7	SYS	SYS	SYS	SYS	NODE	NODE	PIX	 X 	PIX	64-95,192-223	2		N/A
GPU8	SYS	SYS	SYS	SYS	NODE	NODE	PIX	PIX	 X 	64-95,192-223	2		N/A

Legend:

  X    = Self
  SYS  = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI)
  NODE = Connection traversing PCIe as well as the interconnect between PCIe Host Bridges within a NUMA node
  PHB  = Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU)
  PXB  = Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge)
  PIX  = Connection traversing at most a single PCIe bridge
  NV#  = Connection traversing a bonded set of # NVLinks

### nvidia-smi topo -p2p r
 	[4mGPU0	GPU1	GPU2	GPU3	GPU4	GPU5	GPU6	GPU7	GPU8	[0m
 GPU0	X	OK	OK	OK	OK	OK	OK	OK	OK	
 GPU1	OK	X	OK	OK	OK	OK	OK	OK	OK	
 GPU2	OK	OK	X	OK	OK	OK	OK	OK	OK	
 GPU3	OK	OK	OK	X	OK	OK	OK	OK	OK	
 GPU4	OK	OK	OK	OK	X	OK	OK	OK	OK	
 GPU5	OK	OK	OK	OK	OK	X	OK	OK	OK	
 GPU6	OK	OK	OK	OK	OK	OK	X	OK	OK	
 GPU7	OK	OK	OK	OK	OK	OK	OK	X	OK	
 GPU8	OK	OK	OK	OK	OK	OK	OK	OK	X	

Legend:

  X    = Self
  OK   = Status Ok
  CNS  = Chipset not supported
  GNS  = GPU not supported
  TNS  = Topology not supported
  NS   = Not supported
  U    = Unknown

### nvidia-smi topo -p2p w
 	[4mGPU0	GPU1	GPU2	GPU3	GPU4	GPU5	GPU6	GPU7	GPU8	[0m
 GPU0	X	OK	OK	OK	OK	OK	OK	OK	OK	
 GPU1	OK	X	OK	OK	OK	OK	OK	OK	OK	
 GPU2	OK	OK	X	OK	OK	OK	OK	OK	OK	
 GPU3	OK	OK	OK	X	OK	OK	OK	OK	OK	
 GPU4	OK	OK	OK	OK	X	OK	OK	OK	OK	
 GPU5	OK	OK	OK	OK	OK	X	OK	OK	OK	
 GPU6	OK	OK	OK	OK	OK	OK	X	OK	OK	
 GPU7	OK	OK	OK	OK	OK	OK	OK	X	OK	
 GPU8	OK	OK	OK	OK	OK	OK	OK	OK	X	

Legend:

  X    = Self
  OK   = Status Ok
  CNS  = Chipset not supported
  GNS  = GPU not supported
  TNS  = Topology not supported
  NS   = Not supported
  U    = Unknown

### nvidia-smi topo -p2p a
 	[4mGPU0	GPU1	GPU2	GPU3	GPU4	GPU5	GPU6	GPU7	GPU8	[0m
 GPU0	X	NS	NS	NS	NS	NS	NS	NS	NS	
 GPU1	NS	X	NS	NS	NS	NS	NS	NS	NS	
 GPU2	NS	NS	X	NS	NS	NS	NS	NS	NS	
 GPU3	NS	NS	NS	X	NS	NS	NS	NS	NS	
 GPU4	NS	NS	NS	NS	X	NS	NS	NS	NS	
 GPU5	NS	NS	NS	NS	NS	X	NS	NS	NS	
 GPU6	NS	NS	NS	NS	NS	NS	X	NS	NS	
 GPU7	NS	NS	NS	NS	NS	NS	NS	X	NS	
 GPU8	NS	NS	NS	NS	NS	NS	NS	NS	X	

Legend:

  X    = Self
  OK   = Status Ok
  CNS  = Chipset not supported
  GNS  = GPU not supported
  TNS  = Topology not supported
  NS   = Not supported
  U    = Unknown

### nvidia-smi topo -p2p p
 	[4mGPU0	GPU1	GPU2	GPU3	GPU4	GPU5	GPU6	GPU7	GPU8	[0m
 GPU0	X	OK	OK	OK	OK	OK	OK	OK	OK	
 GPU1	OK	X	OK	OK	OK	OK	OK	OK	OK	
 GPU2	OK	OK	X	OK	OK	OK	OK	OK	OK	
 GPU3	OK	OK	OK	X	OK	OK	OK	OK	OK	
 GPU4	OK	OK	OK	OK	X	OK	OK	OK	OK	
 GPU5	OK	OK	OK	OK	OK	X	OK	OK	OK	
 GPU6	OK	OK	OK	OK	OK	OK	X	OK	OK	
 GPU7	OK	OK	OK	OK	OK	OK	OK	X	OK	
 GPU8	OK	OK	OK	OK	OK	OK	OK	OK	X	

Legend:

  X    = Self
  OK   = Status Ok
  CNS  = Chipset not supported
  GNS  = GPU not supported
  TNS  = Topology not supported
  NS   = Not supported
  U    = Unknown

### IOMMU group type per GPU
0000:2b:00.0 -> DMA-FQ (group 37)
0000:2d:00.0 -> DMA-FQ (group 39)
0000:3a:00.0 -> DMA-FQ (group 21)
0000:3b:00.0 -> DMA-FQ (group 22)
0000:ad:00.0 -> DMA-FQ (group 80)
0000:ae:00.0 -> DMA-FQ (group 81)
0000:bc:00.0 -> DMA-FQ (group 65)
0000:bd:00.0 -> DMA-FQ (group 66)
0000:be:00.0 -> DMA-FQ (group 67)

### lspci tree (nvidia)
 |           +-00.4  Intel Corporation Device 0b23
 |           \-01.0-[27-2d]----00.0-[28-2d]--+-00.0-[29]----00.0  Broadcom / LSI Virtual PCIe Placeholder Endpoint
 |                                           +-01.0-[2a]----00.0  Broadcom / LSI Virtual PCIe Placeholder Endpoint
 |                                           +-02.0-[2b]----00.0  NVIDIA Corporation GA100GL [A30 PCIe]
 |                                           +-03.0-[2c]----00.0  Broadcom / LSI Virtual PCIe Placeholder Endpoint
 |                                           \-04.0-[2d]----00.0  NVIDIA Corporation GA100GL [A30 PCIe]
--
 |           +-00.1  Intel Corporation Ice Lake Mesh 2 PCIe
 |           +-00.2  Intel Corporation Ice Lake RAS
 |           +-00.4  Intel Corporation Device 0b23
 |           \-01.0-[38-3f]----00.0-[39-3f]--+-00.0-[3a]----00.0  NVIDIA Corporation GA100GL [A30 PCIe]
 |                                           +-01.0-[3b]----00.0  NVIDIA Corporation GA100GL [A30 PCIe]
--
 |           \-01.0-[a8-ae]----00.0-[a9-ae]--+-00.0-[aa]----00.0  Broadcom / LSI Virtual PCIe Placeholder Endpoint
 |                                           +-01.0-[ab]----00.0  Broadcom / LSI Virtual PCIe Placeholder Endpoint
 |                                           +-02.0-[ac]----00.0  Broadcom / LSI Virtual PCIe Placeholder Endpoint
 |                                           +-03.0-[ad]----00.0  NVIDIA Corporation GA100GL [A30 PCIe]
 |                                           \-04.0-[ae]----00.0  NVIDIA Corporation GA100GL [A30 PCIe]
--
 |           +-00.4  Intel Corporation Device 0b23
 |           \-01.0-[b8-bf]----00.0-[b9-bf]--+-00.0-[ba]----00.0  Broadcom / LSI Virtual PCIe Placeholder Endpoint
 |                                           +-01.0-[bb]----00.0  Broadcom / LSI Virtual PCIe Placeholder Endpoint
 |                                           +-02.0-[bc]----00.0  NVIDIA Corporation GA100GL [A30 PCIe]
 |                                           +-03.0-[bd]----00.0  NVIDIA Corporation GA100GL [A30 PCIe]
 |                                           +-04.0-[be]----00.0  NVIDIA Corporation GA100GL [A30 PCIe]

### nvidia-smi -q (full)

==============NVSMI LOG==============

Timestamp                                 : Mon Jul 27 11:56:44 2026
Driver Version                            : 535.309.01
CUDA Version                              : 12.2

Attached GPUs                             : 9
GPU 00000000:2B:00.0
    Product Name                          : NVIDIA A30
    Product Brand                         : NVIDIA
    Product Architecture                  : Ampere
    Display Mode                          : Enabled
    Display Active                        : Disabled
    Persistence Mode                      : Disabled
    Addressing Mode                       : None
    MIG Mode
        Current                           : Disabled
        Pending                           : Disabled
    Accounting Mode                       : Disabled
    Accounting Mode Buffer Size           : 4000
    Driver Model
        Current                           : N/A
        Pending                           : N/A
    Serial Number                         : 1655223001202
    GPU UUID                              : GPU-b059c845-ec1b-1f6f-b45d-836cb582bafb
    Minor Number                          : 0
    VBIOS Version                         : 92.00.66.00.04
    MultiGPU Board                        : No
    Board ID                              : 0x2b00
    Board Part Number                     : 900-21001-0040-100
    GPU Part Number                       : 20B7-882-A1
    FRU Part Number                       : N/A
    Module ID                             : 1
    Inforom Version
        Image Version                     : 1001.0205.00.02
        OEM Object                        : 2.0
        ECC Object                        : 6.16
        Power Management Object           : N/A
    Inforom BBX Object Flush
        Latest Timestamp                  : N/A
        Latest Duration                   : N/A
    GPU Operation Mode
        Current                           : N/A
        Pending                           : N/A
    GSP Firmware Version                  : 535.309.01
    GPU Virtualization Mode
        Virtualization Mode               : None
        Host VGPU Mode                    : N/A
    GPU Reset Status
        Reset Required                    : No
        Drain and Reset Recommended       : No
    IBMNPU
        Relaxed Ordering Mode             : N/A
    PCI
        Bus                               : 0x2B
        Device                            : 0x00
        Domain                            : 0x0000
        Device Id                         : 0x20B710DE
        Bus Id                            : 00000000:2B:00.0
        Sub System Id                     : 0x153210DE
        GPU Link Info
            PCIe Generation
                Max                       : 4
                Current                   : 4
                Device Current            : 4
                Device Max                : 4
                Host Max                  : 5
            Link Width
                Max                       : 16x
                Current                   : 16x
        Bridge Chip
            Type                          : N/A
            Firmware                      : N/A
        Replays Since Reset               : 0
        Replay Number Rollovers           : 0
        Tx Throughput                     : 0 KB/s
        Rx Throughput                     : 0 KB/s
        Atomic Caps Inbound               : N/A
        Atomic Caps Outbound              : N/A
    Fan Speed                             : N/A
    Performance State                     : P0
    Clocks Event Reasons
        Idle                              : Active
        Applications Clocks Setting       : Not Active
        SW Power Cap                      : Not Active
        HW Slowdown                       : Not Active
            HW Thermal Slowdown           : Not Active
            HW Power Brake Slowdown       : Not Active
        Sync Boost                        : Not Active
        SW Thermal Slowdown               : Not Active
        Display Clock Setting             : Not Active
    Sparse Operation Mode                 : N/A
    FB Memory Usage
        Total                             : 24576 MiB
        Reserved                          : 513 MiB
        Used                              : 12 MiB
        Free                              : 24049 MiB
    BAR1 Memory Usage
        Total                             : 32768 MiB
        Used                              : 2 MiB
        Free                              : 32766 MiB
    Conf Compute Protected Memory Usage
        Total                             : 0 MiB
        Used                              : 0 MiB
        Free                              : 0 MiB
    Compute Mode                          : Default
    Utilization
        Gpu                               : 0 %
        Memory                            : 0 %
        Encoder                           : 0 %
        Decoder                           : 0 %
        JPEG                              : 0 %
        OFA                               : 0 %
    Encoder Stats
        Active Sessions                   : 0
        Average FPS                       : 0
        Average Latency                   : 0
    FBC Stats
        Active Sessions                   : 0
        Average FPS                       : 0
        Average Latency                   : 0
    ECC Mode
        Current                           : Enabled
        Pending                           : Enabled
    ECC Errors
        Volatile
            SRAM Correctable              : 0
            SRAM Uncorrectable Parity     : 0
            SRAM Uncorrectable SEC-DED    : 0
            DRAM Correctable              : 0
            DRAM Uncorrectable            : 0
        Aggregate
            SRAM Correctable              : 0
            SRAM Uncorrectable Parity     : 0
            SRAM Uncorrectable SEC-DED    : 0
            DRAM Correctable              : 0
            DRAM Uncorrectable            : 0
            SRAM Threshold Exceeded       : No
        Aggregate Uncorrectable SRAM Sources
            SRAM L2                       : 0
            SRAM SM                       : 0
            SRAM Microcontroller          : 0
            SRAM PCIE                     : 0
            SRAM Other                    : 0
    Retired Pages
        Single Bit ECC                    : N/A
        Double Bit ECC                    : N/A
        Pending Page Blacklist            : N/A
    Remapped Rows
        Correctable Error                 : 0
        Uncorrectable Error               : 0
        Pending                           : No
        Remapping Failure Occurred        : No
        Bank Remap Availability Histogram
            Max                           : 384 bank(s)
            High                          : 0 bank(s)
            Partial                       : 0 bank(s)
            Low                           : 0 bank(s)
            None                          : 0 bank(s)
    Temperature
        GPU Current Temp                  : 37 C
        GPU T.Limit Temp                  : N/A
        GPU Shutdown Temp                 : 100 C
        GPU Slowdown Temp                 : 97 C
        GPU Max Operating Temp            : 90 C
        GPU Target Temperature            : N/A
        Memory Current Temp               : 35 C
        Memory Max Operating Temp         : 95 C
    GPU Power Readings
        Power Draw                        : 33.78 W
        Current Power Limit               : 165.00 W
        Requested Power Limit             : 165.00 W
        Default Power Limit               : 165.00 W
        Min Power Limit                   : 100.00 W
        Max Power Limit                   : 165.00 W
    Module Power Readings
        Power Draw                        : N/A
        Current Power Limit               : N/A
        Requested Power Limit             : N/A
        Default Power Limit               : N/A
        Min Power Limit                   : N/A
        Max Power Limit                   : N/A
    Clocks
        Graphics                          : 210 MHz
        SM                                : 210 MHz
        Memory                            : 1215 MHz
        Video                             : 585 MHz
    Applications Clocks
        Graphics                          : 930 MHz
        Memory                            : 1215 MHz
    Default Applications Clocks
        Graphics                          : 930 MHz
        Memory                            : 1215 MHz
    Deferred Clocks
        Memory                            : N/A
    Max Clocks
        Graphics                          : 1440 MHz
        SM                                : 1440 MHz
        Memory                            : 1215 MHz
        Video                             : 1305 MHz
    Max Customer Boost Clocks
        Graphics                          : 1440 MHz
    Clock Policy
        Auto Boost                        : N/A
        Auto Boost Default                : N/A
    Voltage
        Graphics                          : 693.750 mV
    Fabric
        State                             : N/A
        Status                            : N/A
    Processes
        GPU instance ID                   : N/A
        Compute instance ID               : N/A
        Process ID                        : 8761
            Type                          : G
            Name                          : /usr/lib/xorg/Xorg
            Used GPU Memory               : 4 MiB

GPU 00000000:2D:00.0
    Product Name                          : NVIDIA A30
    Product Brand                         : NVIDIA
    Product Architecture                  : Ampere
    Display Mode                          : Enabled
    Display Active                        : Disabled
    Persistence Mode                      : Disabled
    Addressing Mode                       : None
    MIG Mode
        Current                           : Disabled
        Pending                           : Disabled
    Accounting Mode                       : Disabled
    Accounting Mode Buffer Size           : 4000
    Driver Model
        Current                           : N/A
        Pending                           : N/A
    Serial Number                         : 1655223001200
    GPU UUID                              : GPU-5f5d1ecc-e995-da2e-1836-5df769ed4271
    Minor Number                          : 1
    VBIOS Version                         : 92.00.66.00.04
    MultiGPU Board                        : No
    Board ID                              : 0x2d00
    Board Part Number                     : 900-21001-0040-100
    GPU Part Number                       : 20B7-882-A1
    FRU Part Number                       : N/A
    Module ID                             : 1
    Inforom Version
        Image Version                     : 1001.0205.00.02
        OEM Object                        : 2.0
        ECC Object                        : 6.16
        Power Management Object           : N/A
    Inforom BBX Object Flush
        Latest Timestamp                  : N/A
        Latest Duration                   : N/A
    GPU Operation Mode
        Current                           : N/A
        Pending                           : N/A
    GSP Firmware Version                  : 535.309.01
    GPU Virtualization Mode
        Virtualization Mode               : None
        Host VGPU Mode                    : N/A
    GPU Reset Status
        Reset Required                    : No
        Drain and Reset Recommended       : No
    IBMNPU
        Relaxed Ordering Mode             : N/A
    PCI
        Bus                               : 0x2D
        Device                            : 0x00
        Domain                            : 0x0000
        Device Id                         : 0x20B710DE
        Bus Id                            : 00000000:2D:00.0
        Sub System Id                     : 0x153210DE
        GPU Link Info
            PCIe Generation
                Max                       : 4
                Current                   : 4
                Device Current            : 4
                Device Max                : 4
                Host Max                  : 5
            Link Width
                Max                       : 16x
                Current                   : 16x
        Bridge Chip
            Type                          : N/A
            Firmware                      : N/A
        Replays Since Reset               : 0
        Replay Number Rollovers           : 0
        Tx Throughput                     : 0 KB/s
        Rx Throughput                     : 0 KB/s
        Atomic Caps Inbound               : N/A
        Atomic Caps Outbound              : N/A
    Fan Speed                             : N/A
    Performance State                     : P0
    Clocks Event Reasons
        Idle                              : Active
        Applications Clocks Setting       : Not Active
        SW Power Cap                      : Not Active
        HW Slowdown                       : Not Active
            HW Thermal Slowdown           : Not Active
            HW Power Brake Slowdown       : Not Active
        Sync Boost                        : Not Active
        SW Thermal Slowdown               : Not Active
        Display Clock Setting             : Not Active
    Sparse Operation Mode                 : N/A
    FB Memory Usage
        Total                             : 24576 MiB
        Reserved                          : 513 MiB
        Used                              : 12 MiB
        Free                              : 24049 MiB
    BAR1 Memory Usage
        Total                             : 32768 MiB
        Used                              : 2 MiB
        Free                              : 32766 MiB
    Conf Compute Protected Memory Usage
        Total                             : 0 MiB
        Used                              : 0 MiB
        Free                              : 0 MiB
    Compute Mode                          : Default
    Utilization
        Gpu                               : 0 %
        Memory                            : 0 %
        Encoder                           : 0 %
        Decoder                           : 0 %
        JPEG                              : 0 %
        OFA                               : 0 %
    Encoder Stats
        Active Sessions                   : 0
        Average FPS                       : 0
        Average Latency                   : 0
    FBC Stats
        Active Sessions                   : 0
        Average FPS                       : 0
        Average Latency                   : 0
    ECC Mode
        Current                           : Enabled
        Pending                           : Enabled
    ECC Errors
        Volatile
            SRAM Correctable              : 0
            SRAM Uncorrectable Parity     : 0
            SRAM Uncorrectable SEC-DED    : 0
            DRAM Correctable              : 0
            DRAM Uncorrectable            : 0
        Aggregate
            SRAM Correctable              : 0
            SRAM Uncorrectable Parity     : 0
            SRAM Uncorrectable SEC-DED    : 0
            DRAM Correctable              : 0
            DRAM Uncorrectable            : 0
            SRAM Threshold Exceeded       : No
        Aggregate Uncorrectable SRAM Sources
            SRAM L2                       : 0
            SRAM SM                       : 0
            SRAM Microcontroller          : 0
            SRAM PCIE                     : 0
            SRAM Other                    : 0
    Retired Pages
        Single Bit ECC                    : N/A
        Double Bit ECC                    : N/A
        Pending Page Blacklist            : N/A
    Remapped Rows
        Correctable Error                 : 0
        Uncorrectable Error               : 0
        Pending                           : No
        Remapping Failure Occurred        : No
        Bank Remap Availability Histogram
            Max                           : 384 bank(s)
            High                          : 0 bank(s)
            Partial                       : 0 bank(s)
            Low                           : 0 bank(s)
            None                          : 0 bank(s)
    Temperature
        GPU Current Temp                  : 37 C
        GPU T.Limit Temp                  : N/A
        GPU Shutdown Temp                 : 100 C
        GPU Slowdown Temp                 : 97 C
        GPU Max Operating Temp            : 90 C
        GPU Target Temperature            : N/A
        Memory Current Temp               : 36 C
        Memory Max Operating Temp         : 95 C
    GPU Power Readings
        Power Draw                        : 32.37 W
        Current Power Limit               : 165.00 W
        Requested Power Limit             : 165.00 W
        Default Power Limit               : 165.00 W
        Min Power Limit                   : 100.00 W
        Max Power Limit                   : 165.00 W
    Module Power Readings
        Power Draw                        : N/A
        Current Power Limit               : N/A
        Requested Power Limit             : N/A
        Default Power Limit               : N/A
        Min Power Limit                   : N/A
        Max Power Limit                   : N/A
    Clocks
        Graphics                          : 210 MHz
        SM                                : 210 MHz
        Memory                            : 1215 MHz
        Video                             : 585 MHz
    Applications Clocks
        Graphics                          : 930 MHz
        Memory                            : 1215 MHz
    Default Applications Clocks
        Graphics                          : 930 MHz
        Memory                            : 1215 MHz
    Deferred Clocks
        Memory                            : N/A
    Max Clocks
        Graphics                          : 1440 MHz
        SM                                : 1440 MHz
        Memory                            : 1215 MHz
        Video                             : 1305 MHz
    Max Customer Boost Clocks
        Graphics                          : 1440 MHz
    Clock Policy
        Auto Boost                        : N/A
        Auto Boost Default                : N/A
    Voltage
        Graphics                          : 693.750 mV
    Fabric
        State                             : N/A
        Status                            : N/A
    Processes
        GPU instance ID                   : N/A
        Compute instance ID               : N/A
        Process ID                        : 8761
            Type                          : G
            Name                          : /usr/lib/xorg/Xorg
            Used GPU Memory               : 4 MiB

GPU 00000000:3A:00.0
    Product Name                          : NVIDIA A30
    Product Brand                         : NVIDIA
    Product Architecture                  : Ampere
    Display Mode                          : Enabled
    Display Active                        : Disabled
    Persistence Mode                      : Disabled
    Addressing Mode                       : None
    MIG Mode
        Current                           : Disabled
        Pending                           : Disabled
    Accounting Mode                       : Disabled
    Accounting Mode Buffer Size           : 4000
    Driver Model
        Current                           : N/A
        Pending                           : N/A
    Serial Number                         : 1655223001630
    GPU UUID                              : GPU-54549f65-cb17-e6d6-d629-6a32ffa46b05
    Minor Number                          : 2
    VBIOS Version                         : 92.00.66.00.04
    MultiGPU Board                        : No
    Board ID                              : 0x3a00
    Board Part Number                     : 900-21001-0040-100
    GPU Part Number                       : 20B7-882-A1
    FRU Part Number                       : N/A
    Module ID                             : 1
    Inforom Version
        Image Version                     : 1001.0205.00.02
        OEM Object                        : 2.0
        ECC Object                        : 6.16
        Power Management Object           : N/A
    Inforom BBX Object Flush
        Latest Timestamp                  : N/A
        Latest Duration                   : N/A
    GPU Operation Mode
        Current                           : N/A
        Pending                           : N/A
    GSP Firmware Version                  : 535.309.01
    GPU Virtualization Mode
        Virtualization Mode               : None
        Host VGPU Mode                    : N/A
    GPU Reset Status
        Reset Required                    : No
        Drain and Reset Recommended       : No
    IBMNPU
        Relaxed Ordering Mode             : N/A
    PCI
        Bus                               : 0x3A
        Device                            : 0x00
        Domain                            : 0x0000
        Device Id                         : 0x20B710DE
        Bus Id                            : 00000000:3A:00.0
        Sub System Id                     : 0x153210DE
        GPU Link Info
            PCIe Generation
                Max                       : 4
                Current                   : 4
                Device Current            : 4
                Device Max                : 4
                Host Max                  : 5
            Link Width
                Max                       : 16x
                Current                   : 16x
        Bridge Chip
            Type                          : N/A
            Firmware                      : N/A
        Replays Since Reset               : 0
        Replay Number Rollovers           : 0
        Tx Throughput                     : 0 KB/s
        Rx Throughput                     : 0 KB/s
        Atomic Caps Inbound               : N/A
        Atomic Caps Outbound              : N/A
    Fan Speed                             : N/A
    Performance State                     : P0
    Clocks Event Reasons
        Idle                              : Active
        Applications Clocks Setting       : Not Active
        SW Power Cap                      : Not Active
        HW Slowdown                       : Not Active
            HW Thermal Slowdown           : Not Active
            HW Power Brake Slowdown       : Not Active
        Sync Boost                        : Not Active
        SW Thermal Slowdown               : Not Active
        Display Clock Setting             : Not Active
    Sparse Operation Mode                 : N/A
    FB Memory Usage
        Total                             : 24576 MiB
        Reserved                          : 513 MiB
        Used                              : 12 MiB
        Free                              : 24049 MiB
    BAR1 Memory Usage
        Total                             : 32768 MiB
        Used                              : 2 MiB
        Free                              : 32766 MiB
    Conf Compute Protected Memory Usage
        Total                             : 0 MiB
        Used                              : 0 MiB
        Free                              : 0 MiB
    Compute Mode                          : Default
    Utilization
        Gpu                               : 0 %
        Memory                            : 0 %
        Encoder                           : 0 %
        Decoder                           : 0 %
        JPEG                              : 0 %
        OFA                               : 0 %
    Encoder Stats
        Active Sessions                   : 0
        Average FPS                       : 0
        Average Latency                   : 0
    FBC Stats
        Active Sessions                   : 0
        Average FPS                       : 0
        Average Latency                   : 0
    ECC Mode
        Current                           : Enabled
        Pending                           : Enabled
    ECC Errors
        Volatile
            SRAM Correctable              : 0
            SRAM Uncorrectable Parity     : 0
            SRAM Uncorrectable SEC-DED    : 0
            DRAM Correctable              : 0
            DRAM Uncorrectable            : 0
        Aggregate
            SRAM Correctable              : 0
            SRAM Uncorrectable Parity     : 0
            SRAM Uncorrectable SEC-DED    : 0
            DRAM Correctable              : 0
            DRAM Uncorrectable            : 0
            SRAM Threshold Exceeded       : No
        Aggregate Uncorrectable SRAM Sources
            SRAM L2                       : 0
            SRAM SM                       : 0
            SRAM Microcontroller          : 0
            SRAM PCIE                     : 0
            SRAM Other                    : 0
    Retired Pages
        Single Bit ECC                    : N/A
        Double Bit ECC                    : N/A
        Pending Page Blacklist            : N/A
    Remapped Rows
        Correctable Error                 : 0
        Uncorrectable Error               : 0
        Pending                           : No
        Remapping Failure Occurred        : No
        Bank Remap Availability Histogram
            Max                           : 384 bank(s)
            High                          : 0 bank(s)
            Partial                       : 0 bank(s)
            Low                           : 0 bank(s)
            None                          : 0 bank(s)
    Temperature
        GPU Current Temp                  : 35 C
        GPU T.Limit Temp                  : N/A
        GPU Shutdown Temp                 : 100 C
        GPU Slowdown Temp                 : 97 C
        GPU Max Operating Temp            : 90 C
        GPU Target Temperature            : N/A
        Memory Current Temp               : 35 C
        Memory Max Operating Temp         : 95 C
    GPU Power Readings
        Power Draw                        : 35.06 W
        Current Power Limit               : 165.00 W
        Requested Power Limit             : 165.00 W
        Default Power Limit               : 165.00 W
        Min Power Limit                   : 100.00 W
        Max Power Limit                   : 165.00 W
    Module Power Readings
        Power Draw                        : N/A
        Current Power Limit               : N/A
        Requested Power Limit             : N/A
        Default Power Limit               : N/A
        Min Power Limit                   : N/A
        Max Power Limit                   : N/A
    Clocks
        Graphics                          : 210 MHz
        SM                                : 210 MHz
        Memory                            : 1215 MHz
        Video                             : 585 MHz
    Applications Clocks
        Graphics                          : 930 MHz
        Memory                            : 1215 MHz
    Default Applications Clocks
        Graphics                          : 930 MHz
        Memory                            : 1215 MHz
    Deferred Clocks
        Memory                            : N/A
    Max Clocks
        Graphics                          : 1440 MHz
        SM                                : 1440 MHz
        Memory                            : 1215 MHz
        Video                             : 1305 MHz
    Max Customer Boost Clocks
        Graphics                          : 1440 MHz
    Clock Policy
        Auto Boost                        : N/A
        Auto Boost Default                : N/A
    Voltage
        Graphics                          : 656.250 mV
    Fabric
        State                             : N/A
        Status                            : N/A
    Processes
        GPU instance ID                   : N/A
        Compute instance ID               : N/A
        Process ID                        : 8761
            Type                          : G
            Name                          : /usr/lib/xorg/Xorg
            Used GPU Memory               : 4 MiB

GPU 00000000:3B:00.0
    Product Name                          : NVIDIA A30
    Product Brand                         : NVIDIA
    Product Architecture                  : Ampere
    Display Mode                          : Enabled
    Display Active                        : Disabled
    Persistence Mode                      : Disabled
    Addressing Mode                       : None
    MIG Mode
        Current                           : Disabled
        Pending                           : Disabled
    Accounting Mode                       : Disabled
    Accounting Mode Buffer Size           : 4000
    Driver Model
        Current                           : N/A
        Pending                           : N/A
    Serial Number                         : 1655223001325
    GPU UUID                              : GPU-18a0a4e4-5ac1-29ea-9f8a-10b15039813a
    Minor Number                          : 3
    VBIOS Version                         : 92.00.66.00.04
    MultiGPU Board                        : No
    Board ID                              : 0x3b00
    Board Part Number                     : 900-21001-0040-100
    GPU Part Number                       : 20B7-882-A1
    FRU Part Number                       : N/A
    Module ID                             : 1
    Inforom Version
        Image Version                     : 1001.0205.00.02
        OEM Object                        : 2.0
        ECC Object                        : 6.16
        Power Management Object           : N/A
    Inforom BBX Object Flush
        Latest Timestamp                  : N/A
        Latest Duration                   : N/A
    GPU Operation Mode
        Current                           : N/A
        Pending                           : N/A
    GSP Firmware Version                  : 535.309.01
    GPU Virtualization Mode
        Virtualization Mode               : None
        Host VGPU Mode                    : N/A
    GPU Reset Status
        Reset Required                    : No
        Drain and Reset Recommended       : No
    IBMNPU
        Relaxed Ordering Mode             : N/A
    PCI
        Bus                               : 0x3B
        Device                            : 0x00
        Domain                            : 0x0000
        Device Id                         : 0x20B710DE
        Bus Id                            : 00000000:3B:00.0
        Sub System Id                     : 0x153210DE
        GPU Link Info
            PCIe Generation
                Max                       : 4
                Current                   : 4
                Device Current            : 4
                Device Max                : 4
                Host Max                  : 5
            Link Width
                Max                       : 16x
                Current                   : 16x
        Bridge Chip
            Type                          : N/A
            Firmware                      : N/A
        Replays Since Reset               : 0
        Replay Number Rollovers           : 0
        Tx Throughput                     : 0 KB/s
        Rx Throughput                     : 0 KB/s
        Atomic Caps Inbound               : N/A
        Atomic Caps Outbound              : N/A
    Fan Speed                             : N/A
    Performance State                     : P0
    Clocks Event Reasons
        Idle                              : Active
        Applications Clocks Setting       : Not Active
        SW Power Cap                      : Not Active
        HW Slowdown                       : Not Active
            HW Thermal Slowdown           : Not Active
            HW Power Brake Slowdown       : Not Active
        Sync Boost                        : Not Active
        SW Thermal Slowdown               : Not Active
        Display Clock Setting             : Not Active
    Sparse Operation Mode                 : N/A
    FB Memory Usage
        Total                             : 24576 MiB
        Reserved                          : 513 MiB
        Used                              : 12 MiB
        Free                              : 24049 MiB
    BAR1 Memory Usage
        Total                             : 32768 MiB
        Used                              : 2 MiB
        Free                              : 32766 MiB
    Conf Compute Protected Memory Usage
        Total                             : 0 MiB
        Used                              : 0 MiB
        Free                              : 0 MiB
    Compute Mode                          : Default
    Utilization
        Gpu                               : 0 %
        Memory                            : 0 %
        Encoder                           : 0 %
        Decoder                           : 0 %
        JPEG                              : 0 %
        OFA                               : 0 %
    Encoder Stats
        Active Sessions                   : 0
        Average FPS                       : 0
        Average Latency                   : 0
    FBC Stats
        Active Sessions                   : 0
        Average FPS                       : 0
        Average Latency                   : 0
    ECC Mode
        Current                           : Enabled
        Pending                           : Enabled
    ECC Errors
        Volatile
            SRAM Correctable              : 0
            SRAM Uncorrectable Parity     : 0
            SRAM Uncorrectable SEC-DED    : 0
            DRAM Correctable              : 0
            DRAM Uncorrectable            : 0
        Aggregate
            SRAM Correctable              : 0
            SRAM Uncorrectable Parity     : 0
            SRAM Uncorrectable SEC-DED    : 0
            DRAM Correctable              : 0
            DRAM Uncorrectable            : 0
            SRAM Threshold Exceeded       : No
        Aggregate Uncorrectable SRAM Sources
            SRAM L2                       : 0
            SRAM SM                       : 0
            SRAM Microcontroller          : 0
            SRAM PCIE                     : 0
            SRAM Other                    : 0
    Retired Pages
        Single Bit ECC                    : N/A
        Double Bit ECC                    : N/A
        Pending Page Blacklist            : N/A
    Remapped Rows
        Correctable Error                 : 0
        Uncorrectable Error               : 0
        Pending                           : No
        Remapping Failure Occurred        : No
        Bank Remap Availability Histogram
            Max                           : 384 bank(s)
            High                          : 0 bank(s)
            Partial                       : 0 bank(s)
            Low                           : 0 bank(s)
            None                          : 0 bank(s)
    Temperature
        GPU Current Temp                  : 37 C
        GPU T.Limit Temp                  : N/A
        GPU Shutdown Temp                 : 100 C
        GPU Slowdown Temp                 : 97 C
        GPU Max Operating Temp            : 90 C
        GPU Target Temperature            : N/A
        Memory Current Temp               : 37 C
        Memory Max Operating Temp         : 95 C
    GPU Power Readings
        Power Draw                        : 34.65 W
        Current Power Limit               : 165.00 W
        Requested Power Limit             : 165.00 W
        Default Power Limit               : 165.00 W
        Min Power Limit                   : 100.00 W
        Max Power Limit                   : 165.00 W
    Module Power Readings
        Power Draw                        : N/A
        Current Power Limit               : N/A
        Requested Power Limit             : N/A
        Default Power Limit               : N/A
        Min Power Limit                   : N/A
        Max Power Limit                   : N/A
    Clocks
        Graphics                          : 210 MHz
        SM                                : 210 MHz
        Memory                            : 1215 MHz
        Video                             : 585 MHz
    Applications Clocks
        Graphics                          : 930 MHz
        Memory                            : 1215 MHz
    Default Applications Clocks
        Graphics                          : 930 MHz
        Memory                            : 1215 MHz
    Deferred Clocks
        Memory                            : N/A
    Max Clocks
        Graphics                          : 1440 MHz
        SM                                : 1440 MHz
        Memory                            : 1215 MHz
        Video                             : 1305 MHz
    Max Customer Boost Clocks
        Graphics                          : 1440 MHz
    Clock Policy
        Auto Boost                        : N/A
        Auto Boost Default                : N/A
    Voltage
        Graphics                          : 668.750 mV
    Fabric
        State                             : N/A
        Status                            : N/A
    Processes
        GPU instance ID                   : N/A
        Compute instance ID               : N/A
        Process ID                        : 8761
            Type                          : G
            Name                          : /usr/lib/xorg/Xorg
            Used GPU Memory               : 4 MiB

GPU 00000000:AD:00.0
    Product Name                          : NVIDIA A30
    Product Brand                         : NVIDIA
    Product Architecture                  : Ampere
    Display Mode                          : Enabled
    Display Active                        : Disabled
    Persistence Mode                      : Disabled
    Addressing Mode                       : None
    MIG Mode
        Current                           : Disabled
        Pending                           : Disabled
    Accounting Mode                       : Disabled
    Accounting Mode Buffer Size           : 4000
    Driver Model
        Current                           : N/A
        Pending                           : N/A
    Serial Number                         : 1655223001921
    GPU UUID                              : GPU-a325d7f7-f14c-0408-3ad7-0e029d016699
    Minor Number                          : 4
    VBIOS Version                         : 92.00.66.00.04
    MultiGPU Board                        : No
    Board ID                              : 0xad00
    Board Part Number                     : 900-21001-0040-100
    GPU Part Number                       : 20B7-882-A1
    FRU Part Number                       : N/A
    Module ID                             : 1
    Inforom Version
        Image Version                     : 1001.0205.00.02
        OEM Object                        : 2.0
        ECC Object                        : 6.16
        Power Management Object           : N/A
    Inforom BBX Object Flush
        Latest Timestamp                  : N/A
        Latest Duration                   : N/A
    GPU Operation Mode
        Current                           : N/A
        Pending                           : N/A
    GSP Firmware Version                  : 535.309.01
    GPU Virtualization Mode
        Virtualization Mode               : None
        Host VGPU Mode                    : N/A
    GPU Reset Status
        Reset Required                    : No
        Drain and Reset Recommended       : No
    IBMNPU
        Relaxed Ordering Mode             : N/A
    PCI
        Bus                               : 0xAD
        Device                            : 0x00
        Domain                            : 0x0000
        Device Id                         : 0x20B710DE
        Bus Id                            : 00000000:AD:00.0
        Sub System Id                     : 0x153210DE
        GPU Link Info
            PCIe Generation
                Max                       : 4
                Current                   : 4
                Device Current            : 4
                Device Max                : 4
                Host Max                  : 5
            Link Width
                Max                       : 16x
                Current                   : 16x
        Bridge Chip
            Type                          : N/A
            Firmware                      : N/A
        Replays Since Reset               : 0
        Replay Number Rollovers           : 0
        Tx Throughput                     : 0 KB/s
        Rx Throughput                     : 0 KB/s
        Atomic Caps Inbound               : N/A
        Atomic Caps Outbound              : N/A
    Fan Speed                             : N/A
    Performance State                     : P0
    Clocks Event Reasons
        Idle                              : Active
        Applications Clocks Setting       : Not Active
        SW Power Cap                      : Not Active
        HW Slowdown                       : Not Active
            HW Thermal Slowdown           : Not Active
            HW Power Brake Slowdown       : Not Active
        Sync Boost                        : Not Active
        SW Thermal Slowdown               : Not Active
        Display Clock Setting             : Not Active
    Sparse Operation Mode                 : N/A
    FB Memory Usage
        Total                             : 24576 MiB
        Reserved                          : 513 MiB
        Used                              : 12 MiB
        Free                              : 24049 MiB
    BAR1 Memory Usage
        Total                             : 32768 MiB
        Used                              : 2 MiB
        Free                              : 32766 MiB
    Conf Compute Protected Memory Usage
        Total                             : 0 MiB
        Used                              : 0 MiB
        Free                              : 0 MiB
    Compute Mode                          : Default
    Utilization
        Gpu                               : 0 %
        Memory                            : 0 %
        Encoder                           : 0 %
        Decoder                           : 0 %
        JPEG                              : 0 %
        OFA                               : 0 %
    Encoder Stats
        Active Sessions                   : 0
        Average FPS                       : 0
        Average Latency                   : 0
    FBC Stats
        Active Sessions                   : 0
        Average FPS                       : 0
        Average Latency                   : 0
    ECC Mode
        Current                           : Enabled
        Pending                           : Enabled
    ECC Errors
        Volatile
            SRAM Correctable              : 0
            SRAM Uncorrectable Parity     : 0
            SRAM Uncorrectable SEC-DED    : 0
            DRAM Correctable              : 0
            DRAM Uncorrectable            : 0
        Aggregate
            SRAM Correctable              : 0
            SRAM Uncorrectable Parity     : 0
            SRAM Uncorrectable SEC-DED    : 0
            DRAM Correctable              : 0
            DRAM Uncorrectable            : 0
            SRAM Threshold Exceeded       : No
        Aggregate Uncorrectable SRAM Sources
            SRAM L2                       : 0
            SRAM SM                       : 0
            SRAM Microcontroller          : 0
            SRAM PCIE                     : 0
            SRAM Other                    : 0
    Retired Pages
        Single Bit ECC                    : N/A
        Double Bit ECC                    : N/A
        Pending Page Blacklist            : N/A
    Remapped Rows
        Correctable Error                 : 0
        Uncorrectable Error               : 0
        Pending                           : No
        Remapping Failure Occurred        : No
        Bank Remap Availability Histogram
            Max                           : 384 bank(s)
            High                          : 0 bank(s)
            Partial                       : 0 bank(s)
            Low                           : 0 bank(s)
            None                          : 0 bank(s)
    Temperature
        GPU Current Temp                  : 41 C
        GPU T.Limit Temp                  : N/A
        GPU Shutdown Temp                 : 100 C
        GPU Slowdown Temp                 : 97 C
        GPU Max Operating Temp            : 90 C
        GPU Target Temperature            : N/A
        Memory Current Temp               : 40 C
        Memory Max Operating Temp         : 95 C
    GPU Power Readings
        Power Draw                        : 31.83 W
        Current Power Limit               : 165.00 W
        Requested Power Limit             : 165.00 W
        Default Power Limit               : 165.00 W
        Min Power Limit                   : 100.00 W
        Max Power Limit                   : 165.00 W
    Module Power Readings
        Power Draw                        : N/A
        Current Power Limit               : N/A
        Requested Power Limit             : N/A
        Default Power Limit               : N/A
        Min Power Limit                   : N/A
        Max Power Limit                   : N/A
    Clocks
        Graphics                          : 210 MHz
        SM                                : 210 MHz
        Memory                            : 1215 MHz
        Video                             : 585 MHz
    Applications Clocks
        Graphics                          : 930 MHz
        Memory                            : 1215 MHz
    Default Applications Clocks
        Graphics                          : 930 MHz
        Memory                            : 1215 MHz
    Deferred Clocks
        Memory                            : N/A
    Max Clocks
        Graphics                          : 1440 MHz
        SM                                : 1440 MHz
        Memory                            : 1215 MHz
        Video                             : 1305 MHz
    Max Customer Boost Clocks
        Graphics                          : 1440 MHz
    Clock Policy
        Auto Boost                        : N/A
        Auto Boost Default                : N/A
    Voltage
        Graphics                          : 681.250 mV
    Fabric
        State                             : N/A
        Status                            : N/A
    Processes
        GPU instance ID                   : N/A
        Compute instance ID               : N/A
        Process ID                        : 8761
            Type                          : G
            Name                          : /usr/lib/xorg/Xorg
            Used GPU Memory               : 4 MiB

GPU 00000000:AE:00.0
    Product Name                          : NVIDIA A30
    Product Brand                         : NVIDIA
    Product Architecture                  : Ampere
    Display Mode                          : Enabled
    Display Active                        : Disabled
    Persistence Mode                      : Disabled
    Addressing Mode                       : None
    MIG Mode
        Current                           : Disabled
        Pending                           : Disabled
    Accounting Mode                       : Disabled
    Accounting Mode Buffer Size           : 4000
    Driver Model
        Current                           : N/A
        Pending                           : N/A
    Serial Number                         : 1655223001302
    GPU UUID                              : GPU-9a26c485-479c-071a-3051-3c35fe929fe6
    Minor Number                          : 5
    VBIOS Version                         : 92.00.66.00.04
    MultiGPU Board                        : No
    Board ID                              : 0xae00
    Board Part Number                     : 900-21001-0040-100
    GPU Part Number                       : 20B7-882-A1
    FRU Part Number                       : N/A
    Module ID                             : 1
    Inforom Version
        Image Version                     : 1001.0205.00.02
        OEM Object                        : 2.0
        ECC Object                        : 6.16
        Power Management Object           : N/A
    Inforom BBX Object Flush
        Latest Timestamp                  : N/A
        Latest Duration                   : N/A
    GPU Operation Mode
        Current                           : N/A
        Pending                           : N/A
    GSP Firmware Version                  : 535.309.01
    GPU Virtualization Mode
        Virtualization Mode               : None
        Host VGPU Mode                    : N/A
    GPU Reset Status
        Reset Required                    : No
        Drain and Reset Recommended       : No
    IBMNPU
        Relaxed Ordering Mode             : N/A
    PCI
        Bus                               : 0xAE
        Device                            : 0x00
        Domain                            : 0x0000
        Device Id                         : 0x20B710DE
        Bus Id                            : 00000000:AE:00.0
        Sub System Id                     : 0x153210DE
        GPU Link Info
            PCIe Generation
                Max                       : 4
                Current                   : 4
                Device Current            : 4
                Device Max                : 4
                Host Max                  : 5
            Link Width
                Max                       : 16x
                Current                   : 16x
        Bridge Chip
            Type                          : N/A
            Firmware                      : N/A
        Replays Since Reset               : 0
        Replay Number Rollovers           : 0
        Tx Throughput                     : 0 KB/s
        Rx Throughput                     : 0 KB/s
        Atomic Caps Inbound               : N/A
        Atomic Caps Outbound              : N/A
    Fan Speed                             : N/A
    Performance State                     : P0
    Clocks Event Reasons
        Idle                              : Active
        Applications Clocks Setting       : Not Active
        SW Power Cap                      : Not Active
        HW Slowdown                       : Not Active
            HW Thermal Slowdown           : Not Active
            HW Power Brake Slowdown       : Not Active
        Sync Boost                        : Not Active
        SW Thermal Slowdown               : Not Active
        Display Clock Setting             : Not Active
    Sparse Operation Mode                 : N/A
    FB Memory Usage
        Total                             : 24576 MiB
        Reserved                          : 513 MiB
        Used                              : 12 MiB
        Free                              : 24049 MiB
    BAR1 Memory Usage
        Total                             : 32768 MiB
        Used                              : 2 MiB
        Free                              : 32766 MiB
    Conf Compute Protected Memory Usage
        Total                             : 0 MiB
        Used                              : 0 MiB
        Free                              : 0 MiB
    Compute Mode                          : Default
    Utilization
        Gpu                               : 0 %
        Memory                            : 0 %
        Encoder                           : 0 %
        Decoder                           : 0 %
        JPEG                              : 0 %
        OFA                               : 0 %
    Encoder Stats
        Active Sessions                   : 0
        Average FPS                       : 0
        Average Latency                   : 0
    FBC Stats
        Active Sessions                   : 0
        Average FPS                       : 0
        Average Latency                   : 0
    ECC Mode
        Current                           : Enabled
        Pending                           : Enabled
    ECC Errors
        Volatile
            SRAM Correctable              : 0
            SRAM Uncorrectable Parity     : 0
            SRAM Uncorrectable SEC-DED    : 0
            DRAM Correctable              : 0
            DRAM Uncorrectable            : 0
        Aggregate
            SRAM Correctable              : 0
            SRAM Uncorrectable Parity     : 0
            SRAM Uncorrectable SEC-DED    : 0
            DRAM Correctable              : 0
            DRAM Uncorrectable            : 0
            SRAM Threshold Exceeded       : No
        Aggregate Uncorrectable SRAM Sources
            SRAM L2                       : 0
            SRAM SM                       : 0
            SRAM Microcontroller          : 0
            SRAM PCIE                     : 0
            SRAM Other                    : 0
    Retired Pages
        Single Bit ECC                    : N/A
        Double Bit ECC                    : N/A
        Pending Page Blacklist            : N/A
    Remapped Rows
        Correctable Error                 : 0
        Uncorrectable Error               : 0
        Pending                           : No
        Remapping Failure Occurred        : No
        Bank Remap Availability Histogram
            Max                           : 384 bank(s)
            High                          : 0 bank(s)
            Partial                       : 0 bank(s)
            Low                           : 0 bank(s)
            None                          : 0 bank(s)
    Temperature
        GPU Current Temp                  : 44 C
        GPU T.Limit Temp                  : N/A
        GPU Shutdown Temp                 : 100 C
        GPU Slowdown Temp                 : 97 C
        GPU Max Operating Temp            : 90 C
        GPU Target Temperature            : N/A
        Memory Current Temp               : 44 C
        Memory Max Operating Temp         : 95 C
    GPU Power Readings
        Power Draw                        : 37.51 W
        Current Power Limit               : 165.00 W
        Requested Power Limit             : 165.00 W
        Default Power Limit               : 165.00 W
        Min Power Limit                   : 100.00 W
        Max Power Limit                   : 165.00 W
    Module Power Readings
        Power Draw                        : N/A
        Current Power Limit               : N/A
        Requested Power Limit             : N/A
        Default Power Limit               : N/A
        Min Power Limit                   : N/A
        Max Power Limit                   : N/A
    Clocks
        Graphics                          : 750 MHz
        SM                                : 750 MHz
        Memory                            : 1215 MHz
        Video                             : 660 MHz
    Applications Clocks
        Graphics                          : 930 MHz
        Memory                            : 1215 MHz
    Default Applications Clocks
        Graphics                          : 930 MHz
        Memory                            : 1215 MHz
    Deferred Clocks
        Memory                            : N/A
    Max Clocks
        Graphics                          : 1440 MHz
        SM                                : 1440 MHz
        Memory                            : 1215 MHz
        Video                             : 1305 MHz
    Max Customer Boost Clocks
        Graphics                          : 1440 MHz
    Clock Policy
        Auto Boost                        : N/A
        Auto Boost Default                : N/A
    Voltage
        Graphics                          : 675.000 mV
    Fabric
        State                             : N/A
        Status                            : N/A
    Processes
        GPU instance ID                   : N/A
        Compute instance ID               : N/A
        Process ID                        : 8761
            Type                          : G
            Name                          : /usr/lib/xorg/Xorg
            Used GPU Memory               : 4 MiB

GPU 00000000:BC:00.0
    Product Name                          : NVIDIA A30
    Product Brand                         : NVIDIA
    Product Architecture                  : Ampere
    Display Mode                          : Enabled
    Display Active                        : Disabled
    Persistence Mode                      : Disabled
    Addressing Mode                       : None
    MIG Mode
        Current                           : Disabled
        Pending                           : Disabled
    Accounting Mode                       : Disabled
    Accounting Mode Buffer Size           : 4000
    Driver Model
        Current                           : N/A
        Pending                           : N/A
    Serial Number                         : 1655223001691
    GPU UUID                              : GPU-634739d3-b7f6-96a6-86bb-a57a573339fa
    Minor Number                          : 6
    VBIOS Version                         : 92.00.66.00.04
    MultiGPU Board                        : No
    Board ID                              : 0xbc00
    Board Part Number                     : 900-21001-0040-100
    GPU Part Number                       : 20B7-882-A1
    FRU Part Number                       : N/A
    Module ID                             : 1
    Inforom Version
        Image Version                     : 1001.0205.00.02
        OEM Object                        : 2.0
        ECC Object                        : 6.16
        Power Management Object           : N/A
    Inforom BBX Object Flush
        Latest Timestamp                  : N/A
        Latest Duration                   : N/A
    GPU Operation Mode
        Current                           : N/A
        Pending                           : N/A
    GSP Firmware Version                  : 535.309.01
    GPU Virtualization Mode
        Virtualization Mode               : None
        Host VGPU Mode                    : N/A
    GPU Reset Status
        Reset Required                    : No
        Drain and Reset Recommended       : No
    IBMNPU
        Relaxed Ordering Mode             : N/A
    PCI
        Bus                               : 0xBC
        Device                            : 0x00
        Domain                            : 0x0000
        Device Id                         : 0x20B710DE
        Bus Id                            : 00000000:BC:00.0
        Sub System Id                     : 0x153210DE
        GPU Link Info
            PCIe Generation
                Max                       : 4
                Current                   : 4
                Device Current            : 4
                Device Max                : 4
                Host Max                  : 5
            Link Width
                Max                       : 16x
                Current                   : 16x
        Bridge Chip
            Type                          : N/A
            Firmware                      : N/A
        Replays Since Reset               : 0
        Replay Number Rollovers           : 0
        Tx Throughput                     : 0 KB/s
        Rx Throughput                     : 0 KB/s
        Atomic Caps Inbound               : N/A
        Atomic Caps Outbound              : N/A
    Fan Speed                             : N/A
    Performance State                     : P0
    Clocks Event Reasons
        Idle                              : Active
        Applications Clocks Setting       : Not Active
        SW Power Cap                      : Not Active
        HW Slowdown                       : Not Active
            HW Thermal Slowdown           : Not Active
            HW Power Brake Slowdown       : Not Active
        Sync Boost                        : Not Active
        SW Thermal Slowdown               : Not Active
        Display Clock Setting             : Not Active
    Sparse Operation Mode                 : N/A
    FB Memory Usage
        Total                             : 24576 MiB
        Reserved                          : 513 MiB
        Used                              : 12 MiB
        Free                              : 24049 MiB
    BAR1 Memory Usage
        Total                             : 32768 MiB
        Used                              : 2 MiB
        Free                              : 32766 MiB
    Conf Compute Protected Memory Usage
        Total                             : 0 MiB
        Used                              : 0 MiB
        Free                              : 0 MiB
    Compute Mode                          : Default
    Utilization
        Gpu                               : 0 %
        Memory                            : 0 %
        Encoder                           : 0 %
        Decoder                           : 0 %
        JPEG                              : 0 %
        OFA                               : 0 %
    Encoder Stats
        Active Sessions                   : 0
        Average FPS                       : 0
        Average Latency                   : 0
    FBC Stats
        Active Sessions                   : 0
        Average FPS                       : 0
        Average Latency                   : 0
    ECC Mode
        Current                           : Enabled
        Pending                           : Enabled
    ECC Errors
        Volatile
            SRAM Correctable              : 0
            SRAM Uncorrectable Parity     : 0
            SRAM Uncorrectable SEC-DED    : 0
            DRAM Correctable              : 0
            DRAM Uncorrectable            : 0
        Aggregate
            SRAM Correctable              : 0
            SRAM Uncorrectable Parity     : 0
            SRAM Uncorrectable SEC-DED    : 0
            DRAM Correctable              : 0
            DRAM Uncorrectable            : 0
            SRAM Threshold Exceeded       : No
        Aggregate Uncorrectable SRAM Sources
            SRAM L2                       : 0
            SRAM SM                       : 0
            SRAM Microcontroller          : 0
            SRAM PCIE                     : 0
            SRAM Other                    : 0
    Retired Pages
        Single Bit ECC                    : N/A
        Double Bit ECC                    : N/A
        Pending Page Blacklist            : N/A
    Remapped Rows
        Correctable Error                 : 0
        Uncorrectable Error               : 0
        Pending                           : No
        Remapping Failure Occurred        : No
        Bank Remap Availability Histogram
            Max                           : 384 bank(s)
            High                          : 0 bank(s)
            Partial                       : 0 bank(s)
            Low                           : 0 bank(s)
            None                          : 0 bank(s)
    Temperature
        GPU Current Temp                  : 37 C
        GPU T.Limit Temp                  : N/A
        GPU Shutdown Temp                 : 100 C
        GPU Slowdown Temp                 : 97 C
        GPU Max Operating Temp            : 90 C
        GPU Target Temperature            : N/A
        Memory Current Temp               : 37 C
        Memory Max Operating Temp         : 95 C
    GPU Power Readings
        Power Draw                        : 33.31 W
        Current Power Limit               : 165.00 W
        Requested Power Limit             : 165.00 W
        Default Power Limit               : 165.00 W
        Min Power Limit                   : 100.00 W
        Max Power Limit                   : 165.00 W
    Module Power Readings
        Power Draw                        : N/A
        Current Power Limit               : N/A
        Requested Power Limit             : N/A
        Default Power Limit               : N/A
        Min Power Limit                   : N/A
        Max Power Limit                   : N/A
    Clocks
        Graphics                          : 690 MHz
        SM                                : 690 MHz
        Memory                            : 1215 MHz
        Video                             : 615 MHz
    Applications Clocks
        Graphics                          : 930 MHz
        Memory                            : 1215 MHz
    Default Applications Clocks
        Graphics                          : 930 MHz
        Memory                            : 1215 MHz
    Deferred Clocks
        Memory                            : N/A
    Max Clocks
        Graphics                          : 1440 MHz
        SM                                : 1440 MHz
        Memory                            : 1215 MHz
        Video                             : 1305 MHz
    Max Customer Boost Clocks
        Graphics                          : 1440 MHz
    Clock Policy
        Auto Boost                        : N/A
        Auto Boost Default                : N/A
    Voltage
        Graphics                          : 668.750 mV
    Fabric
        State                             : N/A
        Status                            : N/A
    Processes
        GPU instance ID                   : N/A
        Compute instance ID               : N/A
        Process ID                        : 8761
            Type                          : G
            Name                          : /usr/lib/xorg/Xorg
            Used GPU Memory               : 4 MiB

GPU 00000000:BD:00.0
    Product Name                          : NVIDIA A30
    Product Brand                         : NVIDIA
    Product Architecture                  : Ampere
    Display Mode                          : Enabled
    Display Active                        : Disabled
    Persistence Mode                      : Disabled
    Addressing Mode                       : None
    MIG Mode
        Current                           : Disabled
        Pending                           : Disabled
    Accounting Mode                       : Disabled
    Accounting Mode Buffer Size           : 4000
    Driver Model
        Current                           : N/A
        Pending                           : N/A
    Serial Number                         : 1655223001811
    GPU UUID                              : GPU-4dbb1e60-2602-0c10-7093-58c284888256
    Minor Number                          : 7
    VBIOS Version                         : 92.00.66.00.04
    MultiGPU Board                        : No
    Board ID                              : 0xbd00
    Board Part Number                     : 900-21001-0040-100
    GPU Part Number                       : 20B7-882-A1
    FRU Part Number                       : N/A
    Module ID                             : 1
    Inforom Version
        Image Version                     : 1001.0205.00.02
        OEM Object                        : 2.0
        ECC Object                        : 6.16
        Power Management Object           : N/A
    Inforom BBX Object Flush
        Latest Timestamp                  : N/A
        Latest Duration                   : N/A
    GPU Operation Mode
        Current                           : N/A
        Pending                           : N/A
    GSP Firmware Version                  : 535.309.01
    GPU Virtualization Mode
        Virtualization Mode               : None
        Host VGPU Mode                    : N/A
    GPU Reset Status
        Reset Required                    : No
        Drain and Reset Recommended       : No
    IBMNPU
        Relaxed Ordering Mode             : N/A
    PCI
        Bus                               : 0xBD
        Device                            : 0x00
        Domain                            : 0x0000
        Device Id                         : 0x20B710DE
        Bus Id                            : 00000000:BD:00.0
        Sub System Id                     : 0x153210DE
        GPU Link Info
            PCIe Generation
                Max                       : 4
                Current                   : 4
                Device Current            : 4
                Device Max                : 4
                Host Max                  : 5
            Link Width
                Max                       : 16x
                Current                   : 16x
        Bridge Chip
            Type                          : N/A
            Firmware                      : N/A
        Replays Since Reset               : 0
        Replay Number Rollovers           : 0
        Tx Throughput                     : 0 KB/s
        Rx Throughput                     : 0 KB/s
        Atomic Caps Inbound               : N/A
        Atomic Caps Outbound              : N/A
    Fan Speed                             : N/A
    Performance State                     : P0
    Clocks Event Reasons
        Idle                              : Active
        Applications Clocks Setting       : Not Active
        SW Power Cap                      : Not Active
        HW Slowdown                       : Not Active
            HW Thermal Slowdown           : Not Active
            HW Power Brake Slowdown       : Not Active
        Sync Boost                        : Not Active
        SW Thermal Slowdown               : Not Active
        Display Clock Setting             : Not Active
    Sparse Operation Mode                 : N/A
    FB Memory Usage
        Total                             : 24576 MiB
        Reserved                          : 513 MiB
        Used                              : 12 MiB
        Free                              : 24049 MiB
    BAR1 Memory Usage
        Total                             : 32768 MiB
        Used                              : 2 MiB
        Free                              : 32766 MiB
    Conf Compute Protected Memory Usage
        Total                             : 0 MiB
        Used                              : 0 MiB
        Free                              : 0 MiB
    Compute Mode                          : Default
    Utilization
        Gpu                               : 0 %
        Memory                            : 0 %
        Encoder                           : 0 %
        Decoder                           : 0 %
        JPEG                              : 0 %
        OFA                               : 0 %
    Encoder Stats
        Active Sessions                   : 0
        Average FPS                       : 0
        Average Latency                   : 0
    FBC Stats
        Active Sessions                   : 0
        Average FPS                       : 0
        Average Latency                   : 0
    ECC Mode
        Current                           : Enabled
        Pending                           : Enabled
    ECC Errors
        Volatile
            SRAM Correctable              : 0
            SRAM Uncorrectable Parity     : 0
            SRAM Uncorrectable SEC-DED    : 0
            DRAM Correctable              : 0
            DRAM Uncorrectable            : 0
        Aggregate
            SRAM Correctable              : 0
            SRAM Uncorrectable Parity     : 0
            SRAM Uncorrectable SEC-DED    : 0
            DRAM Correctable              : 0
            DRAM Uncorrectable            : 0
            SRAM Threshold Exceeded       : No
        Aggregate Uncorrectable SRAM Sources
            SRAM L2                       : 0
            SRAM SM                       : 0
            SRAM Microcontroller          : 0
            SRAM PCIE                     : 0
            SRAM Other                    : 0
    Retired Pages
        Single Bit ECC                    : N/A
        Double Bit ECC                    : N/A
        Pending Page Blacklist            : N/A
    Remapped Rows
        Correctable Error                 : 0
        Uncorrectable Error               : 0
        Pending                           : No
        Remapping Failure Occurred        : No
        Bank Remap Availability Histogram
            Max                           : 384 bank(s)
            High                          : 0 bank(s)
            Partial                       : 0 bank(s)
            Low                           : 0 bank(s)
            None                          : 0 bank(s)
    Temperature
        GPU Current Temp                  : 39 C
        GPU T.Limit Temp                  : N/A
        GPU Shutdown Temp                 : 100 C
        GPU Slowdown Temp                 : 97 C
        GPU Max Operating Temp            : 90 C
        GPU Target Temperature            : N/A
        Memory Current Temp               : 38 C
        Memory Max Operating Temp         : 95 C
    GPU Power Readings
        Power Draw                        : 31.64 W
        Current Power Limit               : 165.00 W
        Requested Power Limit             : 165.00 W
        Default Power Limit               : 165.00 W
        Min Power Limit                   : 100.00 W
        Max Power Limit                   : 165.00 W
    Module Power Readings
        Power Draw                        : N/A
        Current Power Limit               : N/A
        Requested Power Limit             : N/A
        Default Power Limit               : N/A
        Min Power Limit                   : N/A
        Max Power Limit                   : N/A
    Clocks
        Graphics                          : 210 MHz
        SM                                : 210 MHz
        Memory                            : 1215 MHz
        Video                             : 585 MHz
    Applications Clocks
        Graphics                          : 930 MHz
        Memory                            : 1215 MHz
    Default Applications Clocks
        Graphics                          : 930 MHz
        Memory                            : 1215 MHz
    Deferred Clocks
        Memory                            : N/A
    Max Clocks
        Graphics                          : 1440 MHz
        SM                                : 1440 MHz
        Memory                            : 1215 MHz
        Video                             : 1305 MHz
    Max Customer Boost Clocks
        Graphics                          : 1440 MHz
    Clock Policy
        Auto Boost                        : N/A
        Auto Boost Default                : N/A
    Voltage
        Graphics                          : 693.750 mV
    Fabric
        State                             : N/A
        Status                            : N/A
    Processes
        GPU instance ID                   : N/A
        Compute instance ID               : N/A
        Process ID                        : 8761
            Type                          : G
            Name                          : /usr/lib/xorg/Xorg
            Used GPU Memory               : 4 MiB

GPU 00000000:BE:00.0
    Product Name                          : NVIDIA A30
    Product Brand                         : NVIDIA
    Product Architecture                  : Ampere
    Display Mode                          : Enabled
    Display Active                        : Disabled
    Persistence Mode                      : Disabled
    Addressing Mode                       : None
    MIG Mode
        Current                           : Disabled
        Pending                           : Disabled
    Accounting Mode                       : Disabled
    Accounting Mode Buffer Size           : 4000
    Driver Model
        Current                           : N/A
        Pending                           : N/A
    Serial Number                         : 1655223001670
    GPU UUID                              : GPU-de2dd4e8-d9e5-18aa-3ff9-a68acad519e7
    Minor Number                          : 8
    VBIOS Version                         : 92.00.66.00.04
    MultiGPU Board                        : No
    Board ID                              : 0xbe00
    Board Part Number                     : 900-21001-0040-100
    GPU Part Number                       : 20B7-882-A1
    FRU Part Number                       : N/A
    Module ID                             : 1
    Inforom Version
        Image Version                     : 1001.0205.00.02
        OEM Object                        : 2.0
        ECC Object                        : 6.16
        Power Management Object           : N/A
    Inforom BBX Object Flush
        Latest Timestamp                  : N/A
        Latest Duration                   : N/A
    GPU Operation Mode
        Current                           : N/A
        Pending                           : N/A
    GSP Firmware Version                  : 535.309.01
    GPU Virtualization Mode
        Virtualization Mode               : None
        Host VGPU Mode                    : N/A
    GPU Reset Status
        Reset Required                    : No
        Drain and Reset Recommended       : No
    IBMNPU
        Relaxed Ordering Mode             : N/A
    PCI
        Bus                               : 0xBE
        Device                            : 0x00
        Domain                            : 0x0000
        Device Id                         : 0x20B710DE
        Bus Id                            : 00000000:BE:00.0
        Sub System Id                     : 0x153210DE
        GPU Link Info
            PCIe Generation
                Max                       : 4
                Current                   : 4
                Device Current            : 4
                Device Max                : 4
                Host Max                  : 5
            Link Width
                Max                       : 16x
                Current                   : 16x
        Bridge Chip
            Type                          : N/A
            Firmware                      : N/A
        Replays Since Reset               : 0
        Replay Number Rollovers           : 0
        Tx Throughput                     : 0 KB/s
        Rx Throughput                     : 0 KB/s
        Atomic Caps Inbound               : N/A
        Atomic Caps Outbound              : N/A
    Fan Speed                             : N/A
    Performance State                     : P0
    Clocks Event Reasons
        Idle                              : Active
        Applications Clocks Setting       : Not Active
        SW Power Cap                      : Not Active
        HW Slowdown                       : Not Active
            HW Thermal Slowdown           : Not Active
            HW Power Brake Slowdown       : Not Active
        Sync Boost                        : Not Active
        SW Thermal Slowdown               : Not Active
        Display Clock Setting             : Not Active
    Sparse Operation Mode                 : N/A
    FB Memory Usage
        Total                             : 24576 MiB
        Reserved                          : 513 MiB
        Used                              : 12 MiB
        Free                              : 24049 MiB
    BAR1 Memory Usage
        Total                             : 32768 MiB
        Used                              : 2 MiB
        Free                              : 32766 MiB
    Conf Compute Protected Memory Usage
        Total                             : 0 MiB
        Used                              : 0 MiB
        Free                              : 0 MiB
    Compute Mode                          : Default
    Utilization
        Gpu                               : 0 %
        Memory                            : 0 %
        Encoder                           : 0 %
        Decoder                           : 0 %
        JPEG                              : 0 %
        OFA                               : 0 %
    Encoder Stats
        Active Sessions                   : 0
        Average FPS                       : 0
        Average Latency                   : 0
    FBC Stats
        Active Sessions                   : 0
        Average FPS                       : 0
        Average Latency                   : 0
    ECC Mode
        Current                           : Enabled
        Pending                           : Enabled
    ECC Errors
        Volatile
            SRAM Correctable              : 0
            SRAM Uncorrectable Parity     : 0
            SRAM Uncorrectable SEC-DED    : 0
            DRAM Correctable              : 0
            DRAM Uncorrectable            : 0
        Aggregate
            SRAM Correctable              : 0
            SRAM Uncorrectable Parity     : 0
            SRAM Uncorrectable SEC-DED    : 0
            DRAM Correctable              : 0
            DRAM Uncorrectable            : 0
            SRAM Threshold Exceeded       : No
        Aggregate Uncorrectable SRAM Sources
            SRAM L2                       : 0
            SRAM SM                       : 0
            SRAM Microcontroller          : 0
            SRAM PCIE                     : 0
            SRAM Other                    : 0
    Retired Pages
        Single Bit ECC                    : N/A
        Double Bit ECC                    : N/A
        Pending Page Blacklist            : N/A
    Remapped Rows
        Correctable Error                 : 0
        Uncorrectable Error               : 0
        Pending                           : No
        Remapping Failure Occurred        : No
        Bank Remap Availability Histogram
            Max                           : 384 bank(s)
            High                          : 0 bank(s)
            Partial                       : 0 bank(s)
            Low                           : 0 bank(s)
            None                          : 0 bank(s)
    Temperature
        GPU Current Temp                  : 40 C
        GPU T.Limit Temp                  : N/A
        GPU Shutdown Temp                 : 100 C
        GPU Slowdown Temp                 : 97 C
        GPU Max Operating Temp            : 90 C
        GPU Target Temperature            : N/A
        Memory Current Temp               : 41 C
        Memory Max Operating Temp         : 95 C
    GPU Power Readings
        Power Draw                        : 33.28 W
        Current Power Limit               : 165.00 W
        Requested Power Limit             : 165.00 W
        Default Power Limit               : 165.00 W
        Min Power Limit                   : 100.00 W
        Max Power Limit                   : 165.00 W
    Module Power Readings
        Power Draw                        : N/A
        Current Power Limit               : N/A
        Requested Power Limit             : N/A
        Default Power Limit               : N/A
        Min Power Limit                   : N/A
        Max Power Limit                   : N/A
    Clocks
        Graphics                          : 210 MHz
        SM                                : 210 MHz
        Memory                            : 1215 MHz
        Video                             : 585 MHz
    Applications Clocks
        Graphics                          : 930 MHz
        Memory                            : 1215 MHz
    Default Applications Clocks
        Graphics                          : 930 MHz
        Memory                            : 1215 MHz
    Deferred Clocks
        Memory                            : N/A
    Max Clocks
        Graphics                          : 1440 MHz
        SM                                : 1440 MHz
        Memory                            : 1215 MHz
        Video                             : 1305 MHz
    Max Customer Boost Clocks
        Graphics                          : 1440 MHz
    Clock Policy
        Auto Boost                        : N/A
        Auto Boost Default                : N/A
    Voltage
        Graphics                          : 668.750 mV
    Fabric
        State                             : N/A
        Status                            : N/A
    Processes
        GPU instance ID                   : N/A
        Compute instance ID               : N/A
        Process ID                        : 8761
            Type                          : G
            Name                          : /usr/lib/xorg/Xorg
            Used GPU Memory               : 4 MiB

