diff --git a/device/dell/x86_64-dellemc_z9264f_c3538-r0/plugins/sfputil.py b/device/dell/x86_64-dellemc_z9264f_c3538-r0/plugins/sfputil.py index 192fb80f6c56..24938a122c0e 100644 --- a/device/dell/x86_64-dellemc_z9264f_c3538-r0/plugins/sfputil.py +++ b/device/dell/x86_64-dellemc_z9264f_c3538-r0/plugins/sfputil.py @@ -8,6 +8,7 @@ import sys import getopt import time + import select from sonic_sfp.sfputilbase import SfpUtilBase from os import * from mmap import * @@ -24,6 +25,10 @@ class SfpUtil(SfpUtilBase): PORTS_IN_BLOCK = 64 BASE_RES_PATH = "/sys/bus/pci/devices/0000:04:00.0/resource0" + OIR_FD_PATH = "/sys/bus/pci/devices/0000:04:00.0/port_msi" + + oir_fd = -1 + epoll = -1 _port_to_eeprom_mapping = {} @@ -47,8 +52,8 @@ def port_to_eeprom_mapping(self): def pci_mem_read(self, mm, offset): mm.seek(offset) - read_data_stream=mm.read(4) - reg_val=struct.unpack('I',read_data_stream) + read_data_stream = mm.read(4) + reg_val = struct.unpack('I', read_data_stream) mem_val = str(reg_val)[1:-2] # print "reg_val read:%x"%reg_val return mem_val @@ -56,7 +61,7 @@ def pci_mem_read(self, mm, offset): def pci_mem_write(self, mm, offset, data): mm.seek(offset) # print "data to write:%x"%data - mm.write(struct.pack('I',data)) + mm.write(struct.pack('I', data)) def pci_set_value(self, resource, val, offset): fd = open(resource, O_RDWR) @@ -73,7 +78,7 @@ def pci_get_value(self, resource, offset): mm.close() close(fd) return val - + def init_global_port_presence(self): for port_num in range(self.port_start, (self.port_end + 1)): presence = self.get_presence(port_num) @@ -81,17 +86,15 @@ def init_global_port_presence(self): self._global_port_pres_dict[port_num] = '1' else: self._global_port_pres_dict[port_num] = '0' - + def __init__(self): eeprom_path = "/sys/class/i2c-adapter/i2c-{0}/{0}-0050/eeprom" for x in range(self.port_start, self.port_end + 1): - port_num = x + 1 - self.port_to_eeprom_mapping[x] = eeprom_path.format( - port_num) + port_num = x + 1 + self.port_to_eeprom_mapping[x] = eeprom_path.format(port_num) port_num = 0 self.init_global_port_presence() - SfpUtilBase.__init__(self) def get_presence(self, port_num): @@ -100,13 +103,13 @@ def get_presence(self, port_num): return False # Port offset starts with 0x4004 - port_offset = 16388 + ((port_num-1) * 16) + port_offset = 16388 + ((port_num-1) * 16) + + status = self.pci_get_value(self.BASE_RES_PATH, port_offset) + reg_value = int(status) - status = self.pci_get_value(self.BASE_RES_PATH, port_offset) - reg_value = int(status) - # Absence of status throws error - if (reg_value == "" ): + if (reg_value == ""): return False # Mask off 4th bit for presence @@ -124,14 +127,14 @@ def get_low_power_mode(self, port_num): if port_num < self.port_start or port_num > self.port_end: return False - # Port offset starts with 0x4000 - port_offset = 16384 + ((port_num-1) * 16) + # Port offset starts with 0x4000 + port_offset = 16384 + ((port_num-1) * 16) - status = self.pci_get_value(self.BASE_RES_PATH, port_offset) - reg_value = int(status) + status = self.pci_get_value(self.BASE_RES_PATH, port_offset) + reg_value = int(status) # Absence of status throws error - if (reg_value == "" ): + if (reg_value == ""): return False # Mask off 4th bit for presence @@ -149,44 +152,44 @@ def set_low_power_mode(self, port_num, lpmode): if port_num < self.port_start or port_num > self.port_end: return False - # Port offset starts with 0x4000 - port_offset = 16384 + ((port_num-1) * 16) + # Port offset starts with 0x4000 + port_offset = 16384 + ((port_num-1) * 16) - status = self.pci_get_value(self.BASE_RES_PATH, port_offset) - reg_value = int(status) + status = self.pci_get_value(self.BASE_RES_PATH, port_offset) + reg_value = int(status) # Absence of status throws error - if (reg_value == "" ): + if (reg_value == ""): return False # Mask off 4th bit for presence mask = (1 << 6) - - # LPMode is active high; set or clear the bit accordingly + + # LPMode is active high; set or clear the bit accordingly if lpmode is True: reg_value = reg_value | mask else: reg_value = reg_value & ~mask # Convert our register value back to a hex string and write back - status = self.pci_set_value(self.BASE_RES_PATH, reg_value, port_offset) + status = self.pci_set_value(self.BASE_RES_PATH, reg_value, port_offset) return True def reset(self, port_num): - # Check for invalid port_num + # Check for invalid port_num if port_num < self.port_start or port_num > self.port_end: return False - # Port offset starts with 0x4000 - port_offset = 16384 + ((port_num-1) * 16) + # Port offset starts with 0x4000 + port_offset = 16384 + ((port_num-1) * 16) - status = self.pci_get_value(self.BASE_RES_PATH, port_offset) - reg_value = int(status) + status = self.pci_get_value(self.BASE_RES_PATH, port_offset) + reg_value = int(status) # Absence of status throws error - if (reg_value == "" ): + if (reg_value == ""): return False # Mask off 4th bit for presence @@ -195,33 +198,106 @@ def reset(self, port_num): # ResetL is active low reg_value = reg_value & ~mask - # Convert our register value back to a hex string and write back - status = self.pci_set_value(self.BASE_RES_PATH, reg_value, port_offset) + # Convert our register value back to a hex string and write back + status = self.pci_set_value(self.BASE_RES_PATH, reg_value, port_offset) # Sleep 1 second to allow it to settle time.sleep(1) reg_value = reg_value | mask - # Convert our register value back to a hex string and write back - status = self.pci_set_value(self.BASE_RES_PATH, reg_value, port_offset) + # Convert our register value back to a hex string and write back + status = self.pci_set_value(self.BASE_RES_PATH, reg_value, port_offset) return True - def get_transceiver_change_event(self): - port_dict = {} - while True: + def get_register(self, reg_file): + retval = 'ERR' + if (not path.isfile(reg_file)): + print reg_file, 'not found !' + return retval + + try: + with fdopen(open(reg_file, O_RDONLY)) as fd: + retval = fd.read() + except Exception as error: + logging.error("Unable to open ", reg_file, "file !") + + retval = retval.rstrip('\r\n') + retval = retval.lstrip(" ") + return retval + + def check_interrupts(self, port_dict): + retval = 0 + is_port_dict_updated = False for port_num in range(self.port_start, (self.port_end + 1)): presence = self.get_presence(port_num) if(presence and self._global_port_pres_dict[port_num] == '0'): + is_port_dict_updated = True self._global_port_pres_dict[port_num] = '1' port_dict[port_num] = '1' elif(not presence and self._global_port_pres_dict[port_num] == '1'): + is_port_dict_updated = True self._global_port_pres_dict[port_num] = '0' port_dict[port_num] = '0' - - if(len(port_dict) > 0): - return True, port_dict - - time.sleep(0.5) + return retval, is_port_dict_updated + + def get_transceiver_change_event(self, timeout=0): + port_dict = {} + try: + # We get notified when there is a MSI interrupt (vector 4/5)CVR + # Open the sysfs file and register the epoll object + self.oir_fd = fdopen(open(self.OIR_FD_PATH, O_RDONLY)) + if self.oir_fd != -1: + # Do a dummy read before epoll register + self.oir_fd.read() + self.epoll = select.epoll() + self.epoll.register( + self.oir_fd.fileno(), select.EPOLLIN & select.EPOLLET) + else: + print("get_transceiver_change_event : unable to create fd") + return False, {} + + # Check for missed interrupts by invoking self.check_interrupts + # which will update the port_dict. + while True: + interrupt_count_start = self.get_register(self.OIR_FD_PATH) + retval, is_port_dict_updated = \ + self.check_interrupts(port_dict) + if ((retval == 0) and (is_port_dict_updated is True)): + return True, port_dict + interrupt_count_end = self.get_register(self.OIR_FD_PATH) + if (interrupt_count_start == 'ERR' or + interrupt_count_end == 'ERR'): + print("get_transceiver_change_event : \ + unable to retrive interrupt count") + break + + # check_interrupts() itself may take upto 100s of msecs. + # We detect a missed interrupt based on the count + if interrupt_count_start == interrupt_count_end: + break + + # Block until an xcvr is inserted or removed with timeout = -1 + events = self.epoll.poll( + timeout=timeout if timeout != 0 else -1) + if events: + # check interrupts and return the port_dict + retval, is_port_dict_updated = \ + self.check_interrupts(port_dict) + if (retval != 0): + return False, {} + + return True, port_dict + except: + return False, {} + finally: + if self.oir_fd != -1: + self.epoll.unregister(self.oir_fd.fileno()) + self.epoll.close() + self.oir_fd.close() + self.oir_fd = -1 + self.epoll = -1 + + return False, {} \ No newline at end of file diff --git a/platform/broadcom/sonic-platform-modules-dell/debian/platform-modules-z9264f.install b/platform/broadcom/sonic-platform-modules-dell/debian/platform-modules-z9264f.install index 43e876e40e5d..6a2f15511d66 100644 --- a/platform/broadcom/sonic-platform-modules-dell/debian/platform-modules-z9264f.install +++ b/platform/broadcom/sonic-platform-modules-dell/debian/platform-modules-z9264f.install @@ -3,6 +3,7 @@ z9264f/scripts/check_qsfp.sh usr/local/bin z9264f/scripts/platform_sensors.py usr/local/bin z9264f/scripts/sensors usr/bin z9264f/scripts/pcisysfs.py usr/bin +z9264f/scripts/qsfp_irq_enable.py usr/bin z9264f/cfg/z9264f-modules.conf etc/modules-load.d z9264f/systemd/platform-modules-z9264f.service etc/systemd/system common/platform_reboot usr/share/sonic/device/x86_64-dellemc_z9264f_c3538-r0 diff --git a/platform/broadcom/sonic-platform-modules-dell/z9264f/modules/dell_z9264f_fpga_ocores.c b/platform/broadcom/sonic-platform-modules-dell/z9264f/modules/dell_z9264f_fpga_ocores.c index c08a4c210c53..d3a4a51ead72 100644 --- a/platform/broadcom/sonic-platform-modules-dell/z9264f/modules/dell_z9264f_fpga_ocores.c +++ b/platform/broadcom/sonic-platform-modules-dell/z9264f/modules/dell_z9264f_fpga_ocores.c @@ -19,6 +19,7 @@ * @file fpga_ocores.c * @brief This is a driver to interface with Linux Open Cores driver for FPGA i2c access * + * 2019/5/9 - PCIe Interrupt supported is added for OIR Events. ************************************************************************/ #include #include @@ -89,7 +90,7 @@ struct fpgapci_dev { unsigned int irq_first; unsigned int irq_length; unsigned int irq_assigned; - + unsigned int xcvr_intr_count; }; static int use_irq = 1; @@ -226,6 +227,14 @@ enum { #define I2C_PCI_BUS_NUM_12 12 #define I2C_PCI_BUS_NUM_16 16 + +#define IRQ_LTCH_STS 0x20 +#define PRSNT_LTCH_STS 0x10 + +#define PORT_CTRL_OFFSET 0x4000 +#define PORT_STS_OFFSET 0x4004 +#define PORT_IRQ_STS_OFFSET 0x4008 +#define PORT_IRQ_EN_OFFSET 0x400C #define MB_BRD_REV_TYPE 0x0008 #define MB_BRD_REV_MASK 0x00f0 #define MB_BRD_REV_00 0x0000 @@ -248,7 +257,7 @@ enum { #define BRD_TYPE_S5232_NON_NEBS 0xc #define BRD_TYPE_S5232_NEBS 0xd -#define FPGA_CTL_REG_SIZE 0x60 +#define FPGA_CTL_REG_SIZE 0x6000 #define MSI_VECTOR_MAP_MASK 0x1f #define MSI_VECTOR_MAP1 0x58 #define I2C_CH1_MSI_MAP_VECT_8 0x00000008 @@ -284,6 +293,8 @@ enum { #define MSI_VECTOR_REV_00 16 #define MSI_VECTOR_REV_01 32 +#define FPGA_MSI_VECTOR_ID_4 4 +#define FPGA_MSI_VECTOR_ID_5 5 #define FPGA_MSI_VECTOR_ID_8 8 #define FPGA_MSI_VECTOR_ID_9 9 #define FPGA_MSI_VECTOR_ID_10 10 @@ -506,6 +517,73 @@ static int fpgai2c_poll(struct fpgalogic_i2c *i2c) return 0; } +static ssize_t get_mod_msi(struct device *dev, struct device_attribute *devattr, char *buf) +{ + int ind = 0, port_status=0, port_irq_status=0; + struct fpgapci_dev *fpgapci = (struct fpgapci_dev*) dev_get_drvdata(dev); + PRINT("%s:xcvr_intr_count:%u\n", __FUNCTION__, fpgapci->xcvr_intr_count); + for(ind=0;ind<64;ind++) + { + port_status = ioread32(fpga_ctl_addr + PORT_STS_OFFSET + (ind*16)); + port_irq_status = ioread32(fpga_ctl_addr + PORT_IRQ_STS_OFFSET + (ind*16)); + PRINT("%s:port:%d, port_status:%#x, port_irq_status:%#x\n", __FUNCTION__, ind, port_status, port_irq_status); + } + return sprintf(buf,"0x%04x\n",fpgapci->xcvr_intr_count); +} +static DEVICE_ATTR(port_msi, S_IRUGO, get_mod_msi, NULL); + +static struct attribute *port_attrs[] = { + &dev_attr_port_msi.attr, + NULL, +}; + +static struct attribute_group port_attr_grp = { + .attrs = port_attrs, +}; + + +static irqreturn_t fpgaport_1_32_isr(int irq, void *dev) +{ + struct pci_dev *pdev = dev; + struct fpgapci_dev *fpgapci = (struct fpgapci_dev*) dev_get_drvdata(&pdev->dev); + int ind = 0, port_status=0, port_irq_status=0; + for(ind=0;ind<32;ind++) + { + port_irq_status = ioread32(fpga_ctl_addr + PORT_IRQ_STS_OFFSET + (ind*16)); + if(port_irq_status&(IRQ_LTCH_STS|PRSNT_LTCH_STS)) + { + PRINT("%s:port:%d, port_status:%#x, port_irq_status:%#x\n", __FUNCTION__, ind, port_status, port_irq_status); + //write on clear + iowrite32( IRQ_LTCH_STS|PRSNT_LTCH_STS,fpga_ctl_addr + PORT_IRQ_STS_OFFSET + (ind*16)); + } + } + fpgapci->xcvr_intr_count++; + PRINT("%s: xcvr_intr_count:%u\n", __FUNCTION__, fpgapci->xcvr_intr_count); + sysfs_notify(&pdev->dev.kobj, NULL, "port_msi"); + return IRQ_HANDLED; +} + +static irqreturn_t fpgaport_33_64_isr(int irq, void *dev) +{ + struct pci_dev *pdev = dev; + struct fpgapci_dev *fpgapci = (struct fpgapci_dev*) dev_get_drvdata(&pdev->dev); + int ind = 0, port_status=0, port_irq_status=0; + for(ind=32;ind<64;ind++) + { + port_irq_status = ioread32(fpga_ctl_addr + PORT_IRQ_STS_OFFSET + (ind*16)); + if(port_irq_status| (IRQ_LTCH_STS|PRSNT_LTCH_STS)) + { + PRINT("%s:port:%d, port_status:%#x, port_irq_status:%#x\n", __FUNCTION__, ind, port_status, port_irq_status); + //write on clear + iowrite32( IRQ_LTCH_STS|PRSNT_LTCH_STS,fpga_ctl_addr + PORT_IRQ_STS_OFFSET + (ind*16)); + } + } + fpgapci->xcvr_intr_count++; + PRINT("%s: xcvr_intr_count:%u\n", __FUNCTION__, fpgapci->xcvr_intr_count); + sysfs_notify(&pdev->dev.kobj, NULL, "port_msi"); + return IRQ_HANDLED; +} + static void fpgai2c_process(struct fpgalogic_i2c *i2c) { struct i2c_msg *msg = i2c->msg; @@ -1028,6 +1106,18 @@ static int register_intr_handler(struct pci_dev *dev, int irq_num_id) switch(irq_num_id) { /* Currently we only support test vector 2 for FPGA Logic I2C channel * controller 1-7 interrupt*/ + case FPGA_MSI_VECTOR_ID_4: + err = request_irq(dev->irq + irq_num_id, fpgaport_1_32_isr, IRQF_EARLY_RESUME, + FPGA_PCI_NAME, dev); + PRINT ( "%d: fpgapci_dev: irq: %d, %d\n", __LINE__, dev->irq, irq_num_id); + fpgapci->irq_assigned++; + break; + case FPGA_MSI_VECTOR_ID_5: + err = request_irq(dev->irq + irq_num_id, fpgaport_33_64_isr, IRQF_EARLY_RESUME, + FPGA_PCI_NAME, dev); + PRINT ( "%d: fpgapci_dev: irq: %d, %d\n", __LINE__, dev->irq, irq_num_id); + fpgapci->irq_assigned++; + break; case FPGA_MSI_VECTOR_ID_8: err = request_irq(dev->irq + irq_num_id, fpgai2c_isr, IRQF_EARLY_RESUME, FPGA_PCI_NAME, &fpgalogic_i2c[0]); @@ -1074,6 +1164,18 @@ static int register_intr_handler(struct pci_dev *dev, int irq_num_id) ((board_rev_type & MB_BRD_REV_MASK) == MB_BRD_REV_03)) { /* FPGA SPEC 4.3.1.34, First i2c channel mapped to vector 8 */ switch (irq_num_id) { + case FPGA_MSI_VECTOR_ID_4: + err = request_irq(dev->irq + irq_num_id, fpgaport_1_32_isr, IRQF_EARLY_RESUME, + FPGA_PCI_NAME, dev); + PRINT ( "%d: fpgapci_dev: irq: %d, %d\n", __LINE__, dev->irq, irq_num_id); + fpgapci->irq_assigned++; + break; + case FPGA_MSI_VECTOR_ID_5: + err = request_irq(dev->irq + irq_num_id, fpgaport_33_64_isr, IRQF_EARLY_RESUME, + FPGA_PCI_NAME, dev); + PRINT ( "%d: fpgapci_dev: irq: %d, %d\n", __LINE__, dev->irq, irq_num_id); + fpgapci->irq_assigned++; + break; case FPGA_MSI_VECTOR_ID_8: err = request_irq(dev->irq + irq_num_id, fpgai2c_isr, IRQF_EARLY_RESUME, FPGA_PCI_NAME, &fpgalogic_i2c[0]); @@ -1387,6 +1489,7 @@ static int fpgapci_configure_msi(struct fpgapci_dev *fpgapci,struct pci_dev *dev static int fpgapci_probe(struct pci_dev *dev, const struct pci_device_id *id) { struct fpgapci_dev *fpgapci = 0; + int status = 0; #ifdef TEST PRINT ( " vendor = 0x%x, device = 0x%x, class = 0x%x, bus:slot.func = %02x:%02x.%02x\n", @@ -1404,6 +1507,10 @@ static int fpgapci_probe(struct pci_dev *dev, const struct pci_device_id *id) dev_set_drvdata(&dev->dev, (void*)fpgapci); fpgapci->upstream = find_upstream_dev (dev); + status = sysfs_create_group(&dev->dev.kobj, &port_attr_grp); + if (status) { + printk(KERN_INFO "%s:Cannot create sysfs\n", __FUNCTION__); + } if(fpgapci_setup_device(fpgapci,dev)) { goto error_no_device; diff --git a/platform/broadcom/sonic-platform-modules-dell/z9264f/scripts/qsfp_irq_enable.py b/platform/broadcom/sonic-platform-modules-dell/z9264f/scripts/qsfp_irq_enable.py new file mode 100755 index 000000000000..b2eb03dd2011 --- /dev/null +++ b/platform/broadcom/sonic-platform-modules-dell/z9264f/scripts/qsfp_irq_enable.py @@ -0,0 +1,32 @@ +#!/usr/bin/python + +try: + import struct + import sys + from os import * + from mmap import * + +except ImportError as e: + raise ImportError("%s - required module no found" % str(e)) + +BASE_RES_PATH = "/sys/bus/pci/devices/0000:04:00.0/resource0" +PORT_START = 0 +PORT_END = 64 + + +def pci_mem_write(mm, offset, data): + mm.seek(offset) + mm.write(struct.pack('I', data)) + + +def pci_set_value(resource, val, offset): + fd = open(resource, O_RDWR) + mm = mmap(fd, 0) + val = pci_mem_write(mm, offset, val) + mm.close() + close(fd) + return val + +for port_num in range(PORT_START, PORT_END+1): + port_offset = 0x400c + ((port_num) * 16) + pci_set_value(BASE_RES_PATH, 0x30, port_offset) diff --git a/platform/broadcom/sonic-platform-modules-dell/z9264f/scripts/z9264f_platform.sh b/platform/broadcom/sonic-platform-modules-dell/z9264f/scripts/z9264f_platform.sh index 5002ce72c081..4c0646333575 100755 --- a/platform/broadcom/sonic-platform-modules-dell/z9264f/scripts/z9264f_platform.sh +++ b/platform/broadcom/sonic-platform-modules-dell/z9264f/scripts/z9264f_platform.sh @@ -98,6 +98,7 @@ if [ "$1" == "init" ]; then switch_board_qsfp_mux "new_device" switch_board_qsfp "new_device" switch_board_modsel + python /usr/bin/qsfp_irq_enable.py elif [ "$1" == "deinit" ]; then sys_eeprom "delete_device"