Formatting Python Output - regex

I am trying to get server information and add it to dictionary for later to be able to create a web page that contains this information.
How would you format this? Would you add it to dictionary or keep it as a list?
Command: >>> hpilo.Ilo.get_embedded_health(server_conn):
Outout:
{'fans': {'Fan 2': {'status': 'OK', 'speed': (13, 'Percentage'), 'zone': 'System', 'label': 'Fan 2'}, 'Fan 3': {'status': 'OK', 'speed': (6, 'Percentage'), 'zone': 'System', 'label': 'Fan 3'}, 'Fan 1': {'status': 'OK', 'speed': (6, 'Percentage'), 'zone': 'System', 'label': 'Fan 1'}, 'Fan 6': {'status': 'OK', 'speed': (43, 'Percentage'), 'zone': 'System', 'label': 'Fan 6'}, 'Fan 4': {'status': 'OK', 'speed': (34, 'Percentage'), 'zone': 'System', 'label': 'Fan 4'}, 'Fan 5': {'status': 'OK', 'speed': (43, 'Percentage'), 'zone': 'System', 'label': 'Fan 5'}}, 'health_at_a_glance': {'temperature': {'status': 'OK'}, 'storage': {'status': 'OK'}, 'fans': {'status': 'OK', 'redundancy': 'Redundant'}, 'bios_hardware': {'status': 'OK'}, 'memory': {'status': 'OK'}, 'power_supplies': {'status': 'OK', 'redundancy': 'Redundant'}, 'processor': {'status': 'OK'}, 'network': {'status': 'OK'}}, 'temperature': {'05-P1 DIMM 4-6': {'status': 'OK', 'currentreading': (21, 'Celsius'), 'label': '05-P1 DIMM 4-6', 'critical': 'N/A', 'caution': (87, 'Celsius'), 'location': 'Memory'}, '12-HD Max': {'status': 'OK', 'currentreading': (35, 'Celsius'), 'label': '12-HD Max', 'critical': 'N/A', 'caution': (60, 'Celsius'), 'location': 'System'}, '10-P2 DIMM 7-9': {'status': 'OK', 'currentreading': (21, 'Celsius'), 'label': '10-P2 DIMM 7-9', 'critical': 'N/A', 'caution': (87, 'Celsius'), 'location': 'Memory'}, '20-VR P1 Mem': {'status': 'OK', 'currentreading': (27, 'Celsius'), 'label': '20-VR P1 Mem', 'critical': (120, 'Celsius'), 'caution': (115, 'Celsius'), 'location': 'Power Supply'}, '32-PCI 5': {'status': 'Not Installed', 'currentreading': 'N/A', 'label': '32-PCI 5', 'critical': 'N/A', 'caution': 'N/A', 'location': 'I/O Board'}, '27-LOM Card': {'status': 'Not Installed', 'currentreading': 'N/A', 'label': '27-LOM Card', 'critical': 'N/A', 'caution': 'N/A', 'location': 'I/O Board'}, '38-PCI 5 Zone': {'status': 'Not Installed', 'currentreading': 'N/A', 'label': '38-PCI 5 Zone', 'critical': 'N/A', 'caution': 'N/A', 'location': 'I/O Board'}, '14-P/S 1': {'status': 'OK', 'currentreading': (21, 'Celsius'), 'label': '14-P/S 1', 'critical': 'N/A', 'caution': 'N/A', 'location': 'Power Supply'}, '17-VR P1': {'status': 'OK', 'currentreading': (26, 'Celsius'), 'label': '17-VR P1', 'critical': (120, 'Celsius'), 'caution': (115, 'Celsius'), 'location': 'Power Supply'}, '45-SuperCap Max': {'status': 'OK', 'currentreading': (19, 'Celsius'), 'label': '45-SuperCap Max', 'critical': 'N/A', 'caution': (65, 'Celsius'), 'location': 'System'}, '19-VR P1 Mem': {'status': 'OK', 'currentreading': (31, 'Celsius'), 'label': '19-VR P1 Mem', 'critical': (120, 'Celsius'), 'caution': (115, 'Celsius'), 'location': 'Power Supply'}, '30-PCI 3': {'status': 'Not Installed', 'currentreading': 'N/A', 'label': '30-PCI 3', 'critical': 'N/A', 'caution': 'N/A', 'location': 'I/O Board'}, '07-P1 DIMM 10-12': {'status': 'OK', 'currentreading': (20, 'Celsius'), 'label': '07-P1 DIMM 10-12', 'critical': 'N/A', 'caution': (87, 'Celsius'), 'location': 'Memory'}, '44-System Board': {'status': 'OK', 'currentreading': (23, 'Celsius'), 'label': '44-System Board', 'critical': (85, 'Celsius'), 'caution': (80, 'Celsius'), 'location': 'System'}, '24-VR P2Vtt Zone': {'status': 'OK', 'currentreading': (26, 'Celsius'), 'label': '24-VR P2Vtt Zone', 'critical': (95, 'Celsius'), 'caution': (90, 'Celsius'), 'location': 'Power Supply'}, '01-Inlet Ambient': {'status': 'OK', 'currentreading': (17, 'Celsius'), 'label': '01-Inlet Ambient', 'critical': (46, 'Celsius'), 'caution': (42, 'Celsius'), 'location': 'Ambient'}, '31-PCI 4': {'status': 'Not Installed', 'currentreading': 'N/A', 'label': '31-PCI 4', 'critical': 'N/A', 'caution': 'N/A', 'location': 'I/O Board'}, '03-CPU 2': {'status': 'OK', 'currentreading': (40, 'Celsius'), 'label': '03-CPU 2', 'critical': 'N/A', 'caution': (70, 'Celsius'), 'location': 'CPU'}, '43-BIOS Zone': {'status': 'OK', 'currentreading': (30, 'Celsius'), 'label': '43-BIOS Zone', 'critical': (95, 'Celsius'), 'caution': (90, 'Celsius'), 'location': 'System Board'}, '29-PCI 2': {'status': 'Not Installed', 'currentreading': 'N/A', 'label': '29-PCI 2', 'critical': 'N/A', 'caution': 'N/A', 'location': 'I/O Board'}, '34-PCI 1 Zone': {'status': 'OK', 'currentreading': (21, 'Celsius'), 'label': '34-PCI 1 Zone', 'critical': (70, 'Celsius'), 'caution': (65, 'Celsius'), 'location': 'I/O Board'}, '47-Battery Zone': {'status': 'OK', 'currentreading': (25, 'Celsius'), 'label': '47-Battery Zone', 'critical': (80, 'Celsius'), 'caution': (75, 'Celsius'), 'location': 'System'}, '21-VR P2 Mem': {'status': 'OK', 'currentreading': (30, 'Celsius'), 'label': '21-VR P2 Mem', 'critical': (120, 'Celsius'), 'caution': (115, 'Celsius'), 'location': 'Power Supply'}, '13-Chipset': {'status': 'OK', 'currentreading': (44, 'Celsius'), 'label': '13-Chipset', 'critical': 'N/A', 'caution': (105, 'Celsius'), 'location': 'System'}, '02-CPU 1': {'status': 'OK', 'currentreading': (40, 'Celsius'), 'label': '02-CPU 1', 'critical': 'N/A', 'caution': (70, 'Celsius'), 'location': 'CPU'}, '42-VR P1 Zone': {'status': 'OK', 'currentreading': (20, 'Celsius'), 'label': '42-VR P1 Zone', 'critical': (100, 'Celsius'), 'caution': (95, 'Celsius'), 'location': 'Power Supply'}, '23-VR P1Vtt Zone': {'status': 'OK', 'currentreading': (24, 'Celsius'), 'label': '23-VR P1Vtt Zone', 'critical': (95, 'Celsius'), 'caution': (90, 'Celsius'), 'location': 'Power Supply'}, '37-PCI 4 Zone': {'status': 'Not Installed', 'currentreading': 'N/A', 'label': '37-PCI 4 Zone', 'critical': 'N/A', 'caution': 'N/A', 'location': 'I/O Board'}, '09-P2 DIMM 4-6': {'status': 'OK', 'currentreading': (28, 'Celsius'), 'label': '09-P2 DIMM 4-6', 'critical': 'N/A', 'caution': (87, 'Celsius'), 'location': 'Memory'}, '22-VR P2 Mem': {'status': 'OK', 'currentreading': (32, 'Celsius'), 'label': '22-VR P2 Mem', 'critical': (120, 'Celsius'), 'caution': (115, 'Celsius'), 'location': 'Power Supply'}, '26-iLO Zone': {'status': 'OK', 'currentreading': (26, 'Celsius'), 'label': '26-iLO Zone', 'critical': (95, 'Celsius'), 'caution': (90, 'Celsius'), 'location': 'System'}, '50-Sys Exhaust': {'status': 'OK', 'currentreading': (26, 'Celsius'), 'label': '50-Sys Exhaust', 'critical': (80, 'Celsius'), 'caution': (75, 'Celsius'), 'location': 'Chassis'}, '41-I/O Board 2': {'status': 'Not Installed', 'currentreading': 'N/A', 'label': '41-I/O Board 2', 'critical': 'N/A', 'caution': 'N/A', 'location': 'I/O Board'}, '48-I/O Zone': {'status': 'OK', 'currentreading': (27, 'Celsius'), 'label': '48-I/O Zone', 'critical': (80, 'Celsius'), 'caution': (75, 'Celsius'), 'location': 'System'}, '46-Chipset Zone': {'status': 'OK', 'currentreading': (26, 'Celsius'), 'label': '46-Chipset Zone', 'critical': (80, 'Celsius'), 'caution': (75, 'Celsius'), 'location': 'System'}, '04-P1 DIMM 1-3': {'status': 'OK', 'currentreading': (21, 'Celsius'), 'label': '04-P1 DIMM 1-3', 'critical': 'N/A', 'caution': (87, 'Celsius'), 'location': 'Memory'}, '06-P1 DIMM 7-9': {'status': 'OK', 'currentreading': (20, 'Celsius'), 'label': '06-P1 DIMM 7-9', 'critical': 'N/A', 'caution': (87, 'Celsius'), 'location': 'Memory'}, '39-PCI 6 Zone': {'status': 'Not Installed', 'currentreading': 'N/A', 'label': '39-PCI 6 Zone', 'critical': 'N/A', 'caution': 'N/A', 'location': 'I/O Board'}, '16-P/S 2 Zone': {'status': 'OK', 'currentreading': (24, 'Celsius'), 'label': '16-P/S 2 Zone', 'critical': (80, 'Celsius'), 'caution': (75, 'Celsius'), 'location': 'Power Supply'}, '18-VR P2': {'status': 'OK', 'currentreading': (30, 'Celsius'), 'label': '18-VR P2', 'critical': (120, 'Celsius'), 'caution': (115, 'Celsius'), 'location': 'Power Supply'}, '35-PCI 2 Zone': {'status': 'OK', 'currentreading': (23, 'Celsius'), 'label': '35-PCI 2 Zone', 'critical': (71, 'Celsius'), 'caution': (66, 'Celsius'), 'location': 'I/O Board'}, '08-P2 DIMM 1-3': {'status': 'OK', 'currentreading': (26, 'Celsius'), 'label': '08-P2 DIMM 1-3', 'critical': 'N/A', 'caution': (87, 'Celsius'), 'location': 'Memory'}, '40-I/O Board 1': {'status': 'OK', 'currentreading': (25, 'Celsius'), 'label': '40-I/O Board 1', 'critical': (71, 'Celsius'), 'caution': (66, 'Celsius'), 'location': 'I/O Board'}, '28-PCI 1': {'status': 'Not Installed', 'currentreading': 'N/A', 'label': '28-PCI 1', 'critical': 'N/A', 'caution': 'N/A', 'location': 'I/O Board'}, '49-Sys Exhaust': {'status': 'OK', 'currentreading': (26, 'Celsius'), 'label': '49-Sys Exhaust', 'critical': (80, 'Celsius'), 'caution': (75, 'Celsius'), 'location': 'Chassis'}, '25-HD Controller': {'status': 'OK', 'currentreading': (48, 'Celsius'), 'label': '25-HD Controller', 'critical': 'N/A', 'caution': (100, 'Celsius'), 'location': 'System'}, '36-PCI 3 Zone': {'status': 'OK', 'currentreading': (23, 'Celsius'), 'label': '36-PCI 3 Zone', 'critical': (71, 'Celsius'), 'caution': (66, 'Celsius'), 'location': 'I/O Board'}, '33-PCI 6': {'status': 'Not Installed', 'currentreading': 'N/A', 'label': '33-PCI 6', 'critical': 'N/A', 'caution': 'N/A', 'location': 'I/O Board'}, '15-P/S 2': {'status': 'OK', 'currentreading': (23, 'Celsius'), 'label': '15-P/S 2', 'critical': 'N/A', 'caution': 'N/A', 'location': 'Power Supply'}, '11-P2 DIMM 10-12': {'status': 'OK', 'currentreading': (21, 'Celsius'), 'label': '11-P2 DIMM 10-12', 'critical': 'N/A', 'caution': (87, 'Celsius'), 'location': 'Memory'}}, 'memory': {'memory_details': {'CPU_2': {'socket 3': {'status': 'Good, In Use', 'socket': 3, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 2': {'status': 'Good, In Use', 'socket': 2, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 1': {'status': 'Good, In Use', 'socket': 1, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 7': {'status': 'Good, In Use', 'socket': 7, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 6': {'status': 'Good, In Use', 'socket': 6, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 5': {'status': 'Good, In Use', 'socket': 5, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 4': {'status': 'Good, In Use', 'socket': 4, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 9': {'status': 'Good, In Use', 'socket': 9, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 8': {'status': 'Good, In Use', 'socket': 8, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 11': {'status': 'Good, In Use', 'socket': 11, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 10': {'status': 'Good, In Use', 'socket': 10, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 12': {'status': 'Good, In Use', 'socket': 12, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}}, 'CPU_1': {'socket 3': {'status': 'Good, In Use', 'socket': 3, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 2': {'status': 'Good, In Use', 'socket': 2, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 1': {'status': 'Good, In Use', 'socket': 1, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 7': {'status': 'Good, In Use', 'socket': 7, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 6': {'status': 'Good, In Use', 'socket': 6, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 5': {'status': 'Good, In Use', 'socket': 5, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 4': {'status': 'Good, In Use', 'socket': 4, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 9': {'status': 'Good, In Use', 'socket': 9, 'ranks': 1, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '4096 MB'}, 'socket 8': {'status': 'Good, In Use', 'socket': 8, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 11': {'status': 'Good, In Use', 'socket': 11, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 10': {'status': 'Good, In Use', 'socket': 10, 'ranks': 2, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '16384 MB'}, 'socket 12': {'status': 'Good, In Use', 'socket': 12, 'ranks': 1, 'technology': 'RDIMM', 'frequency': '1333 MHz', 'minimum_voltage': '1.35 v', 'hp_smart_memory': 'Yes', 'type': 'DIMM DDR3', 'size': '4096 MB'}}}, 'advanced_memory_protection': {'configured_amp_mode': 'Advanced ECC', 'available_amp_modes': 'On-line Spare, Advanced ECC', 'amp_mode_status': 'Advanced ECC'}, 'memory_details_summary': {'cpu_2': {'operating_frequency': '1067 MHz', 'operating_voltage': '1.35 v', 'total_memory_size': '192 GB', 'number_of_sockets': 12}, 'cpu_1': {'operating_frequency': '1067 MHz', 'operating_voltage': '1.35 v', 'total_memory_size': '168 GB', 'number_of_sockets': 12}}}, 'firmware_information': {'HP ProLiant System ROM Bootblock': '03/11/2012', 'iLO': '1.13 Nov 08 2012', 'Power Management Controller Firmware Bootloader': '2.7', 'SAS Programmable Logic Device': 'Version 0x0C', 'HP ProLiant System ROM': '12/14/2012', 'Server Platform Services (SPS) Firmware': '2.1.5.2B.4', 'HP ProLiant System ROM - Backup': '12/14/2012', 'System Programmable Logic Device': 'Version 0x2D', 'HP Smart Array P420i Controller': '3.42', 'Power Management Controller Firmware': '3.0'}, 'power_supplies': {'Power Supply 2': {'status': 'OK', 'label': 'Power Supply 2'}, 'Power Supply 1': {'status': 'OK', 'label': 'Power Supply 1'}}, 'storage': {'Controller on System Board': {'status': 'OK', 'controller_status': 'OK', 'cache_module_memory': '1048576 KB', 'drive_enclosures': [{'status': 'OK', 'drive_bay': 4, 'label': 'Port 1I Box 2'}, {'status': 'OK', 'drive_bay': 4, 'label': 'Port 2I Box 0'}], 'logical_drives': [{'status': 'OK', 'fault_tolerance': 'RAID 1/RAID 1+0', 'capacity': '136 GB', 'physical_drives': [{'status': 'OK', 'capacity': '136 GB', 'fw_version': 'HPD3', 'label': 'Port 1I Box 2 Bay 1', 'location': 'Port 1I Box 2 Bay 1', 'serial_number': '6XM2R28Z0000B348CPGE', 'model': 'EH0146FBQDC'}, {'status': 'OK', 'capacity': '136 GB', 'fw_version': 'HPD3', 'label': 'Port 1I Box 2 Bay 2', 'location': 'Port 1I Box 2 Bay 2', 'serial_number': '6XM2QLK90000B348DF7V', 'model': 'EH0146FBQDC'}], 'label': '01'}], 'fw_version': '3.42', 'label': 'Controller on System Board', 'cache_module_status': 'OK', 'cache_module_serial_num': 'PBKUC0BRH4A62N', 'serial_number': '5001438025BB71A0', 'model': 'HP Smart Array P420i Controller'}}, 'nic_information': {'iLO Dedicated Network Port': {'network_port': 'iLO Dedicated Network Port', 'status': 'OK', 'port_description': 'iLO Dedicated Network Port', 'ip_address': '10.1.7.84', 'mac_address': '28:92:4a:34:ef:70'}, 'Port 4': {'network_port': 'Port 4', 'status': 'Other', 'port_description': 'N/A', 'ip_address': 'N/A', 'mac_address': 'ac:16:2d:84:56:bb'}, 'Port 3': {'network_port': 'Port 3', 'status': 'Other', 'port_description': 'N/A', 'ip_address': 'N/A', 'mac_address': 'ac:16:2d:84:56:ba'}, 'Port 2': {'network_port': 'Port 2', 'status': 'Other', 'port_description': 'N/A', 'ip_address': 'N/A', 'mac_address': 'ac:16:2d:84:56:b9'}, 'Port 1': {'network_port': 'Port 1', 'status': 'Other', 'port_description': 'N/A', 'ip_address': 'N/A', 'mac_address': 'ac:16:2d:84:56:b8'}}, 'vrm': None, 'processors': {'Proc 2': {'status': 'OK', 'internal_l1_cache': '192 KB', 'name': ' Intel(R) Xeon(R) CPU E5-2620 0 # 2.00GHz ', 'memory_technology': '64-bit Capable', 'internal_l2_cache': '1536 KB', 'label': 'Proc 2', 'internal_l3_cache': '15360 KB', 'speed': '2000 MHz', 'execution_technology': '6/6 cores; 12 threads'}, 'Proc 1': {'status': 'OK', 'internal_l1_cache': '192 KB', 'name': ' Intel(R) Xeon(R) CPU E5-2620 0 # 2.00GHz ', 'memory_technology': '64-bit Capable', 'internal_l2_cache': '1536 KB', 'label': 'Proc 1', 'internal_l3_cache': '15360 KB', 'speed': '2000 MHz', 'execution_technology': '6/6 cores; 12 threads'}}}

pprint module can help you.
8.18. pprint — Data pretty printer
The pprint module provides a capability to “pretty-print” arbitrary Python data structures in a form which can be used as input to the interpreter. If the formatted structures include objects which are not fundamental Python types, the representation may not be loadable. This may be the case if objects such as files, sockets, classes, or instances are included, as well as many other built-in objects which are not representable as Python constants.
The formatted representation keeps objects on a single line if it can, and breaks them onto multiple lines if they don’t fit within the allowed width. Construct PrettyPrinter objects explicitly if you need to adjust the width constraint.

Related

Django Annotate with one model and include "annotated" records in queryset

I am fairly new to Django and have figured out the "basic" annotate function, but I would like to include the records from the annotated values. For example,
qs = People.objects.values('status').annotate(total=Count('id'),totalScore=Sum('score'),averageAge=Avg('age'))
I would like to include the summarized records in the result, in this case the individual people, like so...
[
{'status': 'One',
'total': 2,
'totalScore': 150,
'averageAge': 36,
'people': [
{
'id': 1,
'status': 'One',
'name': 'Jim',
'score': 80,
'age': 40
},
{
'id': 5,
'status': 'One',
'name': 'Jill',
'score': 70,
'age': 32
}
]},
{'status': 'Two',
'total': 1,
'totalScore': 85,
'averageAge': 42,
'people': [
{
'id': 3,
'status': 'Two',
'name': 'Jack',
'score': 85,
'age': 42
}
]
},...
Thanks for any help.

Response 400 while using Requests package in Django

I want to scrape Myntra using Django Framework but when I put my code in Django project I get 400 error. The same code runs without error when I run it on the same file even in Django. Means calling the MyntraScraperClass in MyntraScraper.py file.
Here is my project directory
BackendController
-MyntraScraper.py
myDjangoApp
-views.py
Inside views.py, there is a function where I am calling my MyntraScraperClass
def tst(request):
------------
The same code that is MyntraScraperClass runs error free when I call it on
BackendController
-MyntraScraper.py
Here is my code:
import requests, json
from bs4 import BeautifulSoup
import os, ssl
import certifi
import urllib3
class MyntraScraperClass:
def __init__(self,url):
self.url=url
def myntra(self):
mt={}
http = urllib3.PoolManager(
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where())
if (not os.environ.get('PYTHONHTTPSVERIFY', '') and getattr(ssl, '_create_unverified_context', None)):
ssl._create_default_https_context = ssl._create_unverified_context
proxy = {'http': '-------'}
headers = {'User-Agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'}
for x in range(0,5):
print(x)
try:
s = requests.Session()
res = s.get(self.url, headers=headers, verify=True, proxies=proxy)
print(res)
soup = BeautifulSoup(res.text, "lxml")
print(soup)
if "Access Denied" in soup.title:
continue
break
except requests.exceptions.ProxyError:
continue
except Exception as e:
print(e)
mt['error']=e
script = None
for s in soup.find_all("script"):
if 'pdpData' in s.text:
script = s.get_text(strip=True)
break
else:
continue
mt['data'] =json.loads(script[script.index('{'):])
return mt
I am calling MyntraScraperClass in my Django Views from BackendController directory.
def tst(request):
url = request.GET.get('url')
from BackendController.MyntraScraper import MyntraScraperClass
obj1 = MyntraScraperClass(url)
kk = obj1.myntra()
print(kk)
return JsonResponse(kk)
The response is
<Response [400]>
I am getting this in soup
<html><head>
<title>Invalid URL</title>
</head><body>
<h1>Invalid URL</h1>
The requested URL "[no URL]", is invalid.<p>
Reference #9.6b722c31.1596891375.7bfc4ae
</p></body></html>
You should try to solve your problem using https://scrapy.org/, here are the docs https://docs.scrapy.org/en/latest/
import requests, json
from bs4 import BeautifulSoup
headers = {'User-Agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'}
class MyntraScraperClass:
def __init__(self,url):
self.url=url
self.session = requests.Session()
def myntra(self):
res = self.session.get(self.url, headers=headers)
soup = BeautifulSoup(res.text,"lxml")
script = None
for s in soup.find_all("script"):
if 'pdpData' in s.text:
script = s.get_text(strip=True)
break
try:
return json.loads(script[script.index('{'):])
except:
return None
scraperClass = MyntraScraperClass("https://www.myntra.com/sports-shoes/puma/puma-men-blue-hybrid-fuego-running-shoes/11203218/buy")
data = scraperClass.myntra()
print(data)
Output:
{'pdpData': {'id': 11203218, 'name': 'Puma Men Blue Hybrid Fuego Running Shoes', 'mrp': 6499, 'manufacturer': 'SSIPL RETAIL LIMITED, KUNDLI,75, SERSA ROAD, 131028 SONEPAT', 'countryOfOrigin': 'India', 'colours': None, 'baseColour': 'Blue', 'brand': {'uidx': '', 'name': 'Puma', 'image': '', 'bio': ''}, 'media': {'videos': [], 'albums': [{'name': 'default', 'images': [{'src': 'http://assets.myntassets.com/h_($height),q_($qualityPercentage),w_($width)/v1/assets/images/productimage/2019/12/20/0c15e03c-863b-4a4a-9bb7-709a733fd4821576816965952-1.jpg', 'secureSrc': 'https://assets.myntassets.com/h_($height),q_($qualityPercentage),w_($width)/v1/assets/images/productimage/2019/12/20/0c15e03c-863b-4a4a-9bb7-709a733fd4821576816965952-1.jpg', 'host': None, 'imageURL': 'http://assets.myntassets.com/assets/images/productimage/2019/12/20/0c15e03c-863b-4a4a-9bb7-709a733fd4821576816965952-1.jpg', 'annotation': []}, {'src': 'http://assets.myntassets.com/h_($height),q_($qualityPercentage),w_($width)/v1/assets/images/productimage/2019/12/20/69bfa4e0-1ac4-4adf-b84e-4815ff60e8831576816966007-2.jpg', 'secureSrc': 'https://assets.myntassets.com/h_($height),q_($qualityPercentage),w_($width)/v1/assets/images/productimage/2019/12/20/69bfa4e0-1ac4-4adf-b84e-4815ff60e8831576816966007-2.jpg', 'host': None, 'imageURL': 'http://assets.myntassets.com/assets/images/productimage/2019/12/20/69bfa4e0-1ac4-4adf-b84e-4815ff60e8831576816966007-2.jpg', 'annotation': []}, {'src': 'http://assets.myntassets.com/h_($height),q_($qualityPercentage),w_($width)/v1/assets/images/productimage/2019/12/20/d2fd0ca0-1643-43ae-a0fc-fb1309580e151576816966049-3.jpg', 'secureSrc': 'https://assets.myntassets.com/h_($height),q_($qualityPercentage),w_($width)/v1/assets/images/productimage/2019/12/20/d2fd0ca0-1643-43ae-a0fc-fb1309580e151576816966049-3.jpg', 'host': None, 'imageURL': 'http://assets.myntassets.com/assets/images/productimage/2019/12/20/d2fd0ca0-1643-43ae-a0fc-fb1309580e151576816966049-3.jpg', 'annotation': []}, {'src': 'http://assets.myntassets.com/h_($height),q_($qualityPercentage),w_($width)/v1/assets/images/productimage/2019/12/20/0edae428-b9c0-4755-9127-0961d872b78a1576816966095-4.jpg', 'secureSrc': 'https://assets.myntassets.com/h_($height),q_($qualityPercentage),w_($width)/v1/assets/images/productimage/2019/12/20/0edae428-b9c0-4755-9127-0961d872b78a1576816966095-4.jpg', 'host': None, 'imageURL': 'http://assets.myntassets.com/assets/images/productimage/2019/12/20/0edae428-b9c0-4755-9127-0961d872b78a1576816966095-4.jpg', 'annotation': []}, {'src': 'http://assets.myntassets.com/h_($height),q_($qualityPercentage),w_($width)/v1/assets/images/productimage/2019/12/20/c59c7677-2bbd-4dbe-9b02-7c321c29cb701576816966142-5.jpg', 'secureSrc': 'https://assets.myntassets.com/h_($height),q_($qualityPercentage),w_($width)/v1/assets/images/productimage/2019/12/20/c59c7677-2bbd-4dbe-9b02-7c321c29cb701576816966142-5.jpg', 'host': None, 'imageURL': 'http://assets.myntassets.com/assets/images/productimage/2019/12/20/c59c7677-2bbd-4dbe-9b02-7c321c29cb701576816966142-5.jpg', 'annotation': []}]}, {'name': 'animatedImage', 'images': []}]}, 'sbpEnabled': False, 'sizechart': {'sizeChartUrl': None, 'sizeRepresentationUrl': 'http://assets.myntassets.com/assets/images/sizechart/2016/12/12/11481538267795-footwear.png'}, 'sizeRecoLazy': {'actionType': 'lazy', 'action': '/product/11203218/size/recommendation', 'sizeProfileAction': '/user/size-profiles?gender=male&articleType=Sports%20Shoes'}, 'analytics': {'articleType': 'Sports Shoes', 'subCategory': 'Shoes', 'masterCategory': 'Footwear', 'gender': 'Men', 'brand': 'Puma', 'colourHexCode': None}, 'crossLinks': [{'title': 'More Sports Shoes by Puma', 'url': 'sports-shoes?f=Brand:Puma::Gender:men'}, {'title': 'More Blue Sports Shoes', 'url': 'sports-shoes?f=Color:Blue_0074D9::Gender:men'}, {'title': 'More Sports Shoes', 'url': 'sports-shoes?f=Gender:men'}], 'relatedStyles': None, 'disclaimerTitle': '', 'productDetails': [{'type': None, 'content': None, 'title': 'Product Details', 'description': "<b>FEATURES + BENEFITS</b><br>HYBRID: PUMA's combination of two of its best technologies: IGNITE foam and NRGY beads<br>IGNITE: PUMA's foam midsole and branded heel cage supports and stabilises by locking the heel onto the platform<br>NRGY: PUMA's foam midsole offers superior cushion from heel to toe so you can power through your run<br>Heel-to-toe drop: 12mm<br><br><b>Product Design Details</b><ul><li>A pair of blue & brown running sports shoes, has regular styling, lace-up detail</li><li>Low boot silhouette</li><li>Lightweight synthetic upper</li><li>Overlays to secure the heel</li><li>Classic tongue</li><li>Lace-up closure</li><li>Rubber outsole for traction and durability</li><li>PUMA Wordmark at the tongue</li><li>PUMA Cat Logo at heel</li><li>Warranty: 3 months</li><li>Warranty provided by brand/manufacturer</li></ul><br><b>PRODUCT STORY</b><br>Change the name of the game with the HYBRID Fuego running sneakers. This bold colour-blocked shoe pairs a HYBRID foam midsole and a grippy rubber outsole for the ultimate in comfort and stability while still maintaining a stylish edge."}, {'type': None, 'content': None, 'title': 'MATERIAL & CARE', 'description': 'Textile<br>Wipe with a clean, dry cloth to remove dust'}], 'preOrder': None, 'sizeChartDisclaimerText': '', 'tags': None, 'articleAttributes': {'Ankle Height': 'Regular', 'Arch Type': 'Medium', 'Cleats': 'No Cleats', 'Cushioning': 'Medium', 'Distance': 'Medium', 'Fastening': 'Lace-Ups', 'Material': 'Textile', 'Outsole Type': 'Marking', 'Pronation for Running Shoes': 'Neutral', 'Running Type': 'Road Running', 'Sole Material': 'Rubber', 'Sport': 'Running', 'Surface Type': 'Outdoor', 'Technology': 'NA', 'Warranty': '3 months'}, 'systemAttributes': [], 'ratings': None, 'urgency': [{'value': '0', 'type': 'PURCHASED', 'ptile': 0}, {'value': '0', 'type': 'CART', 'ptile': 0}, {'value': '0', 'type': 'WISHLIST', 'ptile': 0}, {'value': '0', 'type': 'PDP', 'ptile': 0}], 'catalogAttributes': {'catalogDate': '1576751286000', 'season': 'summer', 'year': '2020'}, 'productContentGroupEntries': [{'title': '', 'type': 'DETAILS', 'attributes': [{'attributeName': 'Product Details', 'attributeType': 'STRING', 'value': "<b>FEATURES + BENEFITS</b><br>HYBRID: PUMA's combination of two of its best technologies: IGNITE foam and NRGY beads<br>IGNITE: PUMA's foam midsole and branded heel cage supports and stabilises by locking the heel onto the platform<br>NRGY: PUMA's foam midsole offers superior cushion from heel to toe so you can power through your run<br>Heel-to-toe drop: 12mm<br><br><b>Product Design Details</b><ul><li>A pair of blue & brown running sports shoes, has regular styling, lace-up detail</li><li>Low boot silhouette</li><li>Lightweight synthetic upper</li><li>Overlays to secure the heel</li><li>Classic tongue</li><li>Lace-up closure</li><li>Rubber outsole for traction and durability</li><li>PUMA Wordmark at the tongue</li><li>PUMA Cat Logo at heel</li><li>Warranty: 3 months</li><li>Warranty provided by brand/manufacturer</li></ul><br><b>PRODUCT STORY</b><br>Change the name of the game with the HYBRID Fuego running sneakers. This bold colour-blocked shoe pairs a HYBRID foam midsole and a grippy rubber outsole for the ultimate in comfort and stability while still maintaining a stylish edge."}, {'attributeName': 'Material & Care', 'attributeType': 'STRING', 'value': 'Textile<br>Wipe with a clean, dry cloth to remove dust'}, {'attributeName': 'Style Note', 'attributeType': 'STRING', 'value': "You'll look and feel super stylish in these trendsetting sports shoes by Puma. Match this blue pair with track pants and a sleeveless sports T-shirt when heading out for a casual day with friends."}]}], 'shoppableLooks': None, 'descriptors': [{'title': 'description', 'description': "<b>FEATURES + BENEFITS</b><br>HYBRID: PUMA's combination of two of its best technologies: IGNITE foam and NRGY beads<br>IGNITE: PUMA's foam midsole and branded heel cage supports and stabilises by locking the heel onto the platform<br>NRGY: PUMA's foam midsole offers superior cushion from heel to toe so you can power through your run<br>Heel-to-toe drop: 12mm<br><br><b>Product Design Details</b><ul><li>A pair of blue & brown running sports shoes, has regular styling, lace-up detail</li><li>Low boot silhouette</li><li>Lightweight synthetic upper</li><li>Overlays to secure the heel</li><li>Classic tongue</li><li>Lace-up closure</li><li>Rubber outsole for traction and durability</li><li>PUMA Wordmark at the tongue</li><li>PUMA Cat Logo at heel</li><li>Warranty: 3 months</li><li>Warranty provided by brand/manufacturer</li></ul><br><b>PRODUCT STORY</b><br>Change the name of the game with the HYBRID Fuego running sneakers. This bold colour-blocked shoe pairs a HYBRID foam midsole and a grippy rubber outsole for the ultimate in comfort and stability while still maintaining a stylish edge."}, {'title': 'style_note', 'description': "You'll look and feel super stylish in these trendsetting sports shoes by Puma. Match this blue pair with track pants and a sleeveless sports T-shirt when heading out for a casual day with friends."}, {'title': 'materials_care_desc', 'description': 'Textile<br>Wipe with a clean, dry cloth to remove dust'}], 'flags': {'isExchangeable': True, 'isReturnable': True, 'openBoxPickupEnabled': True, 'tryAndBuyEnabled': True, 'isLarge': False, 'isHazmat': False, 'isFragile': False, 'isJewellery': False, 'outOfStock': False, 'codEnabled': True, 'globalStore': False, 'loyaltyPointsEnabled': False, 'emiEnabled': True, 'chatEnabled': False, 'measurementModeEnabled': False, 'sampleModeEnabled': False, 'disableBuyButton': False}, 'earlyBirdOffer': None, 'serviceability': {'launchDate': '', 'returnPeriod': 30, 'descriptors': ['Pay on delivery might be available', 'Easy 30 days returns and exchanges', 'Try & Buy might be available'], 'procurementTimeInDays': {'6206': 4}}, 'buyButtonSellerOrder': [{'skuId': 38724440, 'sellerPartnerId': 6206}, {'skuId': 38724442, 'sellerPartnerId': 6206}, {'skuId': 38724446, 'sellerPartnerId': 6206}, {'skuId': 38724450, 'sellerPartnerId': 6206}, {'skuId': 38724452, 'sellerPartnerId': 6206}, {'skuId': 38724444, 'sellerPartnerId': 6206}, {'skuId': 38724448, 'sellerPartnerId': 6206}], 'sellers': [{'sellerPartnerId': 6206, 'sellerName': 'Puma Sports India Pvt. Ltd.(NSCM)'}], 'sizes': [{'skuId': 38724440, 'styleId': 11203218, 'action': '/product/11203218/related/6?co=1', 'label': '6', 'available': True, 'sizeType': 'UK Size', 'originalStyle': True, 'measurements': [{'type': 'Body Measurement', 'name': 'To Fit Foot Length', 'value': '24.5', 'minValue': '24.5', 'maxValue': '24.5', 'unit': 'cm', 'displayText': '24.5cm'}], 'allSizesList': [{'scaleCode': 'uk_size', 'sizeValue': '6', 'size': 'UK Size', 'order': 1, 'prefix': 'UK'}, {'scaleCode': 'us_size', 'sizeValue': '7', 'size': 'US Size', 'order': 2, 'prefix': 'US'}, {'scaleCode': 'euro_size', 'sizeValue': '39', 'size': 'Euro Size', 'order': 3, 'prefix': 'EURO'}], 'sizeSellerData': [{'mrp': 6499, 'sellerPartnerId': 6206, 'availableCount': 10, 'sellableInventoryCount': 10, 'warehouses': ['106'], 'supplyType': 'ON_HAND', 'discountId': '11203218:23363948', 'discountedPrice': 2924}]}, {'skuId': 38724442, 'styleId': 11203218, 'action': '/product/11203218/related/7?co=1', 'label': '7', 'available': True, 'sizeType': 'UK Size', 'originalStyle': True, 'measurements': [{'type': 'Body Measurement', 'name': 'To Fit Foot Length', 'value': '25.4', 'minValue': '25.4', 'maxValue': '25.4', 'unit': 'cm', 'displayText': '25.4cm'}], 'allSizesList': [{'scaleCode': 'uk_size', 'sizeValue': '7', 'size': 'UK Size', 'order': 1, 'prefix': 'UK'}, {'scaleCode': 'us_size', 'sizeValue': '8', 'size': 'US Size', 'order': 2, 'prefix': 'US'}, {'scaleCode': 'euro_size', 'sizeValue': '40.5', 'size': 'Euro Size', 'order': 3, 'prefix': 'EURO'}], 'sizeSellerData': [{'mrp': 6499, 'sellerPartnerId': 6206, 'availableCount': 49, 'sellableInventoryCount': 49, 'warehouses': ['106'], 'supplyType': 'ON_HAND', 'discountId': '11203218:23363948', 'discountedPrice': 2924}]}, {'skuId': 38724444, 'styleId': 11203218, 'action': '/product/11203218/related/8?co=1', 'label': '8', 'available': True, 'sizeType': 'UK Size', 'originalStyle': True, 'measurements': [{'type': 'Body Measurement', 'name': 'To Fit Foot Length', 'value': '26.2', 'minValue': '26.2', 'maxValue': '26.2', 'unit': 'cm', 'displayText': '26.2cm'}], 'allSizesList': [{'scaleCode': 'uk_size', 'sizeValue': '8', 'size': 'UK Size', 'order': 1, 'prefix': 'UK'}, {'scaleCode': 'us_size', 'sizeValue': '9', 'size': 'US Size', 'order': 2, 'prefix': 'US'}, {'scaleCode': 'euro_size', 'sizeValue': '42', 'size': 'Euro Size', 'order': 3, 'prefix': 'EURO'}], 'sizeSellerData': [{'mrp': 6499, 'sellerPartnerId': 6206, 'availableCount': 121, 'sellableInventoryCount': 121, 'warehouses': ['106'], 'supplyType': 'ON_HAND', 'discountId': '11203218:23363948', 'discountedPrice': 2924}]}, {'skuId': 38724446, 'styleId': 11203218, 'action': '/product/11203218/related/9?co=1', 'label': '9', 'available': True, 'sizeType': 'UK Size', 'originalStyle': True, 'measurements': [{'type': 'Body Measurement', 'name': 'To Fit Foot Length', 'value': '27.1', 'minValue': '27.1', 'maxValue': '27.1', 'unit': 'cm', 'displayText': '27.1cm'}], 'allSizesList': [{'scaleCode': 'uk_size', 'sizeValue': '9', 'size': 'UK Size', 'order': 1, 'prefix': 'UK'}, {'scaleCode': 'us_size', 'sizeValue': '10', 'size': 'US Size', 'order': 2, 'prefix': 'US'}, {'scaleCode': 'euro_size', 'sizeValue': '43', 'size': 'Euro Size', 'order': 3, 'prefix': 'EURO'}], 'sizeSellerData': [{'mrp': 6499, 'sellerPartnerId': 6206, 'availableCount': 107, 'sellableInventoryCount': 107, 'warehouses': ['106'], 'supplyType': 'ON_HAND', 'discountId': '11203218:23363948', 'discountedPrice': 2924}]}, {'skuId': 38724448, 'styleId': 11203218, 'action': '/product/11203218/related/10?co=1', 'label': '10', 'available': True, 'sizeType': 'UK Size', 'originalStyle': True, 'measurements': [{'type': 'Body Measurement', 'name': 'To Fit Foot Length', 'value': '27.9', 'minValue': '27.9', 'maxValue': '27.9', 'unit': 'cm', 'displayText': '27.9cm'}], 'allSizesList': [{'scaleCode': 'uk_size', 'sizeValue': '10', 'size': 'UK Size', 'order': 1, 'prefix': 'UK'}, {'scaleCode': 'us_size', 'sizeValue': '11', 'size': 'US Size', 'order': 2, 'prefix': 'US'}, {'scaleCode': 'euro_size', 'sizeValue': '44.5', 'size': 'Euro Size', 'order': 3, 'prefix': 'EURO'}], 'sizeSellerData': [{'mrp': 6499, 'sellerPartnerId': 6206, 'availableCount': 112, 'sellableInventoryCount': 112, 'warehouses': ['106'], 'supplyType': 'ON_HAND', 'discountId': '11203218:23363948', 'discountedPrice': 2924}]}, {'skuId': 38724450, 'styleId': 11203218, 'action': '/product/11203218/related/11?co=1', 'label': '11', 'available': True, 'sizeType': 'UK Size', 'originalStyle': True, 'measurements': [{'type': 'Body Measurement', 'name': 'To Fit Foot Length', 'value': '28.8', 'minValue': '28.8', 'maxValue': '28.8', 'unit': 'cm', 'displayText': '28.8cm'}], 'allSizesList': [{'scaleCode': 'uk_size', 'sizeValue': '11', 'size': 'UK Size', 'order': 1, 'prefix': 'UK'}, {'scaleCode': 'us_size', 'sizeValue': '12', 'size': 'US Size', 'order': 2, 'prefix': 'US'}, {'scaleCode': 'euro_size', 'sizeValue': '46', 'size': 'Euro Size', 'order': 3, 'prefix': 'EURO'}], 'sizeSellerData': [{'mrp': 6499, 'sellerPartnerId': 6206, 'availableCount': 18, 'sellableInventoryCount': 18, 'warehouses': ['106'], 'supplyType': 'ON_HAND', 'discountId': '11203218:23363948', 'discountedPrice': 2924}]}, {'skuId': 38724452, 'styleId': 11203218, 'action': '/product/11203218/related/12?co=1', 'label': '12', 'available': False, 'sizeType': 'UK Size', 'originalStyle': True, 'measurements': [{'type': 'Body Measurement', 'name': 'To Fit Foot Length', 'value': '29.6', 'minValue': '29.6', 'maxValue': '29.6', 'unit': 'cm', 'displayText': '29.6cm'}], 'allSizesList': [{'scaleCode': 'uk_size', 'sizeValue': '12', 'size': 'UK Size', 'order': 1, 'prefix': 'UK'}, {'scaleCode': 'us_size', 'sizeValue': '13', 'size': 'US Size', 'order': 2, 'prefix': 'US'}, {'scaleCode': 'euro_size', 'sizeValue': '47', 'size': 'Euro Size', 'order': 3, 'prefix': 'EURO'}], 'sizeSellerData': []}], 'discounts': [{'type': 1, 'freeItem': False, 'label': '(55% OFF)', 'discountText': '', 'timerStart': '0', 'timerEnd': '1597084200', 'discountPercent': 55, 'offer': '', 'discountId': '11203218:23363948', 'heading': None, 'description': None, 'link': None, 'freeItemImage': None}], 'offers': [{'type': 'EMI', 'title': 'EMI option available', 'description': '', 'action': '/faqs', 'image': None}], 'bundledSkus': None, 'richPdp': None, 'landingPageUrl': 'sports-shoes/puma/puma-men-blue-hybrid-fuego-running-shoes/11203218/buy'}, 'pageName': 'Pdp', 'atsa': ['Sport', 'Material', 'Fastening', 'Ankle Height', 'Outsole Type', 'Cleats', 'Pronation for Running Shoes', 'Arch Type', 'Cushioning', 'Running Type', 'Warranty', 'Distance', 'Number of Components', 'Surface Type', 'Technology']}

How to create an element using values of already existed elements in a dict, inside a list?

Here is my example:
print (stock_info)
>>> [{'symbol': 'AAPL', 'name': 'Apple Inc.', 'price': 145.16, 'quantity': 20}, {'symbol': 'AMZN', 'name': 'Amazon.com, Inc.', 'price': 998.61, 'quantity': 20}, {'symbol': 'FB', 'name': 'Facebook, Inc.', 'price': 152.96, 'quantity': 30}, {'symbol': 'GOOG', 'name': 'Alphabet Inc.', 'price': 957.01, 'quantity': 20}]
I have 'price' and 'quantity' fields with values.
And now I would like to create a field called 'total' = price * quantity
How to create a new field ('total': value) based on values of 2 fields, which already exist ( value = price * quantity)?
As a result I want to see:
>>> [{'symbol': 'AAPL', 'name': 'Apple Inc.', 'price': 145.16, 'quantity': 20, 'total' : 2903.2}, {'symbol': 'AMZN', 'name': 'Amazon.com, Inc.', 'price': 998.61, 'quantity': 20, 'total' : 19972.2}, {'symbol': 'FB', 'name': 'Facebook, Inc.', 'price': 152.96, 'quantity': 30, 'total' : 4588.8}, {'symbol': 'GOOG', 'name': 'Alphabet Inc.', 'price': 957.01, 'quantity': 20, 'total' : 19140.2}]
So each dict (dict, yes?) was extended with a new field 'total' and its value.
How to implement this idea?
Any help is greatly appreciated ;)
Thanks!
Just loop through your data and add a new key to your dictionaries:
for stock_item in stock_info:
stock_item["total"] = stock_item["price"] * stock_item["quantity"]
EDIT - Testing with your data:
stock_info = [{'symbol': 'AAPL', 'name': 'Apple Inc.', 'price': 145.16, 'quantity': 20},
{'symbol': 'AMZN', 'name': 'Amazon.com, Inc.', 'price': 998.61, 'quantity':20},
{'symbol': 'FB', 'name': 'Facebook, Inc.', 'price': 152.96, 'quantity': 30},
{'symbol': 'GOOG', 'name': 'Alphabet Inc.', 'price': 957.01, 'quantity': 20}]
for stock_item in stock_info:
stock_item["total"] = stock_item["price"] * stock_item["quantity"]
print(stock_info)
yields:
[{'name': 'Apple Inc.', 'price': 145.16, 'symbol': 'AAPL', 'total': 2903.2, 'quantity': 20},
{'name': 'Amazon.com, Inc.', 'price': 998.61, 'symbol': 'AMZN', 'total': 19972.2, 'quantity': 20},
{'name': 'Facebook, Inc.', 'price': 152.96, 'symbol': 'FB', 'total': 4588.8, 'quantity': 30},
{'name': 'Alphabet Inc.', 'price': 957.01, 'symbol': 'GOOG', 'total': 19140.2, 'quantity': 20}]

Scrapy, showing description image alt

I tried to scrape instagram, for the example i tried to scrape nike instagram. But, ijust want get the discription of image. The description image inside tag . I tried to show it but doesn't work.
This is my code :
import scrapy
class Nike(scrapy.Spider):
name = 'nike'
urls = 'http://www.instagram.com/nike/'
start_urls = [urls]
allowed_domains = ['http://instagram.com']
def parse(self, response):
for N in response.css('div._jjzlb'):
yield{
'name':N.css('alt::text').extract()
}
As requested by OP in the comments, here's an example of parsing JavaScript data from the HTML source directly, using js2xml (disclaimer: I'm the author of js2xml):
$ scrapy version -v
Scrapy : 1.4.0
lxml : 3.7.3.0
libxml2 : 2.9.3
cssselect : 1.0.1
parsel : 1.2.0
w3lib : 1.17.0
Twisted : 17.1.0
Python : 2.7.12+ (default, Sep 17 2016, 12:08:02) - [GCC 6.2.0 20160914]
pyOpenSSL : 17.0.0 (OpenSSL 1.0.2g 1 Mar 2016)
Platform : Linux-4.8.0-53-generic-x86_64-with-Ubuntu-16.10-yakkety
$ scrapy shell http://www.instagram.com/nike/
2017-05-30 10:06:39 [scrapy.utils.log] INFO: Scrapy 1.4.0 started (bot: scrapybot)
(...)
2017-05-30 10:06:40 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.instagram.com/nike/> from <GET http://www.instagram.com/nike/>
2017-05-30 10:06:41 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.instagram.com/nike/> (referer: None)
>>> response.xpath('//script[contains(., "sharedData")]')
[<Selector xpath='//script[contains(., "sharedData")]' data=u'<script type="text/javascript">window._s'>]
>>> response.xpath('string(//script[contains(., "sharedData")])').get()
u'window._sharedData = {"activity_counts": null, "config": (...redacted...) "show_app_install": true};'
>>> import js2xml
>>> # get the JavaScript statements
>>> js = response.xpath('string(//script[contains(., "sharedData")])').get()
>>> # parse them with js2xml
>>> jstree = js2xml.parse(js)
>>> # locate the object we're interested in:
>>> # here it's the right-part of the following assignement:
>>> # |--- this bit
>>> # window._sharedData = {"activity_counts":...
>>>
>>> obj = jstree.xpath('//assign[left//identifier[#name="_sharedData"]]/right/*')[0]
>>> # convert this object to a Python dict
>>> sharedData = js2xml.jsonlike.make_dict(obj)
>>>
>>> from pprint import pprint
>>> # quite a lot of data...
>>> pprint(sharedData)
{'activity_counts': None,
'config': {'csrf_token': 'JzQ8ja9YlG1cKqwAbvwacTKhOc4FuBDT', 'viewer': None},
'country_code': 'FR',
'display_properties_server_guess': {'pixel_ratio': 1.5,
'viewport_width': 360},
'entry_data': {'ProfilePage': [{'logging_page_id': 'profilePage_13460080',
'user': {'biography': 'Just Do It.',
'blocked_by_viewer': False,
'connected_fb_page': None,
'country_block': False,
'external_url': 'http://nike.com/justdoit',
'external_url_linkshimmed': 'http://l.instagram.com/?u=http%3A%2F%2Fnike.com%2Fjustdoit&e=ATOPqaPHEBPK4ylZ6YZRynSTQdp28oGedfFWw3gLtqoII1ZhtlDhLI0vu9Udbjg',
'followed_by': {'count': 72447697},
'followed_by_viewer': False,
'follows': {'count': 136},
'follows_viewer': False,
'full_name': 'nike',
'has_blocked_viewer': False,
'has_requested_viewer': False,
'id': '13460080',
'is_private': False,
'is_verified': True,
'media': {'count': 897,
'nodes': [{'__typename': 'GraphVideo',
'caption': 'Eliud Kipchoge - 2:00:25\nThe barrier just got that much closer.\n#Breaking2 #JustDoIt',
'code': 'BTvxgGEAn-p',
'comments': {'count': 2274},
'comments_disabled': False,
'date': 1494064175,
'dimensions': {'height': 640,
'width': 640},
'display_src': 'https://scontent-cdg2-1.cdninstagram.com/t51.2885-15/s640x640/e15/18380674_1840678496198968_334757937257906176_n.jpg',
'id': '1508642110004428713',
'is_video': True,
'likes': {'count': 296763},
'owner': {'id': '13460080'},
'thumbnail_src': 'https://scontent-cdg2-1.cdninstagram.com/t51.2885-15/s640x640/e15/18380674_1840678496198968_334757937257906176_n.jpg',
'video_views': 1193309},
(...redacted...)
{'__typename': 'GraphVideo',
'caption': u'We step onto the court as equals. Judged only by our performance. New Orleans sets the stage for All Star Weekend #nikebasketball.\xa0#EQUALITY #nike',
'code': 'BQlZFdugTIG',
'comments': {'count': 1172},
'comments_disabled': False,
'date': 1487273379,
'dimensions': {'height': 360,
'width': 640},
'display_src': 'https://scontent-cdg2-1.cdninstagram.com/t51.2885-15/s640x640/e15/16789854_771546756335957_8222167172487577600_n.jpg',
'id': '1451676781575746054',
'is_video': True,
'likes': {'count': 184595},
'owner': {'id': '13460080'},
'thumbnail_src': 'https://scontent-cdg2-1.cdninstagram.com/t51.2885-15/e15/c157.0.406.406/16789854_771546756335957_8222167172487577600_n.jpg',
'video_views': 627647}],
'page_info': {'end_cursor': 'AQCseCeUI_vJ_XzpTTIf7AhmwqbkBSf959347Wl03MM_ErpUQqP6RGjuJRdEcH1qJBCWvPGxUhpm6rPv5esAW5GvXHGodwmYA0eT51wnk8_-KA',
'has_next_page': True}},
'profile_pic_url': 'https://scontent-cdg2-1.cdninstagram.com/t51.2885-19/s150x150/17126848_1779368432381854_3589478532054515712_a.jpg',
'profile_pic_url_hd': 'https://scontent-cdg2-1.cdninstagram.com/t51.2885-19/s320x320/17126848_1779368432381854_3589478532054515712_a.jpg',
'requested_by_viewer': False,
'username': 'nike'}}]},
'environment_switcher_visible_server_guess': True,
'gatekeepers': {'bn': True, 'ld': True, 'pl': True},
'hostname': 'www.instagram.com',
'language_code': 'en',
'platform': 'web',
'probably_has_app': False,
'qe': {'bc3l': {'g': '', 'p': {}},
'create': {'g': '', 'p': {}},
'deact': {'g': '', 'p': {}},
'disc': {'g': '', 'p': {}},
'ebd': {'g': 'holdout', 'p': {'use_new_styles': 'false'}},
'feed': {'g': '', 'p': {}},
'gql': {'g': '', 'p': {}},
'nav': {'g': '', 'p': {}},
'nav_lo': {'g': '', 'p': {}},
'pm': {'g': '', 'p': {}},
'poe': {'g': '', 'p': {}},
'profile': {'g': '', 'p': {}},
'sidecar': {'g': '', 'p': {}},
'su_universe': {'g': '', 'p': {}},
'ufi': {'g': '', 'p': {}},
'ufi_loggedout': {'g': '', 'p': {}},
'us': {'g': '', 'p': {}},
'us_li': {'g': '', 'p': {}},
'video': {'g': '', 'p': {}}},
'show_app_install': True}
>>> # nodes have a 'caption' key that looks like the "alt" attributes
>>> pprint(sharedData['entry_data']['ProfilePage'][0]['user']['media']['nodes'])
[{'__typename': 'GraphVideo',
'caption': 'Eliud Kipchoge - 2:00:25\nThe barrier just got that much closer.\n#Breaking2 #JustDoIt',
'code': 'BTvxgGEAn-p',
'comments': {'count': 2274},
'comments_disabled': False,
'date': 1494064175,
'dimensions': {'height': 640, 'width': 640},
'display_src': 'https://scontent-cdg2-1.cdninstagram.com/t51.2885-15/s640x640/e15/18380674_1840678496198968_334757937257906176_n.jpg',
'id': '1508642110004428713',
'is_video': True,
'likes': {'count': 296763},
'owner': {'id': '13460080'},
'thumbnail_src': 'https://scontent-cdg2-1.cdninstagram.com/t51.2885-15/s640x640/e15/18380674_1840678496198968_334757937257906176_n.jpg',
'video_views': 1193309},
{'__typename': 'GraphImage',
'caption': 'Eliud Kipchoge - 2:00:25\nThe barrier just got that much closer.\n#Breaking2 #JustDoIt',
'code': 'BTvj_-gg6B6',
'comments': {'count': 2269},
'comments_disabled': False,
'date': 1494057096,
'dimensions': {'height': 1080, 'width': 1080},
'display_src': 'https://scontent-cdg2-1.cdninstagram.com/t51.2885-15/e35/18299402_128309274385168_1749644873230712832_n.jpg',
'id': '1508582728264818810',
'is_video': False,
'likes': {'count': 484745},
'owner': {'id': '13460080'},
'thumbnail_src': 'https://scontent-cdg2-1.cdninstagram.com/t51.2885-15/s640x640/sh0.08/e35/18299402_128309274385168_1749644873230712832_n.jpg'},
(...redacted...)
{'__typename': 'GraphVideo',
'caption': u'We step onto the court as equals. Judged only by our performance. New Orleans sets the stage for All Star Weekend #nikebasketball.\xa0#EQUALITY #nike',
'code': 'BQlZFdugTIG',
'comments': {'count': 1172},
'comments_disabled': False,
'date': 1487273379,
'dimensions': {'height': 360, 'width': 640},
'display_src': 'https://scontent-cdg2-1.cdninstagram.com/t51.2885-15/s640x640/e15/16789854_771546756335957_8222167172487577600_n.jpg',
'id': '1451676781575746054',
'is_video': True,
'likes': {'count': 184595},
'owner': {'id': '13460080'},
'thumbnail_src': 'https://scontent-cdg2-1.cdninstagram.com/t51.2885-15/e15/c157.0.406.406/16789854_771546756335957_8222167172487577600_n.jpg',
'video_views': 627647}]
>>>

GAE Recursive Query

I have some code (below) that is producing valid results and overall I am happy with it. However, it's quite 'wordy' and I am interested to know if this good/bad approach - this there something more effective/simpler that I should be doing?
I am really pleased to have the code in the model and not in my api, so I would like to keep it that way.
class ndb_Project(ndb.Model):
name = ndb.StringProperty()
description = ndb.StringProperty()
version = ndb.StructuredProperty(ndb_Version)
parentProj = ndb.KeyProperty()
childProj = ndb.KeyProperty(repeated=True)
#classmethod
def getChildProjects(cls,key):
proj = key.get()
a = []
for c in proj.childProj:
a.append(proj.getChildProjects(c))
o = proj.to_dict()
o['urlSafe'] = proj.key
o['childProj'] = a
return o
Many thanks
Toby
An alternative?
#classmethod
def getChildProjects(cls, proj):
o = proj.to_dict()
o['urlSafe'] = proj.key
temp = []
a = ndb.get_multi(proj.childProj) if proj.childProj else []
for c in a:
temp.append(proj.getChildProjects(c))
o['childProj']
return o
Something like this should work using ndb.get_multi(). It will also be faster since it fetches multiple keys in a single rpc instead of serially like your current loop.
Edit I ran this in /_ah/stats/shell so I know it works as long as it returns the output you desire.
from google.appengine.ext import ndb
from pprint import pprint as pp
class ndb_Project(ndb.Model):
name = ndb.StringProperty()
description = ndb.StringProperty()
parentProj = ndb.KeyProperty()
childProj = ndb.KeyProperty(repeated=True)
def getChildProjects(self):
a = [ent.getChildProjects() for ent in ndb.get_multi(self.childProj)]
o = self.to_dict()
o['urlSafe'] = self.key
o['childProj'] = a
return o
p = ndb_Project(name='first', description='first entity')
p.put()
for x in xrange(5):
tmp = ndb_Project(name=str(x), description='desc %s' % x)
p.childProj.append(tmp.put())
for y in xrange(5):
tmp2 = ndb_Project(name='%s %s' % (x, y), description='desc %s %s' % (x, y))
tmp.childProj.append(tmp2.put())
tmp.put()
p.put()
pp(p.getChildProjects())
output
{'childProj': [{'childProj': [{'childProj': [],
'description': u'desc 0 0',
'name': u'0 0',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5251250308251648)},
{'childProj': [],
'description': u'desc 0 1',
'name': u'0 1',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5245618633048064)},
{'childProj': [],
'description': u'desc 0 2',
'name': u'0 2',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5793979555643392)},
{'childProj': [],
'description': u'desc 0 3',
'name': u'0 3',
'parentProj': None,
'urlSafe': Key('ndb_Project', 6371518539890688)},
{'childProj': [],
'description': u'desc 0 4',
'name': u'0 4',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5208529711398912)}],
'description': u'desc 0',
'name': u'0',
'parentProj': None,
'urlSafe': Key('ndb_Project', 6460121467060224)},
{'childProj': [{'childProj': [],
'description': u'desc 1 0',
'name': u'1 0',
'parentProj': None,
'urlSafe': Key('ndb_Project', 6334429618241536)},
{'childProj': [],
'description': u'desc 1 1',
'name': u'1 1',
'parentProj': None,
'urlSafe': Key('ndb_Project', 6344957925261312)},
{'childProj': [],
'description': u'desc 1 2',
'name': u'1 2',
'parentProj': None,
'urlSafe': Key('ndb_Project', 6448551898906624)},
{'childProj': [],
'description': u'desc 1 3',
'name': u'1 3',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5286432096649216)},
{'childProj': [],
'description': u'desc 1 4',
'name': u'1 4',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5808107145920512)}],
'description': u'desc 1',
'name': u'1',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5282008817205248)},
{'childProj': [{'childProj': [],
'description': u'desc 2 0',
'name': u'2 0',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5808889098403840)},
{'childProj': [],
'description': u'desc 2 1',
'name': u'2 1',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5035775120900096)},
{'childProj': [],
'description': u'desc 2 2',
'name': u'2 2',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5881052870475776)},
{'childProj': [],
'description': u'desc 2 3',
'name': u'2 3',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5850614068150272)},
{'childProj': [],
'description': u'desc 2 4',
'name': u'2 4',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5215350924771328)}],
'description': u'desc 2',
'name': u'2',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5811114428334080)},
{'childProj': [{'childProj': [],
'description': u'desc 3 0',
'name': u'3 0',
'parentProj': None,
'urlSafe': Key('ndb_Project', 6368218830602240)},
{'childProj': [],
'description': u'desc 3 1',
'name': u'3 1',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5287664114728960)},
{'childProj': [],
'description': u'desc 3 2',
'name': u'3 2',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5041177015353344)},
{'childProj': [],
'description': u'desc 3 3',
'name': u'3 3',
'parentProj': None,
'urlSafe': Key('ndb_Project', 6412332003491840)},
{'childProj': [],
'description': u'desc 3 4',
'name': u'3 4',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5231029602222080)}],
'description': u'desc 3',
'name': u'3',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5245939144982528)},
{'childProj': [{'childProj': [],
'description': u'desc 4 0',
'name': u'4 0',
'parentProj': None,
'urlSafe': Key('ndb_Project', 4964143656337408)},
{'childProj': [],
'description': u'desc 4 1',
'name': u'4 1',
'parentProj': None,
'urlSafe': Key('ndb_Project', 4927054734688256)},
{'childProj': [],
'description': u'desc 4 2',
'name': u'4 2',
'parentProj': None,
'urlSafe': Key('ndb_Project', 6348349238149120)},
{'childProj': [],
'description': u'desc 4 3',
'name': u'4 3',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5004957119938560)},
{'childProj': [],
'description': u'desc 4 4',
'name': u'4 4',
'parentProj': None,
'urlSafe': Key('ndb_Project', 4960843947048960)}],
'description': u'desc 4',
'name': u'4',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5250989925859328)}],
'description': u'first entity',
'name': u'first',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5208229332123648)}
original code so that the original comments to this answer make sense
class ndb_Project(ndb.Model):
name = ndb.StringProperty()
description = ndb.StringProperty()
version = ndb.StructuredProperty(ndb_Version)
parentProj = ndb.KeyProperty()
childProj = ndb.KeyProperty(repeated=True)
#classmethod
def getChildProjects(cls, key):
proj = key.get()
a = ndb.get_multi(proj.childProj) if proj.childProj else []
o = proj.to_dict()
o['urlSafe'] = proj.key
o['childProj'] = a
return o