I want to scrape Myntra using Django Framework but when I put my code in Django project I get 400 error. The same code runs without error when I run it on the same file even in Django. Means calling the MyntraScraperClass in MyntraScraper.py file.
Here is my project directory
BackendController
-MyntraScraper.py
myDjangoApp
-views.py
Inside views.py, there is a function where I am calling my MyntraScraperClass
def tst(request):
------------
The same code that is MyntraScraperClass runs error free when I call it on
BackendController
-MyntraScraper.py
Here is my code:
import requests, json
from bs4 import BeautifulSoup
import os, ssl
import certifi
import urllib3
class MyntraScraperClass:
def __init__(self,url):
self.url=url
def myntra(self):
mt={}
http = urllib3.PoolManager(
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where())
if (not os.environ.get('PYTHONHTTPSVERIFY', '') and getattr(ssl, '_create_unverified_context', None)):
ssl._create_default_https_context = ssl._create_unverified_context
proxy = {'http': '-------'}
headers = {'User-Agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'}
for x in range(0,5):
print(x)
try:
s = requests.Session()
res = s.get(self.url, headers=headers, verify=True, proxies=proxy)
print(res)
soup = BeautifulSoup(res.text, "lxml")
print(soup)
if "Access Denied" in soup.title:
continue
break
except requests.exceptions.ProxyError:
continue
except Exception as e:
print(e)
mt['error']=e
script = None
for s in soup.find_all("script"):
if 'pdpData' in s.text:
script = s.get_text(strip=True)
break
else:
continue
mt['data'] =json.loads(script[script.index('{'):])
return mt
I am calling MyntraScraperClass in my Django Views from BackendController directory.
def tst(request):
url = request.GET.get('url')
from BackendController.MyntraScraper import MyntraScraperClass
obj1 = MyntraScraperClass(url)
kk = obj1.myntra()
print(kk)
return JsonResponse(kk)
The response is
<Response [400]>
I am getting this in soup
<html><head>
<title>Invalid URL</title>
</head><body>
<h1>Invalid URL</h1>
The requested URL "[no URL]", is invalid.<p>
Reference #9.6b722c31.1596891375.7bfc4ae
</p></body></html>
You should try to solve your problem using https://scrapy.org/, here are the docs https://docs.scrapy.org/en/latest/
import requests, json
from bs4 import BeautifulSoup
headers = {'User-Agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'}
class MyntraScraperClass:
def __init__(self,url):
self.url=url
self.session = requests.Session()
def myntra(self):
res = self.session.get(self.url, headers=headers)
soup = BeautifulSoup(res.text,"lxml")
script = None
for s in soup.find_all("script"):
if 'pdpData' in s.text:
script = s.get_text(strip=True)
break
try:
return json.loads(script[script.index('{'):])
except:
return None
scraperClass = MyntraScraperClass("https://www.myntra.com/sports-shoes/puma/puma-men-blue-hybrid-fuego-running-shoes/11203218/buy")
data = scraperClass.myntra()
print(data)
Output:
{'pdpData': {'id': 11203218, 'name': 'Puma Men Blue Hybrid Fuego Running Shoes', 'mrp': 6499, 'manufacturer': 'SSIPL RETAIL LIMITED, KUNDLI,75, SERSA ROAD, 131028 SONEPAT', 'countryOfOrigin': 'India', 'colours': None, 'baseColour': 'Blue', 'brand': {'uidx': '', 'name': 'Puma', 'image': '', 'bio': ''}, 'media': {'videos': [], 'albums': [{'name': 'default', 'images': [{'src': 'http://assets.myntassets.com/h_($height),q_($qualityPercentage),w_($width)/v1/assets/images/productimage/2019/12/20/0c15e03c-863b-4a4a-9bb7-709a733fd4821576816965952-1.jpg', 'secureSrc': 'https://assets.myntassets.com/h_($height),q_($qualityPercentage),w_($width)/v1/assets/images/productimage/2019/12/20/0c15e03c-863b-4a4a-9bb7-709a733fd4821576816965952-1.jpg', 'host': None, 'imageURL': 'http://assets.myntassets.com/assets/images/productimage/2019/12/20/0c15e03c-863b-4a4a-9bb7-709a733fd4821576816965952-1.jpg', 'annotation': []}, {'src': 'http://assets.myntassets.com/h_($height),q_($qualityPercentage),w_($width)/v1/assets/images/productimage/2019/12/20/69bfa4e0-1ac4-4adf-b84e-4815ff60e8831576816966007-2.jpg', 'secureSrc': 'https://assets.myntassets.com/h_($height),q_($qualityPercentage),w_($width)/v1/assets/images/productimage/2019/12/20/69bfa4e0-1ac4-4adf-b84e-4815ff60e8831576816966007-2.jpg', 'host': None, 'imageURL': 'http://assets.myntassets.com/assets/images/productimage/2019/12/20/69bfa4e0-1ac4-4adf-b84e-4815ff60e8831576816966007-2.jpg', 'annotation': []}, {'src': 'http://assets.myntassets.com/h_($height),q_($qualityPercentage),w_($width)/v1/assets/images/productimage/2019/12/20/d2fd0ca0-1643-43ae-a0fc-fb1309580e151576816966049-3.jpg', 'secureSrc': 'https://assets.myntassets.com/h_($height),q_($qualityPercentage),w_($width)/v1/assets/images/productimage/2019/12/20/d2fd0ca0-1643-43ae-a0fc-fb1309580e151576816966049-3.jpg', 'host': None, 'imageURL': 'http://assets.myntassets.com/assets/images/productimage/2019/12/20/d2fd0ca0-1643-43ae-a0fc-fb1309580e151576816966049-3.jpg', 'annotation': []}, {'src': 'http://assets.myntassets.com/h_($height),q_($qualityPercentage),w_($width)/v1/assets/images/productimage/2019/12/20/0edae428-b9c0-4755-9127-0961d872b78a1576816966095-4.jpg', 'secureSrc': 'https://assets.myntassets.com/h_($height),q_($qualityPercentage),w_($width)/v1/assets/images/productimage/2019/12/20/0edae428-b9c0-4755-9127-0961d872b78a1576816966095-4.jpg', 'host': None, 'imageURL': 'http://assets.myntassets.com/assets/images/productimage/2019/12/20/0edae428-b9c0-4755-9127-0961d872b78a1576816966095-4.jpg', 'annotation': []}, {'src': 'http://assets.myntassets.com/h_($height),q_($qualityPercentage),w_($width)/v1/assets/images/productimage/2019/12/20/c59c7677-2bbd-4dbe-9b02-7c321c29cb701576816966142-5.jpg', 'secureSrc': 'https://assets.myntassets.com/h_($height),q_($qualityPercentage),w_($width)/v1/assets/images/productimage/2019/12/20/c59c7677-2bbd-4dbe-9b02-7c321c29cb701576816966142-5.jpg', 'host': None, 'imageURL': 'http://assets.myntassets.com/assets/images/productimage/2019/12/20/c59c7677-2bbd-4dbe-9b02-7c321c29cb701576816966142-5.jpg', 'annotation': []}]}, {'name': 'animatedImage', 'images': []}]}, 'sbpEnabled': False, 'sizechart': {'sizeChartUrl': None, 'sizeRepresentationUrl': 'http://assets.myntassets.com/assets/images/sizechart/2016/12/12/11481538267795-footwear.png'}, 'sizeRecoLazy': {'actionType': 'lazy', 'action': '/product/11203218/size/recommendation', 'sizeProfileAction': '/user/size-profiles?gender=male&articleType=Sports%20Shoes'}, 'analytics': {'articleType': 'Sports Shoes', 'subCategory': 'Shoes', 'masterCategory': 'Footwear', 'gender': 'Men', 'brand': 'Puma', 'colourHexCode': None}, 'crossLinks': [{'title': 'More Sports Shoes by Puma', 'url': 'sports-shoes?f=Brand:Puma::Gender:men'}, {'title': 'More Blue Sports Shoes', 'url': 'sports-shoes?f=Color:Blue_0074D9::Gender:men'}, {'title': 'More Sports Shoes', 'url': 'sports-shoes?f=Gender:men'}], 'relatedStyles': None, 'disclaimerTitle': '', 'productDetails': [{'type': None, 'content': None, 'title': 'Product Details', 'description': "<b>FEATURES + BENEFITS</b><br>HYBRID: PUMA's combination of two of its best technologies: IGNITE foam and NRGY beads<br>IGNITE: PUMA's foam midsole and branded heel cage supports and stabilises by locking the heel onto the platform<br>NRGY: PUMA's foam midsole offers superior cushion from heel to toe so you can power through your run<br>Heel-to-toe drop: 12mm<br><br><b>Product Design Details</b><ul><li>A pair of blue & brown running sports shoes, has regular styling, lace-up detail</li><li>Low boot silhouette</li><li>Lightweight synthetic upper</li><li>Overlays to secure the heel</li><li>Classic tongue</li><li>Lace-up closure</li><li>Rubber outsole for traction and durability</li><li>PUMA Wordmark at the tongue</li><li>PUMA Cat Logo at heel</li><li>Warranty: 3 months</li><li>Warranty provided by brand/manufacturer</li></ul><br><b>PRODUCT STORY</b><br>Change the name of the game with the HYBRID Fuego running sneakers. This bold colour-blocked shoe pairs a HYBRID foam midsole and a grippy rubber outsole for the ultimate in comfort and stability while still maintaining a stylish edge."}, {'type': None, 'content': None, 'title': 'MATERIAL & CARE', 'description': 'Textile<br>Wipe with a clean, dry cloth to remove dust'}], 'preOrder': None, 'sizeChartDisclaimerText': '', 'tags': None, 'articleAttributes': {'Ankle Height': 'Regular', 'Arch Type': 'Medium', 'Cleats': 'No Cleats', 'Cushioning': 'Medium', 'Distance': 'Medium', 'Fastening': 'Lace-Ups', 'Material': 'Textile', 'Outsole Type': 'Marking', 'Pronation for Running Shoes': 'Neutral', 'Running Type': 'Road Running', 'Sole Material': 'Rubber', 'Sport': 'Running', 'Surface Type': 'Outdoor', 'Technology': 'NA', 'Warranty': '3 months'}, 'systemAttributes': [], 'ratings': None, 'urgency': [{'value': '0', 'type': 'PURCHASED', 'ptile': 0}, {'value': '0', 'type': 'CART', 'ptile': 0}, {'value': '0', 'type': 'WISHLIST', 'ptile': 0}, {'value': '0', 'type': 'PDP', 'ptile': 0}], 'catalogAttributes': {'catalogDate': '1576751286000', 'season': 'summer', 'year': '2020'}, 'productContentGroupEntries': [{'title': '', 'type': 'DETAILS', 'attributes': [{'attributeName': 'Product Details', 'attributeType': 'STRING', 'value': "<b>FEATURES + BENEFITS</b><br>HYBRID: PUMA's combination of two of its best technologies: IGNITE foam and NRGY beads<br>IGNITE: PUMA's foam midsole and branded heel cage supports and stabilises by locking the heel onto the platform<br>NRGY: PUMA's foam midsole offers superior cushion from heel to toe so you can power through your run<br>Heel-to-toe drop: 12mm<br><br><b>Product Design Details</b><ul><li>A pair of blue & brown running sports shoes, has regular styling, lace-up detail</li><li>Low boot silhouette</li><li>Lightweight synthetic upper</li><li>Overlays to secure the heel</li><li>Classic tongue</li><li>Lace-up closure</li><li>Rubber outsole for traction and durability</li><li>PUMA Wordmark at the tongue</li><li>PUMA Cat Logo at heel</li><li>Warranty: 3 months</li><li>Warranty provided by brand/manufacturer</li></ul><br><b>PRODUCT STORY</b><br>Change the name of the game with the HYBRID Fuego running sneakers. This bold colour-blocked shoe pairs a HYBRID foam midsole and a grippy rubber outsole for the ultimate in comfort and stability while still maintaining a stylish edge."}, {'attributeName': 'Material & Care', 'attributeType': 'STRING', 'value': 'Textile<br>Wipe with a clean, dry cloth to remove dust'}, {'attributeName': 'Style Note', 'attributeType': 'STRING', 'value': "You'll look and feel super stylish in these trendsetting sports shoes by Puma. Match this blue pair with track pants and a sleeveless sports T-shirt when heading out for a casual day with friends."}]}], 'shoppableLooks': None, 'descriptors': [{'title': 'description', 'description': "<b>FEATURES + BENEFITS</b><br>HYBRID: PUMA's combination of two of its best technologies: IGNITE foam and NRGY beads<br>IGNITE: PUMA's foam midsole and branded heel cage supports and stabilises by locking the heel onto the platform<br>NRGY: PUMA's foam midsole offers superior cushion from heel to toe so you can power through your run<br>Heel-to-toe drop: 12mm<br><br><b>Product Design Details</b><ul><li>A pair of blue & brown running sports shoes, has regular styling, lace-up detail</li><li>Low boot silhouette</li><li>Lightweight synthetic upper</li><li>Overlays to secure the heel</li><li>Classic tongue</li><li>Lace-up closure</li><li>Rubber outsole for traction and durability</li><li>PUMA Wordmark at the tongue</li><li>PUMA Cat Logo at heel</li><li>Warranty: 3 months</li><li>Warranty provided by brand/manufacturer</li></ul><br><b>PRODUCT STORY</b><br>Change the name of the game with the HYBRID Fuego running sneakers. This bold colour-blocked shoe pairs a HYBRID foam midsole and a grippy rubber outsole for the ultimate in comfort and stability while still maintaining a stylish edge."}, {'title': 'style_note', 'description': "You'll look and feel super stylish in these trendsetting sports shoes by Puma. Match this blue pair with track pants and a sleeveless sports T-shirt when heading out for a casual day with friends."}, {'title': 'materials_care_desc', 'description': 'Textile<br>Wipe with a clean, dry cloth to remove dust'}], 'flags': {'isExchangeable': True, 'isReturnable': True, 'openBoxPickupEnabled': True, 'tryAndBuyEnabled': True, 'isLarge': False, 'isHazmat': False, 'isFragile': False, 'isJewellery': False, 'outOfStock': False, 'codEnabled': True, 'globalStore': False, 'loyaltyPointsEnabled': False, 'emiEnabled': True, 'chatEnabled': False, 'measurementModeEnabled': False, 'sampleModeEnabled': False, 'disableBuyButton': False}, 'earlyBirdOffer': None, 'serviceability': {'launchDate': '', 'returnPeriod': 30, 'descriptors': ['Pay on delivery might be available', 'Easy 30 days returns and exchanges', 'Try & Buy might be available'], 'procurementTimeInDays': {'6206': 4}}, 'buyButtonSellerOrder': [{'skuId': 38724440, 'sellerPartnerId': 6206}, {'skuId': 38724442, 'sellerPartnerId': 6206}, {'skuId': 38724446, 'sellerPartnerId': 6206}, {'skuId': 38724450, 'sellerPartnerId': 6206}, {'skuId': 38724452, 'sellerPartnerId': 6206}, {'skuId': 38724444, 'sellerPartnerId': 6206}, {'skuId': 38724448, 'sellerPartnerId': 6206}], 'sellers': [{'sellerPartnerId': 6206, 'sellerName': 'Puma Sports India Pvt. Ltd.(NSCM)'}], 'sizes': [{'skuId': 38724440, 'styleId': 11203218, 'action': '/product/11203218/related/6?co=1', 'label': '6', 'available': True, 'sizeType': 'UK Size', 'originalStyle': True, 'measurements': [{'type': 'Body Measurement', 'name': 'To Fit Foot Length', 'value': '24.5', 'minValue': '24.5', 'maxValue': '24.5', 'unit': 'cm', 'displayText': '24.5cm'}], 'allSizesList': [{'scaleCode': 'uk_size', 'sizeValue': '6', 'size': 'UK Size', 'order': 1, 'prefix': 'UK'}, {'scaleCode': 'us_size', 'sizeValue': '7', 'size': 'US Size', 'order': 2, 'prefix': 'US'}, {'scaleCode': 'euro_size', 'sizeValue': '39', 'size': 'Euro Size', 'order': 3, 'prefix': 'EURO'}], 'sizeSellerData': [{'mrp': 6499, 'sellerPartnerId': 6206, 'availableCount': 10, 'sellableInventoryCount': 10, 'warehouses': ['106'], 'supplyType': 'ON_HAND', 'discountId': '11203218:23363948', 'discountedPrice': 2924}]}, {'skuId': 38724442, 'styleId': 11203218, 'action': '/product/11203218/related/7?co=1', 'label': '7', 'available': True, 'sizeType': 'UK Size', 'originalStyle': True, 'measurements': [{'type': 'Body Measurement', 'name': 'To Fit Foot Length', 'value': '25.4', 'minValue': '25.4', 'maxValue': '25.4', 'unit': 'cm', 'displayText': '25.4cm'}], 'allSizesList': [{'scaleCode': 'uk_size', 'sizeValue': '7', 'size': 'UK Size', 'order': 1, 'prefix': 'UK'}, {'scaleCode': 'us_size', 'sizeValue': '8', 'size': 'US Size', 'order': 2, 'prefix': 'US'}, {'scaleCode': 'euro_size', 'sizeValue': '40.5', 'size': 'Euro Size', 'order': 3, 'prefix': 'EURO'}], 'sizeSellerData': [{'mrp': 6499, 'sellerPartnerId': 6206, 'availableCount': 49, 'sellableInventoryCount': 49, 'warehouses': ['106'], 'supplyType': 'ON_HAND', 'discountId': '11203218:23363948', 'discountedPrice': 2924}]}, {'skuId': 38724444, 'styleId': 11203218, 'action': '/product/11203218/related/8?co=1', 'label': '8', 'available': True, 'sizeType': 'UK Size', 'originalStyle': True, 'measurements': [{'type': 'Body Measurement', 'name': 'To Fit Foot Length', 'value': '26.2', 'minValue': '26.2', 'maxValue': '26.2', 'unit': 'cm', 'displayText': '26.2cm'}], 'allSizesList': [{'scaleCode': 'uk_size', 'sizeValue': '8', 'size': 'UK Size', 'order': 1, 'prefix': 'UK'}, {'scaleCode': 'us_size', 'sizeValue': '9', 'size': 'US Size', 'order': 2, 'prefix': 'US'}, {'scaleCode': 'euro_size', 'sizeValue': '42', 'size': 'Euro Size', 'order': 3, 'prefix': 'EURO'}], 'sizeSellerData': [{'mrp': 6499, 'sellerPartnerId': 6206, 'availableCount': 121, 'sellableInventoryCount': 121, 'warehouses': ['106'], 'supplyType': 'ON_HAND', 'discountId': '11203218:23363948', 'discountedPrice': 2924}]}, {'skuId': 38724446, 'styleId': 11203218, 'action': '/product/11203218/related/9?co=1', 'label': '9', 'available': True, 'sizeType': 'UK Size', 'originalStyle': True, 'measurements': [{'type': 'Body Measurement', 'name': 'To Fit Foot Length', 'value': '27.1', 'minValue': '27.1', 'maxValue': '27.1', 'unit': 'cm', 'displayText': '27.1cm'}], 'allSizesList': [{'scaleCode': 'uk_size', 'sizeValue': '9', 'size': 'UK Size', 'order': 1, 'prefix': 'UK'}, {'scaleCode': 'us_size', 'sizeValue': '10', 'size': 'US Size', 'order': 2, 'prefix': 'US'}, {'scaleCode': 'euro_size', 'sizeValue': '43', 'size': 'Euro Size', 'order': 3, 'prefix': 'EURO'}], 'sizeSellerData': [{'mrp': 6499, 'sellerPartnerId': 6206, 'availableCount': 107, 'sellableInventoryCount': 107, 'warehouses': ['106'], 'supplyType': 'ON_HAND', 'discountId': '11203218:23363948', 'discountedPrice': 2924}]}, {'skuId': 38724448, 'styleId': 11203218, 'action': '/product/11203218/related/10?co=1', 'label': '10', 'available': True, 'sizeType': 'UK Size', 'originalStyle': True, 'measurements': [{'type': 'Body Measurement', 'name': 'To Fit Foot Length', 'value': '27.9', 'minValue': '27.9', 'maxValue': '27.9', 'unit': 'cm', 'displayText': '27.9cm'}], 'allSizesList': [{'scaleCode': 'uk_size', 'sizeValue': '10', 'size': 'UK Size', 'order': 1, 'prefix': 'UK'}, {'scaleCode': 'us_size', 'sizeValue': '11', 'size': 'US Size', 'order': 2, 'prefix': 'US'}, {'scaleCode': 'euro_size', 'sizeValue': '44.5', 'size': 'Euro Size', 'order': 3, 'prefix': 'EURO'}], 'sizeSellerData': [{'mrp': 6499, 'sellerPartnerId': 6206, 'availableCount': 112, 'sellableInventoryCount': 112, 'warehouses': ['106'], 'supplyType': 'ON_HAND', 'discountId': '11203218:23363948', 'discountedPrice': 2924}]}, {'skuId': 38724450, 'styleId': 11203218, 'action': '/product/11203218/related/11?co=1', 'label': '11', 'available': True, 'sizeType': 'UK Size', 'originalStyle': True, 'measurements': [{'type': 'Body Measurement', 'name': 'To Fit Foot Length', 'value': '28.8', 'minValue': '28.8', 'maxValue': '28.8', 'unit': 'cm', 'displayText': '28.8cm'}], 'allSizesList': [{'scaleCode': 'uk_size', 'sizeValue': '11', 'size': 'UK Size', 'order': 1, 'prefix': 'UK'}, {'scaleCode': 'us_size', 'sizeValue': '12', 'size': 'US Size', 'order': 2, 'prefix': 'US'}, {'scaleCode': 'euro_size', 'sizeValue': '46', 'size': 'Euro Size', 'order': 3, 'prefix': 'EURO'}], 'sizeSellerData': [{'mrp': 6499, 'sellerPartnerId': 6206, 'availableCount': 18, 'sellableInventoryCount': 18, 'warehouses': ['106'], 'supplyType': 'ON_HAND', 'discountId': '11203218:23363948', 'discountedPrice': 2924}]}, {'skuId': 38724452, 'styleId': 11203218, 'action': '/product/11203218/related/12?co=1', 'label': '12', 'available': False, 'sizeType': 'UK Size', 'originalStyle': True, 'measurements': [{'type': 'Body Measurement', 'name': 'To Fit Foot Length', 'value': '29.6', 'minValue': '29.6', 'maxValue': '29.6', 'unit': 'cm', 'displayText': '29.6cm'}], 'allSizesList': [{'scaleCode': 'uk_size', 'sizeValue': '12', 'size': 'UK Size', 'order': 1, 'prefix': 'UK'}, {'scaleCode': 'us_size', 'sizeValue': '13', 'size': 'US Size', 'order': 2, 'prefix': 'US'}, {'scaleCode': 'euro_size', 'sizeValue': '47', 'size': 'Euro Size', 'order': 3, 'prefix': 'EURO'}], 'sizeSellerData': []}], 'discounts': [{'type': 1, 'freeItem': False, 'label': '(55% OFF)', 'discountText': '', 'timerStart': '0', 'timerEnd': '1597084200', 'discountPercent': 55, 'offer': '', 'discountId': '11203218:23363948', 'heading': None, 'description': None, 'link': None, 'freeItemImage': None}], 'offers': [{'type': 'EMI', 'title': 'EMI option available', 'description': '', 'action': '/faqs', 'image': None}], 'bundledSkus': None, 'richPdp': None, 'landingPageUrl': 'sports-shoes/puma/puma-men-blue-hybrid-fuego-running-shoes/11203218/buy'}, 'pageName': 'Pdp', 'atsa': ['Sport', 'Material', 'Fastening', 'Ankle Height', 'Outsole Type', 'Cleats', 'Pronation for Running Shoes', 'Arch Type', 'Cushioning', 'Running Type', 'Warranty', 'Distance', 'Number of Components', 'Surface Type', 'Technology']}
This is the current json response
[
[
{
"TO":"nathanoluwaseyi#gmail.com",
"FROM":"johndoe#gmail.com",
"SUBJECT":"This is the subject 1",
"MESSAGE":[
"First Message"
],
"NAME":"John Doe",
"DATE":"2019-08-18 19:48:10"
},
{
"TO":"nathanoluwaseyi#gmail.com",
"FROM":"johndoe#gmail.com",
"SUBJECT":"This is the subject 2",
"MESSAGE":[
"Second message"
],
"NAME":"John Doe",
"DATE":"2019-08-18 19:48:10"
}
]
]
But i want a situation whereby more than one of the response have the same FROM then, the MESSAGE should be together in this format or something like this. How do i go about this? Everything i have tried didn't work
the date issue can be left for now. the most important is getting the messages into one.
[
[
{
"TO":"nathanoluwaseyi#gmail.com",
"FROM":"johndoe#gmail.com",
"SUBJECT":"This is the subject 1",
"MESSAGE":[
"First Message", "Second Message"
],
"NAME":"John Doe",
"DATE":"2019-08-18 19:48:10"
}
]
]
import json
s = '[...]' # Json string
loaded = json.loads(s) # [[{one json thing}], [{second json thing}]]
loaded = [t[0] for t in loaded] # [{json}, {json}] :)
dictionary = {}
for item in loaded:
FROM = item["FROM"] # From
MESSAGE = item["MESSAGE"][0] # Message
DATE = item["DATE"] # Date
if FROM not in dictionary: # If it hasn't been seen before
dictionary[FROM] = item.copy() # To preserve TO, SUBJECT, ...
del dictionary[FROM]["MESSAGE"] # No more >:)
del dictionary[FROM]["DATE"] # No more >:)
dictionary[FROM]["MESSAGE-DATE"] = [] # This will store the message-date pairs as a tuple
dictionary[FROM]["MESSAGE-DATE"].append((MESSAGE, DATE)) # Here, the message and date are combined into a single item as a tuple
for key in dictionary:
print (dictionary[key])
I've included comments that should explain. Try and see if it fits your purpose :)
(Btw I haven't dealt with json before, just know that it's basically a dictionary haha)
You can create a dict with the unfied messages, checking if 'FROM' is already present. If yes, append the message, otherwise, just add it:
def unify_messages(messages):
unified_msgs = {}
for message in messages:
if message['FROM'] in unified_msgs:
unified_msgs[message['FROM']]['MESSAGE'].extend(message['MESSAGE'])
else:
unified_msgs[message['FROM']] = message
return [v for v in unified_msgs.values()]
for message in messages:
unified_messages = unify_messages(messages)
If you try with:
messages = [
[
{
"TO":"nathanoluwaseyi#gmail.com",
"FROM":"johndoe#gmail.com",
"SUBJECT":"This is the subject 1",
"MESSAGE":[
"First Message"
],
"NAME":"John Doe",
"DATE":"2019-08-18 19:48:10"
},
{
"TO":"nathanoluwaseyi#gmail.com",
"FROM":"johndoe#gmail.com",
"SUBJECT":"This is the subject 2",
"MESSAGE":[
"Second message"
],
"NAME":"John Doe",
"DATE":"2019-08-18 19:48:10"
},
{
"TO":"nathanoluwaseyi#gmail.com",
"FROM":"alice#gmail.com",
"SUBJECT":"This is the subject 1",
"MESSAGE":[
"First Message from alice"
],
"NAME":"John Doe",
"DATE":"2019-08-18 19:48:10"
},
{
"TO":"nathanoluwaseyi#gmail.com",
"FROM":"sam#gmail.com",
"SUBJECT":"This is the subject 1",
"MESSAGE":[
"First Message from sam"
],
"NAME":"John Doe",
"DATE":"2019-08-18 19:48:10"
},
]
]
You`ll get:
[{'DATE': '2019-08-18 19:48:10',
'FROM': 'johndoe#gmail.com',
'MESSAGE': ['First Message', 'Second message'],
'NAME': 'John Doe',
'SUBJECT': 'This is the subject 1',
'TO': 'nathanoluwaseyi#gmail.com'},
{'DATE': '2019-08-18 19:48:10',
'FROM': 'alice#gmail.com',
'MESSAGE': ['First Message from alice'],
'NAME': 'John Doe',
'SUBJECT': 'This is the subject 1',
'TO': 'nathanoluwaseyi#gmail.com'},
{'DATE': '2019-08-18 19:48:10',
'FROM': 'sam#gmail.com',
'MESSAGE': ['First Message from sam'],
'NAME': 'John Doe',
'SUBJECT': 'This is the subject 1',
'TO': 'nathanoluwaseyi#gmail.com'}]
I have some code (below) that is producing valid results and overall I am happy with it. However, it's quite 'wordy' and I am interested to know if this good/bad approach - this there something more effective/simpler that I should be doing?
I am really pleased to have the code in the model and not in my api, so I would like to keep it that way.
class ndb_Project(ndb.Model):
name = ndb.StringProperty()
description = ndb.StringProperty()
version = ndb.StructuredProperty(ndb_Version)
parentProj = ndb.KeyProperty()
childProj = ndb.KeyProperty(repeated=True)
#classmethod
def getChildProjects(cls,key):
proj = key.get()
a = []
for c in proj.childProj:
a.append(proj.getChildProjects(c))
o = proj.to_dict()
o['urlSafe'] = proj.key
o['childProj'] = a
return o
Many thanks
Toby
An alternative?
#classmethod
def getChildProjects(cls, proj):
o = proj.to_dict()
o['urlSafe'] = proj.key
temp = []
a = ndb.get_multi(proj.childProj) if proj.childProj else []
for c in a:
temp.append(proj.getChildProjects(c))
o['childProj']
return o
Something like this should work using ndb.get_multi(). It will also be faster since it fetches multiple keys in a single rpc instead of serially like your current loop.
Edit I ran this in /_ah/stats/shell so I know it works as long as it returns the output you desire.
from google.appengine.ext import ndb
from pprint import pprint as pp
class ndb_Project(ndb.Model):
name = ndb.StringProperty()
description = ndb.StringProperty()
parentProj = ndb.KeyProperty()
childProj = ndb.KeyProperty(repeated=True)
def getChildProjects(self):
a = [ent.getChildProjects() for ent in ndb.get_multi(self.childProj)]
o = self.to_dict()
o['urlSafe'] = self.key
o['childProj'] = a
return o
p = ndb_Project(name='first', description='first entity')
p.put()
for x in xrange(5):
tmp = ndb_Project(name=str(x), description='desc %s' % x)
p.childProj.append(tmp.put())
for y in xrange(5):
tmp2 = ndb_Project(name='%s %s' % (x, y), description='desc %s %s' % (x, y))
tmp.childProj.append(tmp2.put())
tmp.put()
p.put()
pp(p.getChildProjects())
output
{'childProj': [{'childProj': [{'childProj': [],
'description': u'desc 0 0',
'name': u'0 0',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5251250308251648)},
{'childProj': [],
'description': u'desc 0 1',
'name': u'0 1',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5245618633048064)},
{'childProj': [],
'description': u'desc 0 2',
'name': u'0 2',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5793979555643392)},
{'childProj': [],
'description': u'desc 0 3',
'name': u'0 3',
'parentProj': None,
'urlSafe': Key('ndb_Project', 6371518539890688)},
{'childProj': [],
'description': u'desc 0 4',
'name': u'0 4',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5208529711398912)}],
'description': u'desc 0',
'name': u'0',
'parentProj': None,
'urlSafe': Key('ndb_Project', 6460121467060224)},
{'childProj': [{'childProj': [],
'description': u'desc 1 0',
'name': u'1 0',
'parentProj': None,
'urlSafe': Key('ndb_Project', 6334429618241536)},
{'childProj': [],
'description': u'desc 1 1',
'name': u'1 1',
'parentProj': None,
'urlSafe': Key('ndb_Project', 6344957925261312)},
{'childProj': [],
'description': u'desc 1 2',
'name': u'1 2',
'parentProj': None,
'urlSafe': Key('ndb_Project', 6448551898906624)},
{'childProj': [],
'description': u'desc 1 3',
'name': u'1 3',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5286432096649216)},
{'childProj': [],
'description': u'desc 1 4',
'name': u'1 4',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5808107145920512)}],
'description': u'desc 1',
'name': u'1',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5282008817205248)},
{'childProj': [{'childProj': [],
'description': u'desc 2 0',
'name': u'2 0',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5808889098403840)},
{'childProj': [],
'description': u'desc 2 1',
'name': u'2 1',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5035775120900096)},
{'childProj': [],
'description': u'desc 2 2',
'name': u'2 2',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5881052870475776)},
{'childProj': [],
'description': u'desc 2 3',
'name': u'2 3',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5850614068150272)},
{'childProj': [],
'description': u'desc 2 4',
'name': u'2 4',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5215350924771328)}],
'description': u'desc 2',
'name': u'2',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5811114428334080)},
{'childProj': [{'childProj': [],
'description': u'desc 3 0',
'name': u'3 0',
'parentProj': None,
'urlSafe': Key('ndb_Project', 6368218830602240)},
{'childProj': [],
'description': u'desc 3 1',
'name': u'3 1',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5287664114728960)},
{'childProj': [],
'description': u'desc 3 2',
'name': u'3 2',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5041177015353344)},
{'childProj': [],
'description': u'desc 3 3',
'name': u'3 3',
'parentProj': None,
'urlSafe': Key('ndb_Project', 6412332003491840)},
{'childProj': [],
'description': u'desc 3 4',
'name': u'3 4',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5231029602222080)}],
'description': u'desc 3',
'name': u'3',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5245939144982528)},
{'childProj': [{'childProj': [],
'description': u'desc 4 0',
'name': u'4 0',
'parentProj': None,
'urlSafe': Key('ndb_Project', 4964143656337408)},
{'childProj': [],
'description': u'desc 4 1',
'name': u'4 1',
'parentProj': None,
'urlSafe': Key('ndb_Project', 4927054734688256)},
{'childProj': [],
'description': u'desc 4 2',
'name': u'4 2',
'parentProj': None,
'urlSafe': Key('ndb_Project', 6348349238149120)},
{'childProj': [],
'description': u'desc 4 3',
'name': u'4 3',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5004957119938560)},
{'childProj': [],
'description': u'desc 4 4',
'name': u'4 4',
'parentProj': None,
'urlSafe': Key('ndb_Project', 4960843947048960)}],
'description': u'desc 4',
'name': u'4',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5250989925859328)}],
'description': u'first entity',
'name': u'first',
'parentProj': None,
'urlSafe': Key('ndb_Project', 5208229332123648)}
original code so that the original comments to this answer make sense
class ndb_Project(ndb.Model):
name = ndb.StringProperty()
description = ndb.StringProperty()
version = ndb.StructuredProperty(ndb_Version)
parentProj = ndb.KeyProperty()
childProj = ndb.KeyProperty(repeated=True)
#classmethod
def getChildProjects(cls, key):
proj = key.get()
a = ndb.get_multi(proj.childProj) if proj.childProj else []
o = proj.to_dict()
o['urlSafe'] = proj.key
o['childProj'] = a
return o