Scraping data in to a dataframe - web-scraping

Question

Is there an elegant way to scrape data from URLs such as below directly into a dataframe?:
https://israeldrugs.health.gov.il/#!/medDetails/145%2020%2032014%2000

Answer 1

Try:
import requests
import pandas as pd
url = "https://israeldrugs.health.gov.il/#!/medDetails/145 20 32014 00"
api_url = (
"https://israeldrugs.health.gov.il/GovServiceList/IDRServer/GetSpecificDrug"
)
drug_id = url.split("/")[-1]
data = requests.post(api_url, json={"dragRegNum": drug_id}).json()
df = pd.json_normalize(data)
print(df)
Prints:
dragRegNum dragHebName bitulDate isCytotoxic isVeterinary applicationType brochure brochureUpdate isPrescription iscanceled images dragEnName usageFormHeb usageFormEng dosageForm dosageFormEng dragIndication maxPrice health activeMetirals regOwnerName regManufactureName regDate regExpDate applicationDate custom manufacturers limitations dateOfInclusion indicationIncludedInTheBasket classEffect remarks packingLimitation registeredIndicationsAtTimeOfInclusion frameworkOfInclusion useInClalit salList atc packages videos
0 145 20 32014 00 אדויל פורטה 400 01/01/1900 False False תכשיר גנרי [{'lng': None, 'url': 'Rishum_17_463219520.pdf', 'updateDate': 1607238596000.0, 'type': 'החמרה לעלון', 'display': 'החמרה לעלון', 'updateDateFormat': '06.12.2020', 'creationDateFormat': '25.11.2020'}, {'lng': None, 'url': 'Rishum_20_510626721.pdf', 'updateDate': 1624563325000.0, 'type': 'החמרה לעלון', 'display': 'החמרה לעלון', 'updateDateFormat': '24.06.2021', 'creationDateFormat': '14.06.2021'}, {'lng': 'אנגלית', 'url': 'Rishum_19_248052021.pdf', 'updateDate': 1613923027000.0, 'type': 'עלון לצרכן', 'display': 'עלון לצרכן אנגלית', 'updateDateFormat': '21.02.2021', 'creationDateFormat': '13.01.2021'}, {'lng': 'עברית', 'url': 'Rishum_20_510625821.pdf', 'updateDate': 1624563286000.0, 'type': 'עלון לצרכן', 'display': 'עלון לצרכן עברית', 'updateDateFormat': '24.06.2021', 'creationDateFormat': '14.06.2021'}, {'lng': 'ערבית', 'url': 'Rishum_19_248050821.pdf', 'updateDate': 1613922982000.0, 'type': 'עלון לצרכן', 'display': 'עלון לצרכן ערבית', 'updateDateFormat': '21.02.2021', 'creationDateFormat': '13.01.2021'}, {'lng': None, 'url': 'Rishum_20_510583121.pdf', 'updateDate': 1624560223000.0, 'type': 'עלון לרופא', 'display': 'עלון לרופא', 'updateDateFormat': '24.06.2021', 'creationDateFormat': '14.06.2021'}] None False False [{'url': 'Rishum_1_88158316.jpg', 'updateDate': 1469069874000.0}] ADVIL FORTE 400 פומי PER OS קפסולות ממולאות נוזל CAPSULES LIQUID FILLED Relief of mild to moderate pain such as headache, treatment of pain associated with migraine toothache, menstrual pain, backache, muscular pain, anti-inflammatory for rheumatic disease, reduction of fever. 49.2 False [{'ingredientsDesc': 'IBUPROFEN', 'dosage': '400 MG '}] GSK CONSUMER HEALTHCARE, ISRAEL LTD PFIZER CONSUMER MANUFACTURING ITALY S.R.L 1.454198e+12 1.769818e+12 1.233619e+12 [{'manufactureName': 'CATALENT ARGENTINA SAIC', 'manufactureSite': 'AVENIDA BERNABE MARQUEZ 691 B1657CSA - VILLA LOMA HERMOSA, BUENOS AIRES, ARGENTINA', 'manufactureComments': 'bulk '}, {'manufactureName': 'PFIZER CONSUMER MANUFACTURING ITALY S.R.L.', 'manufactureSite': 'VIA NETTUNENSE 90, APRILILA, ITALY', 'manufactureComments': 'release , testing , labelling , packaging '}] None None None None None None None None None [] [{'atc4Code': 'M02AA', 'atc4Name': 'ANTIINFLAMMATORY PREPARATIONS, NON STEROID FOR TOPICAL USE', 'atc5Code': 'M02AA13', 'atc5Name': 'IBUPROFEN'}] [{'isPrescription': True, 'packageUpdate': 0.0, 'packageDesc': 'BLISTER PVC/PE/PVDC/AL', 'packMaterialDesc': 'PVC/PE/PVDC/AL', 'unitPrice': '0', 'packageMaxPrice': '0', 'quantity': '96 CAPSULES', 'shelfLife': '36', 'unit': 'חדשים', 'barcode': ''}, {'isPrescription': True, 'packageUpdate': 0.0, 'packageDesc': 'BLISTER PVC/PE/PVDC/AL', 'packMaterialDesc': 'PVC/PE/PVDC/AL', 'unitPrice': '0', 'packageMaxPrice': '0', 'quantity': '100 CAPSULES', 'shelfLife': '36', 'unit': 'חדשים', 'barcode': ''}, {'isPrescription': True, 'packageUpdate': 0.0, 'packageDesc': 'BLISTER PVC/PE/PVDC/AL', 'packMaterialDesc': 'PVC/PE/PVDC/AL', 'unitPrice': '0', 'packageMaxPrice': '0', 'quantity': '4 CAPSULES', 'shelfLife': '36', 'unit': 'חדשים', 'barcode': ''}, {'isPrescription': True, 'packageUpdate': 0.0, 'packageDesc': 'BLISTER PVC/PE/PVDC/AL', 'packMaterialDesc': 'PVC/PE/PVDC/AL', 'unitPrice': '0', 'packageMaxPrice': '0', 'quantity': '6 CAPSULES', 'shelfLife': '36', 'unit': 'חדשים', 'barcode': ''}, {'isPrescription': True, 'packageUpdate': 0.0, 'packageDesc': 'BLISTER PVC/PE/PVDC/AL', 'packMaterialDesc': 'PVC/PE/PVDC/AL', 'unitPrice': '0', 'packageMaxPrice': '0', 'quantity': '8 CAPSULES', 'shelfLife': '36', 'unit': 'חדשים', 'barcode': ''}, {'isPrescription': True, 'packageUpdate': 0.0, 'packageDesc': 'BLISTER PVC/PE/PVDC/AL', 'packMaterialDesc': 'PVC/PE/PVDC/AL', 'unitPrice': '0', 'packageMaxPrice': '0', 'quantity': '10 CAPSULES', 'shelfLife': '36', 'unit': 'חדשים', 'barcode': ''}, {'isPrescription': True, 'packageUpdate': 0.0, 'packageDesc': 'BLISTER PVC/PE/PVDC/AL', 'packMaterialDesc': 'PVC/PE/PVDC/AL', 'unitPrice': '0', 'packageMaxPrice': '0', 'quantity': '12 CAPSULES', 'shelfLife': '36', 'unit': 'חדשים', 'barcode': ''}, {'isPrescription': True, 'packageUpdate': 0.0, 'packageDesc': 'BLISTER PVC/PE/PVDC/AL', 'packMaterialDesc': 'PVC/PE/PVDC/AL', 'unitPrice': '0', 'packageMaxPrice': '0', 'quantity': '16 CAPSULES', 'shelfLife': '36', 'unit': 'חדשים', 'barcode': ''}, {'isPrescription': True, 'packageUpdate': 0.0, 'packageDesc': 'BLISTER PVC/PE/PVDC/AL', 'packMaterialDesc': 'PVC/PE/PVDC/AL', 'unitPrice': '2.46', 'packageMaxPrice': '49.2', 'quantity': '20 CAPSULES', 'shelfLife': '36', 'unit': 'חדשים', 'barcode': '7290012368332'}, {'isPrescription': True, 'packageUpdate': 0.0, 'packageDesc': 'BLISTER PVC/PE/PVDC/AL', 'packMaterialDesc': 'PVC/PE/PVDC/AL', 'unitPrice': '0', 'packageMaxPrice': '0', 'quantity': '24 CAPSULES', 'shelfLife': '36', 'unit': 'חדשים', 'barcode': ''}, {'isPrescription': True, 'packageUpdate': 0.0, 'packageDesc': 'BLISTER PVC/PE/PVDC/AL', 'packMaterialDesc': 'PVC/PE/PVDC/AL', 'unitPrice': '0', 'packageMaxPrice': '0', 'quantity': '30 CAPSULES', 'shelfLife': '36', 'unit': 'חדשים', 'barcode': ''}, {'isPrescription': True, 'packageUpdate': 0.0, 'packageDesc': 'BLISTER PVC/PE/PVDC/AL', 'packMaterialDesc': 'PVC/PE/PVDC/AL', 'unitPrice': '0', 'packageMaxPrice': '0', 'quantity': '32 CAPSULES', 'shelfLife': '36', 'unit': 'חדשים', 'barcode': ''}, {'isPrescription': True, 'packageUpdate': 0.0, 'packageDesc': 'BLISTER PVC/PE/PVDC/AL', 'packMaterialDesc': 'PVC/PE/PVDC/AL', 'unitPrice': '0', 'packageMaxPrice': '0', 'quantity': '36 CAPSULES', 'shelfLife': '36', 'unit': 'חדשים', 'barcode': ''}, {'isPrescription': True, 'packageUpdate': 0.0, 'packageDesc': 'BLISTER PVC/PE/PVDC/AL', 'packMaterialDesc': 'PVC/PE/PVDC/AL', 'unitPrice': '0', 'packageMaxPrice': '0', 'quantity': '48 CAPSULES', 'shelfLife': '36', 'unit': 'חדשים', 'barcode': ''}, {'isPrescription': True, 'packageUpdate': 0.0, 'packageDesc': 'BLISTER PVC/PE/PVDC/AL', 'packMaterialDesc': 'PVC/PE/PVDC/AL', 'unitPrice': '0', 'packageMaxPrice': '0', 'quantity': '50 CAPSULES', 'shelfLife': '36', 'unit': 'חדשים', 'barcode': ''}, {'isPrescription': True, 'packageUpdate': 0.0, 'packageDesc': 'BLISTER PVC/PE/PVDC/AL', 'packMaterialDesc': 'PVC/PE/PVDC/AL', 'unitPrice': '0', 'packageMaxPrice': '0', 'quantity': '60 CAPSULES', 'shelfLife': '36', 'unit': 'חדשים', 'barcode': ''}] []