How to extract specific text from a webpage - web-scraping

Question

I am interested in finding patterns of constellations. I am using 'Sky map' android app for visual inspection, now I want to build an app to find similar constellation structures. A sub-problem of that is to find the coordinates for specific celestial objects.
Example: How can I obtain the coordinates of 'Moon' at a given time,date and location.
https://theskylive.com/planetarium provides this information on their webpage in following manner.
Object: Moon [info|live][less]
Right Asc: 04h 15m 12.5s **Decl: 17° 05' 46.3"** (J2000) [HMS|Dec]
Magnitude: -10.54 Altitude: 56° Solar Elongation: 100.4° Constellation: Ari
Sun distance: 147.77 Million Km Earth distance: 0.38 Million Km
Rise: 10:48 Transit: 18:40 Set: 01:35 **Europe/London**
For Moon we can find coordinates using the webpage, Is there some API? or How can we do it by extracting coordinate information from the web page.

Answer 1

I am not an Android expert, but this is what you can do in
build.gradle
plugins {
id 'java'
}
group 'test.test'
version '1.0-SNAPSHOT'
sourceCompatibility = 1.8
repositories {
mavenCentral()
}
dependencies {
testCompile group: 'junit', name: 'junit', version: '4.12'
implementation 'com.squareup.okhttp3:okhttp:3.13.1'
compile group: 'org.json', name: 'json', version: '20180813'
}
Planetarium.java
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import org.json.JSONObject;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
public class Planetarium {
OkHttpClient client = new OkHttpClient();
JSONObject get(String... objects) throws IOException {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
StringBuilder urlBuilder = new StringBuilder("https://theskylive.com/planetariumdata?");
// Current date in YYYY-MM-DD format
urlBuilder.append("date=").append(dateFormat.format(new Date()));
// add url query string for all objects
for (String obj : objects) {
// URL encoded aobj[] => aobj%5B%5D=
urlBuilder.append("&").append("aobj%5B%5D=").append(obj);
}
Request request = new Request.Builder()
.url(urlBuilder.toString())
.build();
try (Response response = client.newCall(request).execute()) {
String json = response.body().string();
return new JSONObject(json);
}
}
public static void main(String[] args) throws IOException {
Planetarium planetarium = new Planetarium();
JSONObject response = planetarium.get("moon", "mars");
System.out.println(response.toString(2));
}
}
OUTPUT:
{
"utc_seconds": 1551816600,
"utc_timestamp": "201903052010",
"objects": {
"moon": {
"distsun": 1.479847408587E8,
"altitude": -32.421642244539,
"dec": -12.501182812768,
"constell": "Cap",
"timezone": "Europe/London",
"hlat": "-0.0075",
"hlong": "163.9072",
"elongation": "9.6",
"lastdate": "2019-Mar-05 00:00",
"hx": -0.95427043393163,
"hy": 0.26061067578779,
"mag": "-4.82",
"hlongRad": 2.8607203077248,
"hz": -1.6343451194632E-4,
"utc_time": 1551816600,
"distearth": 405722.20937018,
"sot": 350.29647638889,
"id": "moon",
"circumstances": {
"transit_local": 11.428494722983,
"raise_ut": 1.5517668981849E9,
"set": 16.623858118962,
"raise_local": 6.3606069281934,
"visibility": "partial",
"azimuth_set": 256.90380469917,
"LSTs": 3.4997935653561,
"LSTr": 17.208442522882,
"set_local": 16.623858118962,
"azimuth_rise": 104.50312047906,
"GSTs": 3.4997935653561,
"GSTr": 17.208442522882,
"transit_ut": 1.551785142581E9,
"transit": 11.428494722983,
"raise": 6.3606069281934,
"set_ut": 1.5518038458892E9,
"transit_height": 24.710020581601
},
"ar": 22.578738425926,
"name": "Moon",
"category": "planets",
"hlatRad": -1.3089969389957E-4,
"age": 27,
"status": true
},
"mars": {
"distsun": 2.2963710671492E8,
"altitude": 27.808183248664,
"circumstances": {
"transit_local": 15.80120694427,
"raise_ut": 1.5517741680418E9,
"set": 23.222402283833,
"raise_local": 8.3800116047075,
"visibility": "partial",
"azimuth_set": 286.34760861411,
"LSTs": 10.11640394619,
"LSTr": 19.233376146402,
"set_local": 23.222402283833,
"azimuth_rise": 73.652391385888,
"GSTs": 10.11640394619,
"GSTr": 19.233376146402,
"transit_ut": 1.551800884345E9,
"transit": 15.80120694427,
"raise": 8.3800116047075,
"set_ut": 1.5518276006482E9,
"transit_height": 54.867608614112
},
"dec": 16.347608614112,
"constell": "Ari",
"timezone": "Europe/London",
"hlat": "0.8142",
"hlong": "75.6345",
"elongation": "58.1",
"lastdate": "2019-Mar-05 00:00",
"hx": 0.36958631955143,
"ar": 2.6748900462963,
"hy": 1.4897081109635,
"mag": "1.23",
"hlongRad": 1.3200710530997,
"hz": 0.022145899657793,
"utc_time": 1551816600,
"distearth": 2.704192732295E8,
"name": "Mars",
"sot": 58.1002,
"id": "mars",
"category": "planets",
"hlatRad": 0.014210470769738,
"status": true
},
"sun": {
"distsun": 0,
"altitude": -22.992657046501,
"circumstances": {
"transit_local": 12.176106019167,
"raise_ut": 1.551767861711E9,
"set": 17.739026911053,
"raise_local": 6.6282530456618,
"visibility": "partial",
"azimuth_set": 263.93596334029,
"LSTs": 4.618015588543,
"LSTr": 17.476821431166,
"set_local": 17.739026911053,
"azimuth_rise": 96.242086753282,
"GSTs": 4.618015588543,
"GSTr": 17.476821431166,
"transit_ut": 1.5517878339817E9,
"transit": 12.176106019167,
"raise": 6.6282530456618,
"set_ut": 1.5518078604969E9,
"transit_height": 32.366908597329
},
"dec": -6.0242450863769,
"constell": "Aqr",
"timezone": "Europe/London",
"hlat": "n.a.",
"hlong": "n.a.",
"elongation": 0,
"lastdate": "2019-Mar-05 00:00",
"hx": 0,
"ar": 23.060617283951,
"hy": 0,
"mag": "-26.76",
"hlongRad": null,
"hz": 0,
"utc_time": 1551816600,
"distearth": 1.4838474994878E8,
"name": "Sun",
"sot": 0,
"id": "sun",
"category": "planets",
"hlatRad": null,
"status": true
}
},
"target": "sun"
}

Answer 2

I'm not sure if this is helpful, but here is a python implementation to it. You'd have to figure out the accepted location parameters, but date, hour, and minute are all there:
import requests
url = 'https://theskylive.com/planetariumdata'
params = {
'obj': 'moon',
'h': '10',
'm': '30',
'date': '2019-02-28',
'localdata': '51.48|0|Greenwich, United Kingdom|Europe/London'}
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'}
response = requests.get(url, params=params, headers=headers).json()
print (response['objects']['moon'])
Output:
{'status': True, 'utc_time': 1551349800, 'ar': 18.114288194444, 'dec': -21.301003146701, 'mag': '-9.11', 'distsun': 148031243.76562, 'distearth': 399053.81054688, 'constell': 'Oph', 'sot': 292.2907375, 'lastdate': '2019-Feb-28 00:00', 'hlong': '158.9866', 'hlongRad': 2.7748396365512, 'hlat': '0.0060', 'hlatRad': 0.00010471975511966, 'hx': -0.92639216172362, 'hy': 0.34779595586615, 'hz': 8.4403227939488e-05, 'elongation': '67.7', 'altitude': 7.7566655880485, 'id': 'moon', 'name': 'Moon', 'category': 'planets', 'circumstances': {'visibility': 'partial', 'raise': 3.0419974882059, 'set': 11.875359660362, 'transit': 7.4771821984014, 'raise_ut': 1551322951.191, 'set_ut': 1551354751.2948, 'transit_ut': 1551338917.8559, 'transit_height': 17.341269275926, 'azimuth_rise': 110.98610232928, 'azimuth_set': 248.66063774998, 'LSTr': 13.552197907652, 'LSTs': 22.409745024956, 'GSTr': 13.552197907652, 'GSTs': 22.409745024956, 'raise_local': 3.0419974882059, 'set_local': 11.875359660362, 'transit_local': 7.4771821984014}, 'timezone': 'Europe/London', 'age': 23}

Answer 3

Below you can find the code on how to do this in python. There are numerous ways to incorporate the code in an app. Just for illustration I casted the results in a pandas dataframe so you could view the results. I also added some code to deal with proxy settings, if this is not so, you could leave it out and simply get the url text with the requests package.
Hope it helps.
import urllib
import pandas as pd
import numpy as np
username = 'userID' # ex. ID
password = "password!" # password
dataURL = "https://theskylive.com/moon-info"
proxies = {
'https': 'https://{}:{}#proxy:port'.format(username, password)}
proxy = urllib.request.ProxyHandler(proxies)
opener = urllib.request.build_opener(proxy)
urllib.request.install_opener(opener)
with urllib.request.urlopen(dataURL) as url:
text = str(url.read())
tableStart = text.find('The Moon Ephemeris')
tableEnd = text.find('Distance of The Moon from Earth')
tableProgress = tableStart
findSTR = 'moon&date='
loc = text.find(findSTR,tableStart,tableEnd)
startDate = text[loc+len(findSTR):loc+len(findSTR)+10]
table = []
tableRow = []
counter = 0
counter2 = 0
diff = [20,4]
while loc>0:
loc1 = text.find('<td class="desktop">',tableProgress,tableEnd)
loc2 = text.find('<td>',tableProgress,tableEnd)
if loc1<0:
if loc2<0:
loc = -1
else:
loc = loc2
pos = 1
else:
if loc2<0:
loc = loc1
pos = 0
else:
loc = np.min([loc1,loc2])
pos = np.argmin([loc1,loc2])
if loc>0:
locStart = loc+diff[pos]
loc = text.find('</td>',loc,tableEnd)
if loc>0:
extractedText = text[locStart:loc]
if counter ==1:
extractedText = extractedText.replace('°',' deg')
extractedText = extractedText.replace('’',' min')
extractedText = extractedText.replace('”',' sec')
elif counter ==3:
extractedText = extractedText.replace('”',' arcsec')
tableRow = tableRow+ [extractedText]
tableProgress = loc
counter = counter+1
if counter==5:
counter2 = counter2+1
counter = 0
table = table+[tableRow]
tableRow = []
idx = pd.date_range(start='2019-02-26', periods=len(table), freq='D')
cols = ['Right Ascension','Declination','Magnitude','Apparent Diameter','Constellation']
Data = pd.DataFrame(table,index=idx,columns=cols)
print(Data)

Answer 4

Just instead setting up your own data scraping server you could use IFTTT for getting the data and storeing it at first.
Here are some nice tutorials: https://public.tableau.com/en-us/s/blog/2013/08/data-scraping-part-i-ifttt
Im sure you don't want your app to do the scraping.
After you got it stored you can further manipulate it with whatever language and provide it in e.g. json format as a cacheable datasource for your app.