\n'''<\/code><\/p>\nThe program collects the names of the links, and opens all the pages,
\nthen searches for every
\ndiv style=”overflow-x: auto;”,
\nthen for all of them grabs the text inList of shipwrecks: 3 August 1914
\nelaborate the string taking off the “List of shipwrecks:”
\nthen collects all the(only if it hasand not)
\nPERHAPS NOT searches if the 3rdhas the coordinates and collects them
\nthen writes a line on a file with all the text in theseparated by @ and followed by the coordinates
\n”’
\ndef CheckLocation(Check_Link_Url,CheckLinkName): #filters the CheckLinkName string, and if passes the filter then scrape the page Check_Link_Url
\nFilterList =[“”,]
\nwith urllib.request.urlopen(Check_Link_Url) as response:
\ndata = response.read()<\/p>\n
def search_for_overflow(soup):<\/p>\n
for overflow_List in soup.findAll(‘div’, attrs={‘style’:’overflow-x:auto’}): # we search for the data. In our case the data we were looking for is always
\n#in a div with this specific attribute. these div represented each day of the month. They contained a table that listed all the ships sank in that date.<\/p>\n
date = overflow_List.caption.text
\ndate = date.replace(“List of shipwrecks: “,””) # cleans the title of the table for the day, to have the date in a simpler format
\npretty2=overflow_List.prettify() # there must be a more elegant way, but I found simpler to create a new beautifulsoup objec for every div found
\nsoup2 = BeautifulSoup(pretty2,’lxml’)
\nfor List_tr in soup2.findAll(‘tr’):# for all the div found we search for the table rows
\npretty3=List_tr.prettify() # and again we create a soup object of them
\nsoup3 = BeautifulSoup(pretty3,’lxml’)
\nfor index,List_td in enumerate(soup3.findAll(‘td’)): #then we search for all the cells<\/p>\n
stringLinea =re.sub(r’\\s+’, ‘ ‘, List_td.text)# we eliminate all the multiple spaces and transform them in single ones
\nstringLinea =re.sub(‘(\\[\\d+\\])’, ‘ ‘, stringLinea)#then we delete the quotes like for example “[1]”
\nif index stringLinea=stringLinea+”@” # we add @ as character separator
\nstringLinea=stringLinea.replace(“\\n”,””)# we delete the endline
\nwith codecs.open(“List_Results_wikipedia_All_Pages_Plus_Coordinates and Names.txt”, “a”,encoding=’utf8’) as file:
\nfile.write(stringLinea) #we write the cell on the file
\nelse: # this is the more complicated fourth cell
\nstringLinea=stringLinea.replace(“\\n”,””)
\nm = re.search(‘(?<=\/).+(?=\/)’, stringLinea) # we search for coordinates in the string
\nif m == None:
\nresult = “”
\nelse:
\nresult = m.group(0) # result = the found coordinates<\/p>\n
pretty4=List_td.prettify()# once again we create the object for beautifulsoup
\nsoup4 = BeautifulSoup(pretty4, ‘lxml’)
\nstringLinks =”none”
\nif soup4!=None:
\nListNameLinks = soup4.findAll(‘a’) #simplifying strategy: wikipedia often puts as a link interesting stuff in the texts, we search for them and store them
\n# to help the subsequent work<\/p>\n
if len(ListNameLinks)==0: # there are not links in the text
\nstringLinks=”none”
\nelse:
\nstringLinks=””
\nfor index,element in enumerate(ListNameLinks): # put in a single string all the links, separated by the semicolon
\nif index<\/p>\n
stringLinks = element.text
\nstringLinks = stringLinks.replace(“\\n”,””)
\nelse: # multiple links
\ntextadd= element.text
\ntextadd= textadd.replace(“\\n”,””)
\nstringLinks = stringLinks+”;”+textadd<\/p>\n
stringLinks =re.sub(r’\\s+’, ‘ ‘, stringLinks)# in some cases the string still had multiple spaces, just for good measure we eliminate them
\nwith codecs.open(“List_Results_wikipedia_All_Pages_Plus_Coordinates and Names.txt”, “a”,encoding=’utf8’) as file:<\/p>\n
file.write(stringLinea+”@”+date+”@”+result+”@”+stringLinks+”\\n”) # we organize the whole string as name@date@coordinates@the list of links in the text.<\/p>\n
def main():
\nwith open(“List_Results_wikipedia_All_Pages_Plus_Coordinates and Names.txt”, “w”): # this is the file where we will store all the data
\npass
\nwith open(‘List of Wikipedia links by month August 1914-December 1918’)as fileUrls: # we open the text file where we put all the web pages that we will scrape
\naddresses = fileUrls.readlines()<\/p>\n
for page in addresses: # for every page we open it
\nwith urllib.request.urlopen(page) as response:
\ndata = response.read()
\nprint(page)
\nsoup = BeautifulSoup(data, “lxml”) # we create the beautifulsoup object
\nsearch_for_overflow(soup) # and we search for the data that we want with our function<\/p>\n
if __name__ == “__main__”:
\nmain()<\/p>\n
<\/p>\n","protected":false},"excerpt":{"rendered":"
By Andrea Siotto<\/p>\n","protected":false},"author":9944,"featured_media":2145,"comment_status":"open","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"_monsterinsights_skip_tracking":false,"_monsterinsights_sitenote_active":false,"_monsterinsights_sitenote_note":"","_monsterinsights_sitenote_category":0,"footnotes":""},"categories":[2],"tags":[64,169],"class_list":["post-1960","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-grad-students","tag-coding","tag-web-scraping"],"_links":{"self":[{"href":"https:\/\/sites.temple.edu\/tudsc\/wp-json\/wp\/v2\/posts\/1960","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/sites.temple.edu\/tudsc\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/sites.temple.edu\/tudsc\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/sites.temple.edu\/tudsc\/wp-json\/wp\/v2\/users\/9944"}],"replies":[{"embeddable":true,"href":"https:\/\/sites.temple.edu\/tudsc\/wp-json\/wp\/v2\/comments?post=1960"}],"version-history":[{"count":0,"href":"https:\/\/sites.temple.edu\/tudsc\/wp-json\/wp\/v2\/posts\/1960\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/sites.temple.edu\/tudsc\/wp-json\/wp\/v2\/media\/2145"}],"wp:attachment":[{"href":"https:\/\/sites.temple.edu\/tudsc\/wp-json\/wp\/v2\/media?parent=1960"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/sites.temple.edu\/tudsc\/wp-json\/wp\/v2\/categories?post=1960"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/sites.temple.edu\/tudsc\/wp-json\/wp\/v2\/tags?post=1960"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}
![\"htmlcode\"](\"https:\/\/sites.temple.edu\/tudsc\/files\/2016\/08\/htmlcode.jpg\")
<\/a><\/p>\n