1. #!/usr/bin/env python
2. ​
3. import csv
4. import re
5. from time import sleep
6. ​
7. from lxml import etree, html
8. from selenium import webdriver
9. from selenium.common.exceptions import NoSuchElementException
10. from selenium.webdriver.common.keys import Keys
11. from selenium.webdriver.common.by import By
12. from selenium.webdriver.support.ui import WebDriverWait
13. from selenium.webdriver.support.expected_conditions import visibility_of_element_located
14. ​
15. # This is for test, load the urls list from https://dpaste.org/nZpuq
16. ​
17. urls_list = [
18. "https://www.flashscore.com/football/zimbabwe/premier-soccer-league-2021-2022/",
19. ]
20. ​
21. for item in urls_list:
22. base_url = item + "results/"
23. print(f"Working on {item}")
24. # As this code was created originaly in Linux, this is the source of Geecko driver.
25. path = "/usr/local/bin/geckodriver"
26. driver_service = webdriver.FirefoxService(executable_path=path)
27. driver = webdriver.Firefox(service=driver_service)
28. driver.get(base_url)
29. ​
30. # Reject cookies
31. WebDriverWait(driver, 10).until(visibility_of_element_located((By.XPATH, '//button[contains(@id, "onetrust-reject-all-handler")]'))).location
32. driver.find_element(By.XPATH, '//button[contains(@id, "onetrust-reject-all-handler")]').click()
33. ​
34. # Look for all content of results page, as it loads the content dynamically
35. while True:
36. html_content = html.fromstring(driver.page_source)
37. tree = etree.ElementTree(html_content)
38. show_more_item = html_content.xpath(".//span[text()='Show more matches']")
39. if not show_more_item == []:
40. try:
41. while True:
42. show_more_path = tree.getpath(html_content.xpath(".//span[text()='Show more matches']")[0].getparent())
43. driver.find_element(By.XPATH, show_more_path).send_keys(Keys.PAGE_DOWN)
44. sleep(5)
45. driver.find_element(By.XPATH, show_more_path).click()
46. except NoSuchElementException:
47. print("finish")
48. break
49. else:
50. break
51. ​
52. html_content = html.fromstring(driver.page_source)
53. driver.close()
54. ​
55. # Div container of all results
56. results = html_content.xpath('//div[@id = "live-table"]//div[@class = "event event--results"]//div[@class = "leagues--static event--leagues results"]//div[@class = "sportName soccer"]')[0]
57. ​
58. standings = []
59. round = None
60. tournament = None
61. last_month = None
62. year = None
63. ​
64. for row in results.getchildren():
65. if row.attrib.get("class") == "wcl-header_uBhYi wclLeagueHeader wclLeagueHeader--collapsed wclLeagueHeader--noCheckBox wclLeagueHeader--indent":
66. tournament = row.text_content().replace("Standings", "").replace("Draw", "").replace(chr(160), "")
67. if row.attrib.get("class") == "event__round event__round--static":
68. round = row.text_content()
69. if row.attrib.get("class").startswith("event__match"):
70. date = row.xpath(".//div[@class = 'event__time']")[0].text_content().split()[0][:-1]
71. years = re.search(r"\d{4}\-\d{4}", base_url)
72. match = {
73. "date": date,
74. "time": row.xpath(".//div[@class = 'event__time']")[0].text_content().split()[1],
75. "home_team": row.xpath('.//div[contains(@class, "homeParticipant")]')[0].text_content(),
76. "away_team": row.xpath('.//div[contains(@class, "awayParticipant")]')[0].text_content(),
77. "home_score": row.xpath('.//span[contains(@class, "event__score--home")]')[0].text_content(),
78. "away_score": row.xpath('.//span[contains(@class, "event__score--away")]')[0].text_content()
79. }
80. if years:
81. _years = years.group().split("-")
82. if last_month == None: last_month = int(date.split(".")[-1])
83. if year == None: year = _years[-1]
84. if int(date.split(".")[-1]) <= last_month:
85. match["date"] = f"{date}.{year}"
86. last_month = int(date.split(".")[-1])
87. else:
88. print("change year")
89. year = _years[0]
90. match["date"] = f"{date}.{year}"
91. last_month = int(date.split(".")[-1])
92. ​
93. standings.append({"tournament": tournament, "round": round} | match)
94. ​
95. # Save results to file
96. fieldnames = ['tournament', 'round', 'date', 'time', 'home_team', 'away_team', 'home_score', 'away_score']
97. ​
98. with open('./tournament_results.csv', 'a', newline='') as csvfile:
99. writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
100. writer.writeheader()
101. writer.writerows(standings)
102. sleep(5)