Snippet content copied to clipboard.
Are you sure to delete this snippet? No, don't delete
  1. #!/usr/bin/env python
  2. ​
  3. import csv
  4. import re
  5. from time import sleep
  6. ​
  7. from lxml import etree, html
  8. from selenium import webdriver
  9. from selenium.common.exceptions import NoSuchElementException
  10. from selenium.webdriver.common.keys import Keys
  11. from selenium.webdriver.common.by import By
  12. from selenium.webdriver.support.ui import WebDriverWait
  13. from selenium.webdriver.support.expected_conditions import visibility_of_element_located
  14. ​
  15. # This is for test, load the urls list from https://dpaste.org/nZpuq
  16. ​
  17. urls_list = [
  18. "https://www.flashscore.com/football/zimbabwe/premier-soccer-league-2021-2022/",
  19. ]
  20. ​
  21. for item in urls_list:
  22. base_url = item + "results/"
  23. print(f"Working on {item}")
  24. # As this code was created originaly in Linux, this is the source of Geecko driver.
  25. path = "/usr/local/bin/geckodriver"
  26. driver_service = webdriver.FirefoxService(executable_path=path)
  27. driver = webdriver.Firefox(service=driver_service)
  28. driver.get(base_url)
  29. ​
  30. # Reject cookies
  31. WebDriverWait(driver, 10).until(visibility_of_element_located((By.XPATH, '//button[contains(@id, "onetrust-reject-all-handler")]'))).location
  32. driver.find_element(By.XPATH, '//button[contains(@id, "onetrust-reject-all-handler")]').click()
  33. ​
  34. # Look for all content of results page, as it loads the content dynamically
  35. while True:
  36. html_content = html.fromstring(driver.page_source)
  37. tree = etree.ElementTree(html_content)
  38. show_more_item = html_content.xpath(".//span[text()='Show more matches']")
  39. if not show_more_item == []:
  40. try:
  41. while True:
  42. show_more_path = tree.getpath(html_content.xpath(".//span[text()='Show more matches']")[0].getparent())
  43. driver.find_element(By.XPATH, show_more_path).send_keys(Keys.PAGE_DOWN)
  44. sleep(5)
  45. driver.find_element(By.XPATH, show_more_path).click()
  46. except NoSuchElementException:
  47. print("finish")
  48. break
  49. else:
  50. break
  51. ​
  52. html_content = html.fromstring(driver.page_source)
  53. driver.close()
  54. ​
  55. # Div container of all results
  56. results = html_content.xpath('//div[@id = "live-table"]//div[@class = "event event--results"]//div[@class = "leagues--static event--leagues results"]//div[@class = "sportName soccer"]')[0]
  57. ​
  58. standings = []
  59. round = None
  60. tournament = None
  61. last_month = None
  62. year = None
  63. ​
  64. for row in results.getchildren():
  65. if row.attrib.get("class") == "wcl-header_uBhYi wclLeagueHeader wclLeagueHeader--collapsed wclLeagueHeader--noCheckBox wclLeagueHeader--indent":
  66. tournament = row.text_content().replace("Standings", "").replace("Draw", "").replace(chr(160), "")
  67. if row.attrib.get("class") == "event__round event__round--static":
  68. round = row.text_content()
  69. if row.attrib.get("class").startswith("event__match"):
  70. date = row.xpath(".//div[@class = 'event__time']")[0].text_content().split()[0][:-1]
  71. years = re.search(r"\d{4}\-\d{4}", base_url)
  72. match = {
  73. "date": date,
  74. "time": row.xpath(".//div[@class = 'event__time']")[0].text_content().split()[1],
  75. "home_team": row.xpath('.//div[contains(@class, "homeParticipant")]')[0].text_content(),
  76. "away_team": row.xpath('.//div[contains(@class, "awayParticipant")]')[0].text_content(),
  77. "home_score": row.xpath('.//span[contains(@class, "event__score--home")]')[0].text_content(),
  78. "away_score": row.xpath('.//span[contains(@class, "event__score--away")]')[0].text_content()
  79. }
  80. if years:
  81. _years = years.group().split("-")
  82. if last_month == None: last_month = int(date.split(".")[-1])
  83. if year == None: year = _years[-1]
  84. if int(date.split(".")[-1]) <= last_month:
  85. match["date"] = f"{date}.{year}"
  86. last_month = int(date.split(".")[-1])
  87. else:
  88. print("change year")
  89. year = _years[0]
  90. match["date"] = f"{date}.{year}"
  91. last_month = int(date.split(".")[-1])
  92. ​
  93. standings.append({"tournament": tournament, "round": round} | match)
  94. ​
  95. # Save results to file
  96. fieldnames = ['tournament', 'round', 'date', 'time', 'home_team', 'away_team', 'home_score', 'away_score']
  97. ​
  98. with open('./tournament_results.csv', 'a', newline='') as csvfile:
  99. writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
  100. writer.writeheader()
  101. writer.writerows(standings)
  102. sleep(5)

Edit this Snippet