main.py 8.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182
  1. import pandas as pd
  2. from icalendar import Calendar, Event
  3. from datetime import datetime, timedelta
  4. import uuid
  5. import os
  6. import csv
  7. import requests
  8. import lxml.html as lh
  9. telegram_bot_api_key = USER = os.getenv('TELEGRAM_BOT_API_KEY')
  10. telegram_bot_chat_id = USER = os.getenv('TELEGRAM_BOT_CHAT_ID')
  11. def send_message(message:str)->None:
  12. """Send message to me on Telegram when updated.
  13. Args:
  14. message (str): String of message to send.
  15. """
  16. requests.post(f'https://api.telegram.org/bot{telegram_bot_api_key}/sendMessage', json={'chat_id': telegram_bot_chat_id, 'text': message})
  17. def store_df_as_csv(df:pd.DataFrame, name:str)->None:
  18. """Store dataframe as a CSV file.
  19. Args:
  20. df (pd.DataFrame): Dataframe of fixtures.
  21. name (str): Name of the CSV file.
  22. """
  23. df.to_csv(f'./{name}.csv', index=False)
  24. def compare_dfs(df:pd.DataFrame, name:str)->bool:
  25. """Compare the latest DF with the stored DF for any changes.
  26. Args:
  27. df (pd.DataFrame): Latest copy of fixtures in dataframe
  28. name (str): Name of the CSV file.
  29. Returns:
  30. bool: True if match, False if no match.
  31. """
  32. df2 = pd.read_csv(f'./{name}.csv')
  33. return df.equals(df2)
  34. def write_calendar(cal:Calendar)->None:
  35. """Write the cal object to an ics file.
  36. Args:
  37. cal (Calendar): iCalendar object with all the ics details.
  38. """
  39. f = open(os.path.join('./', 'fixtures.ics'), 'wb')
  40. f.write(cal.to_ical())
  41. f.close()
  42. def does_csv_exist()->bool:
  43. """Check if the CSV file exists.
  44. Returns:
  45. bool: True if CSV file exists, False if not.
  46. """
  47. return os.path.isfile('./fixtures.csv')
  48. def make_ordinal(n:int)->str:
  49. '''
  50. Convert an integer into its ordinal representation::
  51. make_ordinal(0) => '0th'
  52. make_ordinal(3) => '3rd'
  53. make_ordinal(122) => '122nd'
  54. make_ordinal(213) => '213th'
  55. '''
  56. n = int(n)
  57. if 11 <= (n % 100) <= 13:
  58. suffix = 'th'
  59. else:
  60. suffix = ['th', 'st', 'nd', 'rd', 'th'][min(n % 10, 4)]
  61. return str(n) + suffix
  62. def create_ical_file(df:pd.DataFrame, cal:Calendar, table:pd.DataFrame)->None:
  63. """Create an iCalendar file from a dataframe.
  64. Args:
  65. df (pd.DataFrame): Dataframe of fixtures.
  66. cal (Calendar): iCalendar object with all the ics details.
  67. table (pd.DataFrame): Dataframe of table details.
  68. """
  69. for index, row in df.iterrows():
  70. event = Event()
  71. match_type = str(row['Type'])
  72. home_team = str(row['Home Team'])
  73. if ("Tongham") not in home_team:
  74. home_team = str(row['Home Team']).replace(" U12","")
  75. away_team = str(row['Away Team.1'])
  76. if ("Tongham") not in away_team:
  77. away_team = str(row['Away Team.1']).replace(" U12","")
  78. venue = str(row['Venue'])
  79. print(row['Date / Time'], home_team, away_team, venue)
  80. if row['Date / Time'] == 'TBC':
  81. continue
  82. start_date_time = datetime.strptime(row['Date / Time'], '%d/%m/%y %H:%M')
  83. # Set default 8am start time to normal 930 kickoff time.
  84. if start_date_time.hour == 8:
  85. start_date_time = start_date_time + timedelta(hours=1, minutes=30)
  86. # Arrival time is 30 mins before kickoff time.
  87. arrival_time = start_date_time + timedelta(minutes=-30)
  88. if match_type == 'L':
  89. summary = "(League) " + home_team + f" ({make_ordinal(table.loc[table['Team'] == home_team, 'POS'].iloc[0])})" + f" {str(row['Unnamed: 4'])} " + away_team + f" ({make_ordinal(table.loc[table['Team'] == away_team, 'POS'].iloc[0])})"
  90. else:
  91. summary = "(Cup) " + home_team + f" {str(row['Unnamed: 4'])} " + away_team
  92. event.add('summary', summary)
  93. notes = row['Status / Notes']
  94. if pd.isna(notes):
  95. notes = 'No Match Notes'
  96. elif notes == 'Postponed':
  97. continue
  98. event.add('description', "Arrive by - " + str(arrival_time) + "\n" + notes + "\nTable -\n" + "https://fulltime.thefa.com/table.html?selectedSeason=19010414&selectedDivision=165601607&ftsTablePageContent.fixtureAnalysisForm.standingsTableDay=14&ftsTablePageContent.fixtureAnalysisForm.standingsTableMonth=0&ftsTablePageContent.fixtureAnalysisForm.standingsTableYear=2024&activeTab=1")
  99. event.add('dtstart', start_date_time)
  100. # End 2 hours after start_date_time
  101. event.add('dtend', start_date_time + timedelta(hours=2))
  102. event.add('dtstamp', start_date_time)
  103. event.add('uid', str(uuid.uuid4()))
  104. event.add('location', venue)
  105. cal.add_component(event)
  106. write_calendar(cal)
  107. def process_table(table_df:pd.DataFrame)->pd.DataFrame:
  108. table_df = table_df[:-1]
  109. table_df.drop(table_df.columns[len(table_df.columns)-1], axis=1, inplace=True)
  110. table_df['POS'] = table_df['POS'].astype('int')
  111. table_df['P'] = table_df['P'].astype('int')
  112. table_df['W'] = table_df['W'].astype('int')
  113. table_df['D'] = table_df['D'].astype('int')
  114. table_df['L'] = table_df['L'].astype('int')
  115. table_df['PTS'] = table_df['PTS'].astype('int')
  116. store_df_as_csv(table_df, "table")
  117. return table_df
  118. def process_results()->None:
  119. req = requests.get("https://fulltime.thefa.com/results.html?selectedSeason=19010414&selectedFixtureGroupAgeGroup=11&selectedFixtureGroupKey=1_579285719&selectedRelatedFixtureOption=3&selectedClub=&selectedTeam=466317969&selectedDateCode=all&previousSelectedFixtureGroupAgeGroup=11&previousSelectedFixtureGroupKey=1_579285719&previousSelectedClub=")
  120. doc = lh.fromstring(req.text)
  121. headers = ['Date', 'Home Team', 'Score', 'Away Team']
  122. with open('results.csv', 'w', newline='') as fp:
  123. file = csv.writer(fp)
  124. file.writerow(headers)
  125. for idx,row in enumerate(doc.xpath("//div[contains(@id,'fixture')]"), start=1):
  126. date = row.xpath(f'/html[1]/body[1]/main[1]/div[2]/section[1]/div[1]/div[3]/div[1]/div[2]/div[{idx}]/div[1]/div[3]/a[1]/span[1]//text()')[0]
  127. home_team = row.xpath(f'/html[1]/body[1]/main[1]/div[2]/section[1]/div[1]/div[3]/div[1]/div[2]/div[{idx}]/div[1]/div[4]/div[1]/a[1]//text()')[0].strip()
  128. score = row.xpath(f'/html[1]/body[1]/main[1]/div[2]/section[1]/div[1]/div[3]/div[1]/div[2]/div[{idx}]/div[1]/div[5]//text()')[0].strip()
  129. if score == 'X - X':
  130. continue
  131. away_team = row.xpath(f'/html[1]/body[1]/main[1]/div[2]/section[1]/div[1]/div[3]/div[1]/div[2]/div[{idx}]/div[1]/div[6]/div[2]/a[1]//text()')[0].strip()
  132. file.writerow([date,home_team,score,away_team])
  133. def compare_table():
  134. table_df = pd.read_html("https://fulltime.thefa.com/table.html?league=9268728&selectedSeason=19010414&selectedDivision=165601607&selectedCompetition=0&selectedFixtureGroupKey=1_579285719")[0]
  135. store_df_as_csv(table_df, "base_table")
  136. return table_df
  137. cal = Calendar()
  138. cal.add('prodid', 'Down Grange Pumas Fixtures')
  139. cal.add('version', '2.0')
  140. fixtures_df = pd.read_html("https://fulltime.thefa.com/fixtures.html?selectedSeason=19010414&selectedFixtureGroupAgeGroup=11&selectedFixtureGroupKey=1_579285719&selectedDateCode=all&selectedClub=&selectedTeam=466317969&selectedRelatedFixtureOption=3&selectedFixtureDateStatus=&selectedFixtureStatus=&previousSelectedFixtureGroupAgeGroup=11&previousSelectedFixtureGroupKey=1_579285719&previousSelectedClub=&itemsPerPage=25")[0]
  141. fixtures_df.head()
  142. process_results()
  143. table = compare_table()
  144. exists = does_csv_exist()
  145. if exists:
  146. fixtures_change = compare_dfs(fixtures_df, "fixtures")
  147. table_change = compare_dfs(table, "base_table")
  148. if not table_change:
  149. send_message("Table has updated")
  150. if not all([fixtures_change, table_change]):
  151. print("Data Updated, ical updated")
  152. store_df_as_csv(fixtures_df, "fixtures")
  153. create_ical_file(fixtures_df, cal, process_table(table))
  154. send_message("Fixtures updated, ical updated")
  155. else:
  156. print("No Data Updated, No update to ical")
  157. else:
  158. store_df_as_csv(fixtures_df, "fixtures")
  159. create_ical_file(fixtures_df, cal, process_table(table))
  160. send_message("New ical file created")
  161. print("New ical file created")