From 74707ac3b498e74a2b3f9607ca1098a2da5dd213 Mon Sep 17 00:00:00 2001
From: Anton Sarukhanov <code@ant.sr>
Date: Thu, 28 Sep 2023 15:52:37 -0400
Subject: [PATCH] FTLive added recaptcha; can no longer get a list of
tournaments.
---
app.py | 33 +++++++++++++++----------
requirements.txt | 4 ++--
scraper.py | 56 ++++++++++++++++---------------------------
static/css/style.css | 57 +++++++++++++++++++++++++++++++-------------
templates/error.html | 23 ++++++++++++++++++
templates/index.html | 47 +++++++++++++-----------------------
templates/live.html | 3 +++
7 files changed, 128 insertions(+), 95 deletions(-)
create mode 100644 templates/error.html
diff --git a/app.py b/app.py
index d476316..527c5ed 100644
--- a/app.py
+++ b/app.py
@@ -2,7 +2,7 @@
from flask import Flask, render_template, redirect, request, url_for
from flask_caching import Cache
-from scraper import FTPScraper, FTLiveScraper
+from scraper import FTPScraper, FTLiveScraper, ScrapeError
import models
# pylint: disable=invalid-name ; These module-level variables are standard for Flask.
@@ -13,9 +13,18 @@ DISPLAY_DATETIME_FORMAT = '%A, %B %d, %Y at %-I:%M %p'
DISPLAY_DATE_FORMAT = '%A, %B %-d, %Y'
DISPLAY_TIME_FORMAT = '%-I:%M %p on %A'
+DISCLAIMER_TEXT = ("Not affiliated with Fencing Time or US Fencing."
+ " Information accuracy and availability is not guaranteed."
+ " Use at your own risk.")
+
+FTLIVE_BASE_URL = 'https://fencingtimelive.com/'
+
app.config['APPLICATION_ROOT'] = '/armory'
+app.add_template_global(name='DISCLAIMER_TEXT', f=DISCLAIMER_TEXT)
+
+
def _make_cache_key():
"""Create a cache key for Flask-Caching."""
path = request.path
@@ -42,9 +51,7 @@ def _jinja2_filter_datetime(datetime, date=True, time=True):
@cache.cached(timeout=300)
def index():
"""Render the app landing page."""
- ftl_scraper = FTLiveScraper()
- return render_template('index.html',
- tournaments=ftl_scraper.list_tournaments())
+ return render_template('index.html')
@app.route("/live")
@@ -52,14 +59,16 @@ def index():
def live():
"""Render the primary view of live tournament stats."""
results_url = request.args.get('results_url')
- ftl_id = request.args.get('ftl_id')
-
- if ftl_id:
- tournament = FTLiveScraper().scrape_tournament(tournament_id=ftl_id)
- elif results_url:
- tournament = FTPScraper(results_url).scrape_tournament()
- else:
- return redirect(url_for('index'))
+
+ try:
+ if results_url.startswith(FTLIVE_BASE_URL):
+ tournament = FTLiveScraper().scrape_tournament(results_url)
+ elif results_url:
+ tournament = FTPScraper(results_url).scrape_tournament()
+ else:
+ return redirect(url_for('index'))
+ except ScrapeError as e:
+ return render_template('error.html', error_text=e)
return render_template('live.html', tournament=tournament, phases=models.EventPhase)
diff --git a/requirements.txt b/requirements.txt
index 064cd40..8633d51 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-lxml==4.4.2
+lxml==4.9.3
requests==2.22.0
Flask==2.1.0
-Flask-Caching==2.0.1
+Flask-Caching==2.0.2
diff --git a/scraper.py b/scraper.py
index d44d9bb..4b81c89 100644
--- a/scraper.py
+++ b/scraper.py
@@ -2,10 +2,9 @@
import asyncio
from concurrent.futures import ThreadPoolExecutor
-from datetime import date, datetime, timedelta
-from urllib.parse import urlparse, urljoin, urlencode
+from datetime import datetime, timedelta
+from urllib.parse import urlparse, urljoin
from lxml import html # nosec ; Bandit suggests defusedxml but defusedxml.lxml is dead
-from json.decoder import JSONDecodeError
import requests
from models import Event, EventPhase, Fencer, Tournament
@@ -38,13 +37,18 @@ class FTPScraper(Scraper):
def scrape_tournament(self):
"""Get all tournament information."""
try:
- results = requests.get(self.tournament_url)
- except requests.exceptions.MissingSchema:
- results = requests.get("http://{}".format(self.tournament_url))
+ try:
+ results = requests.get(self.tournament_url)
+ except requests.exceptions.MissingSchema:
+ results = requests.get("https://{}".format(self.tournament_url))
+ except requests.exceptions.ConnectionError:
+ raise ScrapeError("Can't read the tournament link. "
+ "Please check the link and try again. "
+ "You entered: {}".format(self.tournament_url))
tournament_etree = html.fromstring(results.content)
try:
tournament_name = tournament_etree.xpath(
- '//span[@class="tournName"]/text()')[0]
+ '//div[@class="desktop tournName"]/text()')[0]
updated_str = (tournament_etree.xpath(
'//span[@class="lastUpdate"]/text()')[0]
.replace('Last Updated:', '').strip())
@@ -135,41 +139,23 @@ class FTLiveScraper(Scraper):
MAX_AGO = timedelta(days=7)
MAX_AHEAD = timedelta(days=7)
- def list_tournaments(self, search=None, from_date=None, to_date=None):
- """Get a list of tournaments in FTLive."""
- if not from_date:
- from_date = date.today() - self.MAX_AGO
- if not to_date:
- to_date = date.today() + self.MAX_AHEAD
- args = {
- 'tname': search or '',
- 'from': from_date or '',
- 'to': to_date or ''
- }
- url = self.TOURNAMENTS_URL.format(query=urlencode(args))
- try:
- tournaments = requests.get(url).json()
- except JSONDecodeError:
- raise ScrapeError("Failed to decode tournament list from URL {url}"
- .format(url=url))
- return [{'start': datetime.strptime(t['start'], self.START_FORMAT),
- 'id': t['id'],
- 'name': t['name'],
- 'location': t['location']}
- for t in tournaments]
-
- def scrape_tournament(self, tournament_id):
+ def scrape_tournament(self, tournament_url):
"""Get all tournament information."""
- tournament_url = self.TOURNAMENT_URL.format(tournament_id=tournament_id)
- tournament_html = requests.get(tournament_url).content
+ try:
+ tournament_html = requests.get(tournament_url).content
+ except requests.exceptions.ConnectionError:
+ raise ScrapeError("Can't read the tournament link."
+ "Please check the link and try again."
+ "You entered: {}".format(tournament_url))
tournament_etree = html.fromstring(tournament_html)
try:
tournament_name = tournament_etree.xpath(
'//div[@class="desktop tournName"]/text()')[0]
except IndexError:
- raise ScrapeError("Tournament info not found.")
+ raise ScrapeError("Tournament info not found. Please check the link."
+ " You entered: {}".format(tournament_url))
self.tournament = Tournament(name=tournament_name, url=tournament_url,
- ftl_id=tournament_id)
+ updated=datetime.now())
event_data = tournament_etree.xpath(
"//tr[re:test(@id, 'ev_.*')]",
diff --git a/static/css/style.css b/static/css/style.css
index 5c55384..ab65655 100644
--- a/static/css/style.css
+++ b/static/css/style.css
@@ -4,9 +4,32 @@ body {
background-color: #000;
max-width: 100%;
}
+body > header > * {
+ font-size: 2em;
+ font-weight: 100;
+ text-align: center;
+}
h1 {
font-size: 1.2em;
}
+dt {
+ margin: 0 0 .5em;
+ font-weight: bold;
+}
+dd {
+ margin: 0 0 .5em 1em;
+}
+code {
+ border: dashed 1px #999;
+ padding: .1em;
+ border-radius: .25em;
+}
+body p.error_detail {
+ font-family: monospace;
+ border: dashed 1px #999;
+ padding: .5em;
+ border-radius: .25em;
+}
div.name,
div.time {
margin: .5em 0;
@@ -17,7 +40,7 @@ div.name {
section {
border-radius: 1em;
border: 2px solid #555;
- margin: .5em;
+ margin: .5em 0;
padding: .75em;
flex: 15em 1 0;
}
@@ -114,27 +137,29 @@ body.page-live main > section {
body.page-live p, ul {
margin: .25em 0;
}
-body.page-index form {
- display: flex;
- flex-wrap: wrap;
-}
-body.page-index form > * {
- flex: auto 1 0;
+body.page-index form {
+ width: 100%;
+ text-align: center;
}
-body.page-index label {
- display: flex;
- flex-wrap: wrap;
+body.page-index form input {
+ font-size: 1em;
+ padding: .5em .5em;
+ width: 40em;
+ max-width: 95%;
+ margin: .5rem;
}
-body.page-index input,
-body.page-index select {
- flex: 15em 1 0;
-}
-body.page-index input[type=submit] {
- flex: 6em 0 0;
+body.page-index form input[type=submit] {
+ font-weight: bold;
+ width: auto;
}
/* Footer */
+p.disclaimer {
+ text-align: center;
+ font-size: .85em;
+ color: #aaa;
+}
div.updated {
text-align: center;
margin: .5em 0;
diff --git a/templates/error.html b/templates/error.html
new file mode 100644
index 0000000..ec1dda0
--- /dev/null
+++ b/templates/error.html
@@ -0,0 +1,23 @@
+{% extends "base.html" %}
+{% block content %}
+ <header>
+ <h1>Oops!</h1>
+ </header>
+ <main>
+ <section>
+ <p>
+ Something went wrong. {% if error_text %}Here are the details:{% endif %}
+ </p>
+ {% if error_text %}
+ <p class="error_detail">
+ {{ error_text }}
+ </p>
+ {% endif %}
+ </section>
+ </main>
+ <footer>
+ <p class="disclaimer">
+ {{ DISCLAIMER_TEXT }}
+ </p>
+ </footer>
+{% endblock content %}
diff --git a/templates/index.html b/templates/index.html
index 8561459..ab8bcc1 100644
--- a/templates/index.html
+++ b/templates/index.html
@@ -4,37 +4,24 @@
<h1>Armory Dashboard</h1>
</header>
<main>
- <p>
- Welcome! Please select a Live Results URL, or enter your own.
- </p>
- <section>
- <h2>Fencing Time Live</h2>
- <p>
- <form action="{{ url_for('live') }}" method="get">
- <label>Choose a Tournament:
- <select name="ftl_id" onchange="this.form.submit()">
- <option value="" selected disabled>-- Select One --</option>
- {% for tournament in tournaments | sort(attribute='start') %}
- {% if loop.changed(tournament.start) %}</optgroup><optgroup label="{{ tournament.start | strftime(time=False) }}">{% endif %}
- <option value="{{ tournament['id'] }}">{{ tournament['name'] }} ({{ tournament['location'] }})</option>
- {% if loop.last %}</optgroup>{% endif %}
- {% endfor %}
- </select>
- </label>
- <input type="submit" value="Go!">
- </form>
- </p>
- </section>
<section>
- <h2>FTP Live Results</h2>
- <p>
- <form action="{{ url_for('live') }}" method="get">
- <label>Enter a custom Live Results link:
- <input name="results_url" placeholder="example.com/liveresults">
- </label>
- <input type="submit" value="Go!">
- </form>
- </p>
+ <form action="{{ url_for('live') }}" method="get">
+ <label>Please enter the link for the tournament you'd like to view.<br>
+ <input name="results_url" placeholder="e.g. https://fencingtimelive.com/tournaments/eventSchedule/XYZ">
+ </label>
+ <input type="submit" value="Go!">
+ </form>
+ <dl class="shh">
+ <dt>How do I find the tournament link?</dt>
+ <dd>Fencing Time Live: Go to the Fencing Time Live website, search for tournaments, and click on the tournament you are interested in. When you see the Event Schedule, copy the URL from your browser's address bar.</dd>
+ <dd>Fencing Time Live links will follow this format:<br><code><strong>https://fencingtimelive.com/tournaments/eventSchedule/</strong>XYZ123</code></dd>
+ <dd>Events using "FTP Upload" reult pages (not Fencing Time Live): paste the link to the main page of that tournament. FTP links do not follow a standard format.</dd>
+ </dl>
</section>
</main>
+ <footer>
+ <p class="disclaimer">
+ {{ DISCLAIMER_TEXT }}
+ </p>
+ </footer>
{% endblock content %}
diff --git a/templates/live.html b/templates/live.html
index b55e483..bcf623c 100644
--- a/templates/live.html
+++ b/templates/live.html
@@ -45,5 +45,8 @@
{% if tournament.updated %}
<div class="updated">Last updated: <span class="date">{{ tournament.updated | strftime }}</span></div>
{% endif %}
+ <p class="disclaimer">
+ {{ DISCLAIMER_TEXT }}
+ </p>
</footer>
{% endblock content %}
--
GitLab