diff --git a/tools/fetch.py b/tools/fetch.py index 15aa520..adb8b1e 100644 --- a/tools/fetch.py +++ b/tools/fetch.py @@ -72,6 +72,80 @@ def reader_mode(html_content: str) -> str: body = BeautifulSoup(doc.summary(), "html.parser") return title_html + str(body) +def is_twitter_url(url: str) -> bool: + """Check if URL is a Twitter/X link""" + parsed = urlparse(url) + return parsed.netloc in ('x.com', 'www.x.com', 'twitter.com', 'www.twitter.com', 'mobile.twitter.com') + +def clean_twitter_url(url: str) -> str: + """Strip tracking query params from Twitter/X URLs""" + parsed = urlparse(url) + return f"{parsed.scheme}://{parsed.netloc}{parsed.path}" + +def archive_twitter(url: str) -> str: + """Archive Twitter/X post using fxtwitter API proxy""" + try: + parsed = urlparse(url) + # Convert x.com/user/status/123 -> api.fxtwitter.com/user/status/123 + api_url = f"https://api.fxtwitter.com{parsed.path}" + headers = {"User-Agent": "Mozilla/5.0 (ArchiveBot/1.0)"} + + response = requests.get(api_url, timeout=30, headers=headers) + response.raise_for_status() + data = response.json() + + tweet = data.get('tweet', {}) + author = tweet.get('author', {}) + text = tweet.get('text', '') + author_name = author.get('name', '') + screen_name = author.get('screen_name', '') + likes = tweet.get('likes', 0) + retweets = tweet.get('retweets', 0) + replies = tweet.get('replies', 0) + views = tweet.get('views', 0) + created_at = tweet.get('created_at', '') + + # Build clean HTML + html_content = f'

{html.escape(author_name)} (@{html.escape(screen_name)})

\n' + html_content += f'

{html.escape(created_at)}

\n' + + # Tweet text - preserve newlines + escaped_text = html.escape(text).replace('\n', '
\n') + html_content += f'
{escaped_text}
\n' + + # Media (photos and videos) + media = tweet.get('media', {}) + all_media = media.get('all', []) + for item in all_media: + if item.get('type') == 'photo': + img_url = item.get('url', '') + html_content += f'
Tweet image
\n' + elif item.get('type') == 'video' or item.get('type') == 'gif': + thumb = item.get('thumbnail_url', '') + html_content += f'
Tweet video thumbnail

(Video - see original tweet)

\n' + + # Engagement stats + html_content += f'
' + html_content += f'{replies:,} replies • {retweets:,} reposts • {likes:,} likes' + if views: + html_content += f' • {views:,} views' + html_content += '
\n' + + # Quote tweet if present + quote = tweet.get('quote') + if quote: + q_author = quote.get('author', {}) + q_text = html.escape(quote.get('text', '')).replace('\n', '
\n') + html_content += f'
' + html_content += f'{html.escape(q_author.get("name", ""))} (@{html.escape(q_author.get("screen_name", ""))})
' + html_content += f'{q_text}
\n' + + return html_content + + except Exception as e: + print(f"⚠ Twitter API proxy failed ({e}), returning minimal content...") + return f"

Tweet

Failed to fetch tweet content: {html.escape(str(e))}

View original tweet

" + def is_reddit_url(url: str) -> bool: """Check if URL is a Reddit link""" parsed = urlparse(url) @@ -427,8 +501,13 @@ def convert_ihsoyct_to_api_url(url: str) -> str: def archive(url: str, out_dir: pathlib.Path, force: bool): out_dir.mkdir(parents=True, exist_ok=True) + + # Clean Twitter URLs before slugging so filenames match the JS slug + if is_twitter_url(url): + url = clean_twitter_url(url) + fname = out_dir / slug.slug(url) - + # Check if this is a Reddit search tool and convert to API URL original_url = url if is_reddit_search_tool(url): @@ -450,7 +529,42 @@ def archive(url: str, out_dir: pathlib.Path, force: bool): try: archive_date = datetime.datetime.now(datetime.timezone.utc) - if is_arctic_shift_api(url): + if is_twitter_url(original_url): + content = archive_twitter(original_url) + archive_style = """ + + """ + final_content = ( + "\n" + + "\n" + + archive_style + "\n" + + generate_archive_header(original_url, archive_date) + "\n" + + content + ) + elif is_arctic_shift_api(url): content = archive_arctic_shift_api(url) # Enhanced styling with archive header for HTML archive_style = """