{ "submission_id": "3517775", "keywords": [ { "keyword_id": "14319", "keyword_name": "code", "contributed": "f", "submissions_count": "133" }, { "keyword_id": "569", "keyword_name": "inkbunny", "contributed": "f", "submissions_count": "1269" }, { "keyword_id": "24188", "keyword_name": "program", "contributed": "f", "submissions_count": "226" }, { "keyword_id": "11078", "keyword_name": "python", "contributed": "f", "submissions_count": "1025" }, { "keyword_id": "6695", "keyword_name": "utility", "contributed": "f", "submissions_count": "21" } ], "hidden": "f", "scraps": "f", "favorite": "f", "favorites_count": "1", "create_datetime": "2025-01-05 13:49:51.091245+00", "create_datetime_usertime": "05 Jan 2025 14:49 CET", "last_file_update_datetime": "2025-01-05 13:35:19.638684+00", "last_file_update_datetime_usertime": "05 Jan 2025 14:35 CET", "username": "JustLurking", "user_id": "11985", "user_icon_file_name": "96044_JustLurking_final_spiral2.gif", "user_icon_url_large": "https://nl1.ib.metapix.net/usericons/large/96/96044_JustLurking_final_spiral2.gif", "user_icon_url_medium": "https://nl1.ib.metapix.net/usericons/medium/96/96044_JustLurking_final_spiral2.gif", "user_icon_url_small": "https://nl1.ib.metapix.net/usericons/small/96/96044_JustLurking_final_spiral2.gif", "file_name": "5376851_JustLurking_ib-maildir-fetch.txt", "file_url_full": "https://nl1.ib.metapix.net/files/full/5376/5376851_JustLurking_ib-maildir-fetch.txt", "file_url_screen": "https://nl1.ib.metapix.net/files/screen/5376/5376851_JustLurking_ib-maildir-fetch.txt", "file_url_preview": "https://nl1.ib.metapix.net/files/preview/5376/5376851_JustLurking_ib-maildir-fetch.txt", "thumbnail_url_huge": "https://nl1.ib.metapix.net/thumbnails/huge/5376/5376851_JustLurking_ib-maildir-fetch.jpg", "thumbnail_url_large": "https://nl1.ib.metapix.net/thumbnails/large/5376/5376851_JustLurking_ib-maildir-fetch.jpg", "thumbnail_url_medium": "https://nl1.ib.metapix.net/thumbnails/medium/5376/5376851_JustLurking_ib-maildir-fetch.jpg", "thumb_huge_x": "100", "thumb_huge_y": "100", "thumb_large_x": "100", "thumb_large_y": "100", "thumb_medium_x": "100", "thumb_medium_y": "100", "files": [ { "file_id": "5376851", "file_name": "5376851_JustLurking_ib-maildir-fetch.txt", "file_url_full": "https://nl1.ib.metapix.net/files/full/5376/5376851_JustLurking_ib-maildir-fetch.txt", "file_url_screen": "https://nl1.ib.metapix.net/files/screen/5376/5376851_JustLurking_ib-maildir-fetch.txt", "file_url_preview": "https://nl1.ib.metapix.net/files/preview/5376/5376851_JustLurking_ib-maildir-fetch.txt", "mimetype": "text/plain", "submission_id": "3517775", "user_id": "11985", "submission_file_order": "0", "full_size_x": null, "full_size_y": null, "screen_size_x": null, "screen_size_y": null, "preview_size_x": null, "preview_size_y": null, "initial_file_md5": "a9703eac55eae903380b80fcd32098fc", "full_file_md5": "a9703eac55eae903380b80fcd32098fc", "large_file_md5": "", "small_file_md5": "", "thumbnail_md5": "40f9f166c8797ff7bdfbf6d99c84acd7", "deleted": "f", "create_datetime": "2025-01-05 13:35:19.638684+00", "create_datetime_usertime": "05 Jan 2025 14:35 CET", "thumbnail_url_huge": "https://nl1.ib.metapix.net/thumbnails/huge/5376/5376851_JustLurking_ib-maildir-fetch.jpg", "thumbnail_url_large": "https://nl1.ib.metapix.net/thumbnails/large/5376/5376851_JustLurking_ib-maildir-fetch.jpg", "thumbnail_url_medium": "https://nl1.ib.metapix.net/thumbnails/medium/5376/5376851_JustLurking_ib-maildir-fetch.jpg", "thumb_huge_x": "100", "thumb_huge_y": "100", "thumb_large_x": "100", "thumb_large_y": "100", "thumb_medium_x": "100", "thumb_medium_y": "100" } ], "pools": [ { "pool_id": "3115", "name": "Everything in Order of Creation", "description": "Everything in my gallery in the order it was created.", "count": "70", "submission_left_submission_id": "3517765", "submission_left_file_name": "5376925_JustLurking_ib-status.txt", "submission_left_thumbnail_url_huge": "https://nl1.ib.metapix.net/thumbnails/huge/5376/5376925_JustLurking_ib-status.jpg", "submission_left_thumbnail_url_large": "https://nl1.ib.metapix.net/thumbnails/large/5376/5376925_JustLurking_ib-status.jpg", "submission_left_thumbnail_url_medium": "https://nl1.ib.metapix.net/thumbnails/medium/5376/5376925_JustLurking_ib-status.jpg", "submission_left_thumb_huge_x": "100", "submission_left_thumb_huge_y": "100", "submission_left_thumb_large_x": "100", "submission_left_thumb_large_y": "100", "submission_left_thumb_medium_x": "100", "submission_left_thumb_medium_y": "100", "submission_right_submission_id": "3606528", "submission_right_file_name": "5543647_JustLurking_vr_school_takeover.png", "submission_right_thumbnail_url_huge_noncustom": "https://nl1.ib.metapix.net/files/preview/5543/5543647_JustLurking_vr_school_takeover.jpg", "submission_right_thumbnail_url_large_noncustom": "https://nl1.ib.metapix.net/thumbnails/large/5543/5543647_JustLurking_vr_school_takeover_noncustom.jpg", "submission_right_thumbnail_url_medium_noncustom": "https://nl1.ib.metapix.net/thumbnails/medium/5543/5543647_JustLurking_vr_school_takeover_noncustom.jpg", "submission_right_thumb_medium_noncustom_x": "90", "submission_right_thumb_medium_noncustom_y": "120", "submission_right_thumb_large_noncustom_x": "150", "submission_right_thumb_large_noncustom_y": "200", "submission_right_thumb_huge_noncustom_x": "225", "submission_right_thumb_huge_noncustom_y": "300" }, { "pool_id": "98445", "name": "Inkbunny Maildir Fetch", "description": "Software for fetching messages from Inkbunny and storing them locally in maildir format.", "count": "1" } ], "description": "A program which downloads new messages from Inkbunny and backs them up into a maildir on your local system. The script attempts to to be frugal with its requests and only download new data from the site (so it doesn't have to trawl your entire inbox and sent items every time it is run). It also attempts to convert Inkbunny's human-friendly dates back into time stamps, maintain threading information and undo BBCode rendering (the extracted, unmodified HTML is saved as the first alternative).\n\nBe careful what automation you setup with this. If you fail to add generous sleeps/delays/waits between requests you will only piss off the admins and no one wants that.\n\nAlso be aware that: [b]unless told otherwise, this script will treat the current directory as the destination maildir[/b]. If this is your home or documents directory [b]you almost certainly do not want this[/b]. Create a new directory for this script and change to it before running this program. [b]You have been warned[/b].\n\nThe intended usage is something like:\n\n[q]export PHPSESSID='my-session-id'\nmkdir -p ~/inkbuny-messages/\ncd ~/inkbunny-messages\nib-maildir-fetch[/q]\n\nMy Inkbunny Status Reporter script can be used in conjunction with this script to automatically download new messages (but be careful not to flood the site with requests in a tight loop; no one will appreciate that, the site admins least of all.)\n\nThe script doesn't do any login of its own because I felt trying to automate the login process would not endear me to the site admins. I trust that users who want to run this script will know how to extract the session cookie from their browser.\n\nThere is no official API for fetching messages on inkbunny so this script may break in the future as the site is updated. There's not much that can be done about that.\n\n[name]GreenReaper[/name], [name]Salmy[/name], [name]keito[/name], I believe these scripts fit in with the spirit of the other scripts found in the [url=Apps, Scripts and Mods]Apps, Scripts and Mods[/url] page and don't violate the ToS, please remove them if you disagree. I will be happy to make any modifications you require to these scripts if there is something in them that concerns you.\n\n[center][i]This program is uploaded in the hopes that it may be useful to others. No guarantee is made that this program is correct and free from faults. Never run code on your system that you haven't first read and understood.[/i][/center]", "description_bbcode_parsed": "A program which downloads new messages from Inkbunny and backs them up into a maildir on your local system.  The script attempts to to be frugal with its requests and only download new data from the site (so it doesn't have to trawl your entire inbox and sent items every time it is run).  It also attempts to convert Inkbunny's human-friendly dates back into time stamps, maintain threading information and undo BBCode rendering (the extracted, unmodified HTML is saved as the first alternative).

Be careful what automation you setup with this.  If you fail to add generous sleeps/delays/waits between requests you will only piss off the admins and no one wants that.

Also be aware that: unless told otherwise, this script will treat the current directory as the destination maildir.  If this is your home or documents directory you almost certainly do not want this.  Create a new directory for this script and change to it before running this program.  You have been warned.

The intended usage is something like:

\n\t\t\t\t\t
\n\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t
"\n\t\t\t\t\t\t\t\t\texport PHPSESSID='my-session-id'
mkdir -p ~/inkbuny-messages/
cd ~/inkbunny-messages
ib-maildir-fetch\n\t\t\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t\t

My Inkbunny Status Reporter script can be used in conjunction with this script to automatically download new messages (but be careful not to flood the site with requests in a tight loop; no one will appreciate that, the site admins least of all.)

The script doesn't do any login of its own because I felt trying to automate the login process would not endear me to the site admins.  I trust that users who want to run this script will know how to extract the session cookie from their browser.

There is no official API for fetching messages on inkbunny so this script may break in the future as the site is updated.  There's not much that can be done about that.

GreenReaper, Salmy, keito, I believe these scripts fit in with the spirit of the other scripts found in the Apps, Scripts and Mods page and don't violate the ToS, please remove them if you disagree.  I will be happy to make any modifications you require to these scripts if there is something in them that concerns you.

This program is uploaded in the hopes that it may be useful to others.  No guarantee is made that this program is correct and free from faults.  Never run code on your system that you haven't first read and understood.
", "writing": "[code]#!/usr/bin/python3\n\n# Inkbunny Maildir Fetch 0.1.0\n# Copyright 2025 JustLurking\n\n# This program is free software: you can redistribute it and/or modify it\n# under the terms of the GNU General Public License as published by the\n# Free Software Foundation, either version 3 of the License, or (at your\n# option) any later version.\n#\n# This program is distributed in the hope that it will be useful, but\n# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY\n# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License\n# for more details.\n#\n# You should have received a copy of the GNU General Public License along\n# with this program. If not, see .\n\n# About\n# This program will connect to Inkbunny using the session cookie in the\n# PHPSESSID environment variable then download the messages in the user's\n# inbox and sent items, storing them as emails in maildir format in the\n# current working directory.\n#\n# As the program treats the current working directory as a maildir mailbox\n# it is important to change to the correct directory before running it.\n#\n# Only new items will be downloaded and saved.\n#\n# The program does its best to convert the HTML of a message back to BBCode,\n# but this is an imperfect conversion and the original HTML is saved also\n# so that either maybe used.\n#\n# Similarly, it is not possible to convert timestamps such as `a moment ago`\n# or `1 hrs, 20 mins ago` to the correct dates and times with full accuracy\n# the program does its best in those cases but some messages may be saved\n# with dates different to those shown on the site.\n\n# Changelog\n# 2025-01-03 JustLurking: Initial Release.\n\nimport argparse\nimport bs4\nimport dataclasses\nimport email\nimport logging\nimport mailbox\nimport os\nimport re\nimport requests\nimport time\nimport urllib.parse\n\n# Variables used throughout the program.\n\n# Used for identifying the program.\nprogram_file = \"ib-maildir-fetch\"\nprogram_name = \"Inkbunny Maildir Fetch\"\nversion = \"0.1.0\"\nbug_reports = \"https://Inkbunny.net/JustLurking/\"\nhomepage = \"https://inkbunny.net/submissionsviewall.php?mode=pool&pool_id=98445\"\n\n# Used to download pages.\nbase_url = \"https://inkbunny.net/privatemessages.php\"\ncookie = None\ncookies = requests.cookies.RequestsCookieJar()\n\n# Used for rate limiting.\nrequest_pause = 0.25\nlast_request = 0\n\n# Used to hold site details.\nuser = None\ndownloaded = set()\nmaildir = None\ninbox = None\nsent = None\nlatest_sent = 0\nlatest_inbox = 0\nearly_quit = True\n\n# Used for logging.\nlog = logging.getLogger(__name__)\n\n# Variables used when converting HTML to BBCode.\n\n# These replacements are all simple substitutions.\nsimple_replacements = {\n \"b\": ((\"strong\",), {}),\n \"center\": ((\"div\",), {\"class_\": \"align_center\"}),\n \"i\": ((\"em\",), {}),\n \"left\": ((\"div\",), {\"class_\": \"align_left\"}),\n \"right\": ((\"div\",), {\"class_\": \"align_right\"}),\n \"s\": ((\"span\",), {\"class_\": \"strikethrough\"}),\n \"t\": ((\"span\",), {\"class_\": \"font_title\"}),\n \"u\": ((\"span\",), {\"class_\": \"underline\"})\n}\n\n# Map titles back to tag names.\noffsite_mapping = {\n \"deviantART\": \"da\",\n \"Fur Affinity\": \"fa\",\n \"SoFurry\": \"sf\",\n \"Weasyl\": \"w\"\n}\n\n# Regular expressions.\nre_offsite_title = re.compile(\" on (deviantART|Fur Affinity|SoFurry|Weasyl)$\")\nre_usericon = re.compile(\"^https://inkbunny.net/usericons/\")\nre_color = re.compile(\"^color: [^;]*;$\")\nre_size = re.compile(\"/(small|medium|large|huge)/\")\nre_pool_url = re.compile(\"^/poolview_process.php\\\\?pool_id=\")\nre_bare = re.compile(\"\\\\[(b|center|i|left|right|s|t|u|q|smallpool|mediumpool|smallthumb|mediumthumb|largethumb|hugethumb|color|icon|iconname|name|da|fa|sf|w|url)(=[^]]*)?]|da!|fa!|sf!|w!\")\nre_date = re.compile(\"^((\\\\d+) hrs?,? )?((\\\\d+) mins?,? )?((\\\\d+) secs? )?ago$\")\n\n# Classes and functions.\n\n[at_iconname]dataclasses[/at_iconname].dataclass\nclass MessageMetadata:\n \"\"\"\n Class that holds the metadata identifying a specific Inkbunny message.\n \"\"\"\n msg_id: int\n sender: str\n receiver: str\n subject: str\n date: str\n in_reply_to: str\n\n def download_messages_in_thread(self):\n \"\"\"\n Fetch the thread for this message if it has not been downloaded yet\n and extracts all messages from it, saving them to the correct folder\n if they haven't been downloaded already.\n \"\"\"\n global user, downloaded\n\n if int(self.msg_id) in downloaded:\n log.debug(\"Ignoring thread for message %d, message already saved.\", self.msg_id)\n return\n\n log.debug(\"Fetching thread for message %d.\", self.msg_id)\n\n soup = download(\n \"https://inkbunny.net/privatemessageview.php\",\n private_message_id=str(self.msg_id)\n )\n get_logged_in_user(soup)\n other_user = self.sender if self.sender != user else self.receiver\n\n for msg in read_thread(soup, other_user):\n msg_id = int(msg[\"Message-Id\"])\n if msg_id in downloaded:\n log.debug(\"Ignoring message %d in thread %d, message already saved.\", msg_id, self.msg_id)\n continue\n\n if msg[\"To\"] == user:\n log.debug(\"Saving message %d in thread %d to inbox.\", msg_id, self.msg_id)\n inbox.add(msg)\n else:\n log.debug(\"Saving message %d in thread %d to sent.\", msg_id, self.msg_id)\n sent.add(msg)\n downloaded.add(msg_id)\n\n def create_message(self):\n \"\"\"\n Create a new multi-part email.message.EmailMessage with the headers\n set from this metadata object. Does not add a body, that is up to\n the caller.\n \"\"\"\n msg = email.message.EmailMessage()\n msg[\"Message-Id\"] = str(self.msg_id)\n msg[\"From\"] = self.sender\n msg[\"To\"] = self.receiver\n msg[\"Date\"] = self.date\n msg[\"Subject\"] = self.subject\n if self.in_reply_to is not None:\n msg[\"In-Reply-To\"] = str(self.in_reply_to)\n msg.make_alternative()\n return msg\n\ndef download(url, **kwargs):\n \"\"\"\n Utility wrapper for boiler plate around downloading and parsing pages.\n Also enforces the rate limit.\n \"\"\"\n global cookies, last_request, request_pause\n now = time.time()\n if now - last_request < request_pause:\n time.sleep(request_pause - now + last_request)\n resp = requests.get(url, cookies=cookies, params=kwargs)\n for (i, step) in enumerate(resp.history):\n log.debug(\"[%d] Downloaded: %s\", i, step.url)\n log.debug(\"[Final] Downloaded: %s\", resp.url)\n resp.raise_for_status()\n last_request = time.time()\n return bs4.BeautifulSoup(resp.content, features=\"lxml\")\n\ndef parse_date(text):\n \"\"\"\n Convert a human-readable time to something that can be stored in an\n email header.\n \"\"\"\n global re_date, last_request\n\n to_subtract = None\n if text == \"a moment ago\":\n to_subtract = 0\n\n m = re_date.match(text)\n if m is not None:\n to_subtract = 0\n if m.group(2) is not None:\n to_subtract += int(m.group(2)) * 60 * 60\n if m.group(4) is not None:\n to_subtract += int(m.group(4)) * 60\n if m.group(6) is not None:\n to_subtract += int(m.group(6))\n\n if to_subtract is not None:\n result = time.strftime(\"%a, %d %b %Y %H:%M:%S +0000\", time.gmtime(last_request - to_subtract))\n return result\n\n return text\n\ndef get_logged_in_user(tag):\n \"\"\"\n Sets the logged-in user name if it has not been set yet using the\n supplied HTML.\n \"\"\"\n global user\n if user is not None:\n return\n nav = tag.find(class_=\"userdetailsnavigation\")\n if nav is None:\n log.critical(\"Unable to find user details on page.\")\n exit(1)\n widget = nav.find(class_=\"widget_userNameSmall\")\n if widget is None:\n log.critical(\"Unable to find user details on page.\")\n exit(1)\n user = widget.get_text().strip()\n log.info(\"Logged in as %s.\", user)\n\ndef get_next_page(tag):\n \"\"\"\n Returns the URL of the first next-page link in the supplied HTML.\n \"\"\"\n next_page_link = tag.find(\"a\", title=\"next page\")\n if next_page_link is None:\n return None\n return next_page_link[\"href\"]\n\ndef read_box_page(tag, in_inbox):\n \"\"\"\n A generator which extracts rows from the index view of a box.\n If is_inbox is True operates over the inbox, otherwise it operates\n over the sent items.\n \"\"\"\n global user\n get_logged_in_user(tag)\n\n # Set the variables for the box we will be reading.\n min_columns = 5\n user_column = 1\n subject_column = 3\n date_column = 4\n\n if in_inbox:\n min_columns = 6\n user_column = 2\n subject_column = 4\n date_column = 5\n\n # Read the rows from this page of the index.\n for row in tag.find_all(id=re.compile(\"^m_\\\\d+$\")):\n columns = [*row.find_all(\"td\", recursive=False)]\n if len(columns) < min_columns:\n # Malformed row? Skip it.\n continue\n\n other_user = columns[user_column].get_text().strip()\n subject = columns[subject_column].get_text().strip()\n date = parse_date(columns[date_column].get_text().strip())\n msg_id = int(row[\"id\"][2:])\n\n if in_inbox:\n yield MessageMetadata(msg_id, other_user, user, subject, date, None)\n else:\n yield MessageMetadata(msg_id, user, other_user, subject, date, None)\n\ndef read_box(is_inbox):\n \"\"\"\n A generator which extracts rows from a box.\n If is_inbox is True operates over the inbox, otherwise it operates\n over the sent items.\n \"\"\"\n global latest_inbox, latest_sent, early_quit\n\n # Set the variables for the box we will be reading.\n latest = latest_inbox if is_inbox else latest_sent\n end_of_new_messages = False\n\n next_page = \"https://inkbunny.net/privatemessages_process.php?mode=\"\n if is_inbox:\n log.info(\"Reading Inbox.\")\n next_page += \"inbox\"\n else:\n log.info(\"Reading Sent.\")\n next_page += \"sent\"\n\n # Download and parse the next page of the index.\n while next_page is not None:\n next_page = urllib.parse.urljoin(base_url, next_page)\n soup = download(next_page)\n\n # Read the rows in the index and yield them to the caller.\n for row in read_box_page(soup, is_inbox):\n if row.msg_id <= latest:\n end_of_new_messages = True\n yield row\n\n # Exit the loop early if we've encountered a message on this page\n # older than one we've already downloaded.\n if early_quit and end_of_new_messages:\n log.info(\"End of new messages in this box.\")\n break\n\n next_page = get_next_page(soup)\n\ndef unparse_html_to_bbcode(tag):\n \"\"\"\n Given some HTML generated by Inkbunny's BBCode renderer attempt to return\n the BBCode which might have generated it. Since the transformation is\n non-injective it's impossible to reverse with 100% accuracy.\n White-space will likely not be preserved either.\n \"\"\"\n global simple_replacements, offsite_mapping, re_offsite_title, re_usericon, re_color, re_size, re_pool_url, re_bare\n\n # [code]\n for string in tag.find_all(string=re_bare):\n string.replace_with(re_bare.sub(lambda m: \"[code]\"+m.group(0)+\"[​/code]\", string.string))\n\n # [q] and [q=someone]\n for quote in tag.find_all(class_=\"bbcode_quote\"):\n author = quote.find(class_=\"bbcode_quote_author\")\n body = quote.find(class_=\"bbcode_quote_quote\").extract()\n argument = \"\"\n if author is not None:\n argument = \"=\" + author.get_text().strip()[:-7]\n quote.insert_before(bs4.NavigableString(\"[q\"+argument+\"]\"))\n quote.insert_before(body)\n\n children = (*body.contents,)\n if len(children) == 1:\n if isinstance(children[0], bs4.NavigableString):\n children[0].replace_with(children[0].string.strip())\n elif len(children) > 1:\n if isinstance(children[0], bs4.NavigableString):\n children[0].replace_with(children[0].string.lstrip())\n if isinstance(children[-1], bs4.NavigableString):\n children[-1].replace_with(children[-1].string.rstrip())\n\n body.unwrap()\n quote.insert_after(bs4.NavigableString(\"[/q]\"))\n quote.extract()\n\n # [smallpool], [mediumpool]\n for pool in tag.find_all(class_=\"widget_imageFromSubmission\"):\n table = pool.find_parent(\"table\")\n if table is None:\n continue\n table = table.find_parent(\"table\")\n if table is None or table.parent is None:\n # There are three divs with widget_imageFromSubmissions as their\n # class per pool table, so it is possible we've already processed\n # this pool.\n # This also filters out thumbnails which get processed after.\n continue\n size = re_size.search(pool.find(\"img\")[\"src\"]).group(1)\n pool_id = table.find(\"a\", href=re_pool_url)[\"href\"][31:]\n table.replace_with(bs4.NavigableString(\"[\"+size+\"pool]\"+pool_id+\"[/\"+size+\"pool]\"))\n\n # [smallthumb], [mediumthumb], [largethumb], [hugethumb]\n for thumb in tag.find_all(class_=\"widget_imageFromSubmission\"):\n table = thumb.find_parent(\"table\")\n size = re_size.search(thumb.find(\"img\")[\"src\"]).group(1)\n submission_id = thumb.find(\"a\")[\"href\"][3:]\n table.replace_with(bs4.NavigableString(\"[\"+size+\"thumb]\"+submission_id+\"[/\"+size+\"thumb]\"))\n\n # [color]\n for color in tag.find_all(\"span\", style=re_color):\n color.insert_before(bs4.NavigableString(\"[color=\"+color[\"style\"][7:-1]+\"]\"))\n color.insert_after(bs4.NavigableString(\"[/color]\"))\n color.unwrap()\n\n # Newlines\n for br in tag.find_all(\"br\"):\n br.replace_with(bs4.NavigableString(\"\\n\"))\n\n # [icon], [iconname]\n for icon in tag.find_all(\"img\", src=re_usericon):\n table = icon.find_parent(\"table\")\n link = icon.find_parent(\"a\")\n name = link[\"href\"][21:]\n if len((*table.find_all(\"a\"),)) > 1:\n table.replace_with(\"[iconname]\"+name+\"[/iconname]\")\n else:\n table.replace_with(\"[icon]\"+name+\"[/icon]\")\n\n # [name]\n for namelink in tag.find_all(\"span\", class_=\"widget_userNameSmall\"):\n name = namelink.find(\"a\")[\"href\"][1:]\n namelink.replace_with(\"[name]\"+name+\"[/name]\")\n\n # [da], [fa], [sf], [w]\n for link in tag.find_all(\"a\", title=re_offsite_title):\n # Each off-site link generates two sibling elements in the output, we\n # can replace either, but we should remove the one we don't replace.\n if link.find(\"img\") is None:\n # Replace the textual link.\n m = re_offsite_title.search(link[\"title\"])\n site = offsite_mapping[m.group(1)]\n link.replace_with(\"[\"+site+\"]\"+link.get_text().strip()+\"[/\"+site+\"]\")\n else:\n # Remove the image link.\n link.extract()\n\n # [url]\n for a in tag.find_all(\"a\"):\n a.insert_before(bs4.NavigableString(\"[url=\"+a[\"href\"]+\"]\"))\n a.insert_after(bs4.NavigableString(\"[/url]\"))\n a.unwrap()\n\n # simple replacements\n for (name, args) in simple_replacements.items():\n for target in tag.find_all(*args[0], **args[1]):\n target.insert_before(bs4.NavigableString(\"[\"+name+\"]\"))\n target.insert_after(bs4.NavigableString(\"[/\"+name+\"]\"))\n target.unwrap()\n\n # combine all text nodes and return the string representation of the child\n # nodes.\n tag.smooth()\n return \"\".join(str(n) for n in tag.contents)\n\ndef read_thread(tag, other_user):\n \"\"\"\n A generator which extracts messages from thread view.\n \"\"\"\n global user, messages\n previous_subject = None\n previous_msg_id = None\n\n got_result = False\n for elem in tag.find_all(id=re.compile(\"^irt_message_\\\\d+$\")):\n # Extract the next message's data from the page.\n msg_id = int(elem[\"id\"][12:])\n\n children = [e for e in elem.children if not isinstance(e, bs4.NavigableString)]\n\n if len(children) < 5:\n # Malformed\n continue\n\n date_field = children[2].find(True)\n body_span = children[2].find(\"span\", style=\"word-wrap: break-word;\")\n\n if date_field is None or body_span is None:\n # Malformed\n continue\n\n subject_div = children[2].find(style=\"margin-bottom: 5px;\")\n left_link = children[0].find(class_=\"widget_userNameSmall\")\n right_link = children[4].find(class_=\"widget_userNameSmall\")\n\n date = parse_date(date_field.get_text().strip())\n subject = subject_div.get_text().strip() if subject_div is not None else previous_subject\n in_reply_to = previous_msg_id\n sender = None\n\n if left_link is not None:\n sender = left_link.get_text().strip()\n elif right_link is not None:\n sender = right_link.get_text().strip()\n else:\n continue\n\n receiver = other_user if sender == user else user\n\n # Create, populate and yield an EmailMessage for this message.\n content = bs4.BeautifulSoup(\"<body/>\", features=\"lxml\")\n content.title.string = subject\n content.body.append(body_span)\n body_span.unwrap()\n\n metadata = MessageMetadata(msg_id, sender, receiver, subject, date, in_reply_to)\n msg = metadata.create_message()\n msg.add_alternative(\"<!DOCTYPE html>\" + str(content), \"html\")\n msg.add_alternative(str(unparse_html_to_bbcode(content.body)))\n yield msg\n got_result = True\n\n # Update information on the previous message (the one we just yielded)\n # prior to extracting next message.\n if subject is not None:\n previous_subject = subject\n previous_msg_id = msg_id\n\n # This should only be reached if a page other than the one we were\n # expecting is returned, i.e. an error page or similar.\n # Since something has clearly gone wrong just inform the user and abort.\n if not got_result:\n log.critical(\"Failed to parse at least one message in thread. Got: %s\", str(tag))\n exit(1)\n\n# Main Program starts here.\n\n# Configure logging.\nlog.addHandler(logging.StreamHandler())\nlog.setLevel(logging.INFO)\n\n# Handle arguments.\narg_parser = argparse.ArgumentParser(\n prog = program_file,\n description = \"\".join((\n \"Downloads messages from the Inkbunny.net website and stores them\",\n \" as emails in maildir format.\"\n )),\n epilog = \"\".join((\n \"If no DIRECTORY is specified then the current directory is\",\n \" assumed to be the maildir, care should be taken to ensure that\",\n \" the program is run in the correct directory or it may read or\",\n \" write to files the user does not want it to.\\n\",\n \"\\n\",\n \"The SESSION may be passed using the PHPSESSID environment variable.\"\n \" this may be useful if you wish to pass the same value to multiple\"\n \" programs, or if you wish to set the value in your shell's profile.\\n\"\n \"\\n\",\n \"Report bugs to: \"+bug_reports+\"\\n\",\n program_name + \" home page: <\"+homepage+\"\\n\"\n )),\n formatter_class = argparse.RawDescriptionHelpFormatter\n)\narg_parser.add_argument(\n \"-s\",\n \"--session\",\n nargs = 1,\n help = \"the session id to send with requests\"\n)\narg_parser.add_argument(\n \"-C\",\n \"--directory\",\n nargs = 1,\n help = \"change directory when program starts\"\n)\narg_parser.add_argument(\n \"-q\",\n \"--quiet\",\n action = \"count\",\n default = 2,\n help = \"decrease verbosity\"\n)\narg_parser.add_argument(\n \"-v\",\n \"--verbose\",\n action = \"count\",\n default = 0,\n help = \"increase verbosity\"\n)\narg_parser.add_argument(\n \"-f\",\n \"--full-scan\",\n action = \"store_true\",\n help = \"read the full message index even if it seems unnecessary\"\n)\narg_parser.add_argument(\n \"-V\",\n \"--version\",\n action = \"store_true\"\n)\n\nargs = arg_parser.parse_args()\n\nif args.version:\n print(\n \" \".join((program_name, version)),\n \"Copyright (C) 2025 JustLurking<https://inkbunny.net/JustLurking>\",\n \"License GPLv3+: GNU GPL version 3 or later \"+\n \"<https://gnu.org/licenses/gpl.html>\",\n \"\",\n \"This is free software: you are free to change and redistribute it.\",\n \"There is NO WARRANTY, to the extent permitted by law.\",\n sep=\"\\n\"\n )\n exit(0)\n\nlog.setLevel(max(1, min(5, args.quiet-args.verbose))*10)\n\nif args.directory is not None:\n log.info(\"Changing to directory %d.\")\n os.chdir(args.directory)\n\nif args.full_scan:\n early_quit = False\n\nif args.session is not None:\n cookie = args.session\n\n# Open or create Maildir.\nmaildir = mailbox.Maildir(\".\")\ninbox = maildir.add_folder(\"inbox\")\nsent = maildir.add_folder(\"sent\")\n\n# Set the Session Cookie from the environment.\nif cookie is None:\n cookie = os.getenv(\"PHPSESSID\")\n\nif cookie is None:\n log.error(\"Please use the -s argument or set the PHPSESSID environment variable to set the session id.\")\n exit(1)\ncookies.set(\"PHPSESSID\", cookie, domain=\"inkbunny.net\", path=\"/\")\n\n# Make a list of all already downloaded messages.\nfor msg in inbox:\n msg_id = int(msg[\"Message-Id\"])\n downloaded.add(msg_id)\n latest_inbox = max(latest_inbox, msg_id)\n\nfor msg in sent:\n msg_id = int(msg[\"Message-Id\"])\n downloaded.add(msg_id)\n latest_sent = max(latest_sent, msg_id)\n\n# Fetch message indices for inbox and sent and for each message found fetch the\n# appropriate thread view and download all new messages in that thread.\nfor row in read_box(True):\n row.download_messages_in_thread()\n\nfor row in read_box(False):\n row.download_messages_in_thread()\n[/code]", "writing_bbcode_parsed": "<span style='word-wrap: break-word;'>#!/usr/bin/python3<br /><br /># Inkbunny Maildir Fetch 0.1.0<br /># Copyright 2025 JustLurking<br /><br /># This program is free software: you can redistribute it and/or modify it<br /># under the terms of the GNU General Public License as published by the<br /># Free Software Foundation, either version 3 of the License, or (at your<br /># option) any later version.<br />#<br /># This program is distributed in the hope that it will be useful, but<br /># WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY<br /># or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License<br /># for more details.<br />#<br /># You should have received a copy of the GNU General Public License along<br /># with this program. If not, see <<a href=\"https://www.gnu.org/licenses/>\" rel=\"nofollow\">https://www.gnu.org/licenses/></a>.<br /><br /># About<br /># This program will connect to Inkbunny using the session cookie in the<br /># PHPSESSID environment variable then download the messages in the user's<br /># inbox and sent items, storing them as emails in maildir format in the<br /># current working directory.<br />#<br /># As the program treats the current working directory as a maildir mailbox<br /># it is important to change to the correct directory before running it.<br />#<br /># Only new items will be downloaded and saved.<br />#<br /># The program does its best to convert the HTML of a message back to BBCode,<br /># but this is an imperfect conversion and the original HTML is saved also<br /># so that either maybe used.<br />#<br /># Similarly, it is not possible to convert timestamps such as `a moment ago`<br /># or `1 hrs, 20 mins ago` to the correct dates and times with full accuracy<br /># the program does its best in those cases but some messages may be saved<br /># with dates different to those shown on the site.<br /><br /># Changelog<br /># 2025-01-03 JustLurking: Initial Release.<br /><br />import argparse<br />import bs4<br />import dataclasses<br />import email<br />import logging<br />import mailbox<br />import os<br />import re<br />import requests<br />import time<br />import urllib.parse<br /><br /># Variables used throughout the program.<br /><br /># Used for identifying the program.<br />program_file  = "ib-maildir-fetch"<br />program_name  = "Inkbunny Maildir Fetch"<br />version       = "0.1.0"<br />bug_reports   = "<a href=\"https://Inkbunny.net/JustLurking/"\" rel=\"nofollow\">https://Inkbunny.net/JustLurking/"</a>;<br />homepage      = "<a href=\"https://inkbunny.net/submissionsviewall.php?mode=pool&pool_id=98445"\" rel=\"nofollow\">https://inkbunny.net/submissionsviewall.php?mode=pool&a...</a>;<br /><br /># Used to download pages.<br />base_url      = "<a href=\"https://inkbunny.net/privatemessages.php"\" rel=\"nofollow\">https://inkbunny.net/privatemessages.php"</a>;<br />cookie        = None<br />cookies       = requests.cookies.RequestsCookieJar()<br /><br /># Used for rate limiting.<br />request_pause = 0.25<br />last_request  = 0<br /><br /># Used to hold site details.<br />user          = None<br />downloaded    = set()<br />maildir       = None<br />inbox         = None<br />sent          = None<br />latest_sent   = 0<br />latest_inbox  = 0<br />early_quit    = True<br /><br /># Used for logging.<br />log           = logging.getLogger(__name__)<br /><br /># Variables used when converting HTML to BBCode.<br /><br /># These replacements are all simple substitutions.<br />simple_replacements = {<br />    "b":        (("strong",),   {}),<br />    "center":   (("div",),      {"class_": "align_center"}),<br />    "i":        (("em",),       {}),<br />    "left":     (("div",),      {"class_": "align_left"}),<br />    "right":    (("div",),      {"class_": "align_right"}),<br />    "s":        (("span",),     {"class_": "strikethrough"}),<br />    "t":        (("span",),     {"class_": "font_title"}),<br />    "u":        (("span",),     {"class_": "underline"})<br />}<br /><br /># Map titles back to tag names.<br />offsite_mapping = {<br />    "deviantART": "da",<br />    "Fur Affinity": "fa",<br />    "SoFurry": "sf",<br />    "Weasyl": "w"<br />}<br /><br /># Regular expressions.<br />re_offsite_title = re.compile(" on (deviantART|Fur Affinity|SoFurry|Weasyl)$")<br />re_usericon      = re.compile("^<a href=\"https://inkbunny.net/usericons/"\" rel=\"nofollow\">https://inkbunny.net/usericons/"</a>)<br />re_color         = re.compile("^color: [^;]*;$")<br />re_size          = re.compile("/(small|medium|large|huge)/")<br />re_pool_url      = re.compile("^/poolview_process.php\\\\?pool_id=")<br />re_bare          = re.compile("\\\\[(b|center|i|left|right|s|t|u|q|smallpool|mediumpool|smallthumb|mediumthumb|largethumb|hugethumb|color|icon|iconname|name|da|fa|sf|w|url)(=[^]]*)?]|da!|fa!|sf!|w!")<br />re_date          = re.compile("^((\\\\d+) hrs?,? )?((\\\\d+) mins?,? )?((\\\\d+) secs? )?ago$")<br /><br /># Classes and functions.<br /><br />[at_iconname]dataclasses[/at_iconname].dataclass<br />class MessageMetadata:<br />    """<br />    Class that holds the metadata identifying a specific Inkbunny message.<br />    """<br />    msg_id: int<br />    sender: str<br />    receiver: str<br />    subject: str<br />    date: str<br />    in_reply_to: str<br /><br />    def download_messages_in_thread(self):<br />        """<br />        Fetch the thread for this message if it has not been downloaded yet<br />        and extracts all messages from it, saving them to the correct folder<br />        if they haven't been downloaded already.<br />        """<br />        global user, downloaded<br /><br />        if int(self.msg_id) in downloaded:<br />            log.debug("Ignoring thread for message %d, message already saved.", self.msg_id)<br />            return<br /><br />        log.debug("Fetching thread for message %d.", self.msg_id)<br /><br />        soup = download(<br />                "<a href=\"https://inkbunny.net/privatemessageview.php"\" rel=\"nofollow\">https://inkbunny.net/privatemessageview.php"</a>,<br />                private_message_id=str(self.msg_id)<br />            )<br />        get_logged_in_user(soup)<br />        other_user = self.sender if self.sender != user else self.receiver<br /><br />        for msg in read_thread(soup, other_user):<br />            msg_id = int(msg["Message-Id"])<br />            if msg_id in downloaded:<br />                log.debug("Ignoring message %d in thread %d, message already saved.", msg_id, self.msg_id)<br />                continue<br /><br />            if msg["To"] == user:<br />                log.debug("Saving message %d in thread %d to inbox.", msg_id, self.msg_id)<br />                inbox.add(msg)<br />            else:<br />                log.debug("Saving message %d in thread %d to sent.", msg_id, self.msg_id)<br />                sent.add(msg)<br />            downloaded.add(msg_id)<br /><br />    def create_message(self):<br />        """<br />        Create a new multi-part email.message.EmailMessage with the headers<br />        set from this metadata object.  Does not add a body, that is up to<br />        the caller.<br />        """<br />        msg = email.message.EmailMessage()<br />        msg["Message-Id"] = str(self.msg_id)<br />        msg["From"]       = self.sender<br />        msg["To"]         = self.receiver<br />        msg["Date"]       = self.date<br />        msg["Subject"]    = self.subject<br />        if self.in_reply_to is not None:<br />            msg["In-Reply-To"] = str(self.in_reply_to)<br />        msg.make_alternative()<br />        return msg<br /><br />def download(url, **kwargs):<br />    """<br />    Utility wrapper for boiler plate around downloading and parsing pages.<br />    Also enforces the rate limit.<br />    """<br />    global cookies, last_request, request_pause<br />    now = time.time()<br />    if now - last_request < request_pause:<br />        time.sleep(request_pause - now + last_request)<br />    resp = requests.get(url, cookies=cookies, params=kwargs)<br />    for (i, step) in enumerate(resp.history):<br />        log.debug("[%d] Downloaded: %s", i, step.url)<br />    log.debug("[Final] Downloaded: %s", resp.url)<br />    resp.raise_for_status()<br />    last_request = time.time()<br />    return bs4.BeautifulSoup(resp.content, features="lxml")<br /><br />def parse_date(text):<br />    """<br />    Convert a human-readable time to something that can be stored in an<br />    email header.<br />    """<br />    global re_date, last_request<br /><br />    to_subtract = None<br />    if text == "a moment ago":<br />        to_subtract = 0<br /><br />    m = re_date.match(text)<br />    if m is not None:<br />        to_subtract = 0<br />        if m.group(2) is not None:<br />            to_subtract += int(m.group(2)) * 60 * 60<br />        if m.group(4) is not None:<br />            to_subtract += int(m.group(4)) * 60<br />        if m.group(6) is not None:<br />            to_subtract += int(m.group(6))<br /><br />    if to_subtract is not None:<br />        result = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(last_request - to_subtract))<br />        return result<br /><br />    return text<br /><br />def get_logged_in_user(tag):<br />    """<br />    Sets the logged-in user name if it has not been set yet using the<br />    supplied HTML.<br />    """<br />    global user<br />    if user is not None:<br />        return<br />    nav = tag.find(class_="userdetailsnavigation")<br />    if nav is None:<br />        log.critical("Unable to find user details on page.")<br />        exit(1)<br />    widget = nav.find(class_="widget_userNameSmall")<br />    if widget is None:<br />        log.critical("Unable to find user details on page.")<br />        exit(1)<br />    user = widget.get_text().strip()<br />    log.info("Logged in as %s.", user)<br /><br />def get_next_page(tag):<br />    """<br />    Returns the URL of the first next-page link in the supplied HTML.<br />    """<br />    next_page_link = tag.find("a", title="next page")<br />    if next_page_link is None:<br />        return None<br />    return next_page_link["href"]<br /><br />def read_box_page(tag, in_inbox):<br />    """<br />    A generator which extracts rows from the index view of a box.<br />    If is_inbox is True operates over the inbox, otherwise it operates<br />    over the sent items.<br />    """<br />    global user<br />    get_logged_in_user(tag)<br /><br />    # Set the variables for the box we will be reading.<br />    min_columns    = 5<br />    user_column    = 1<br />    subject_column = 3<br />    date_column    = 4<br /><br />    if in_inbox:<br />        min_columns    = 6<br />        user_column    = 2<br />        subject_column = 4<br />        date_column    = 5<br /><br />    # Read the rows from this page of the index.<br />    for row in tag.find_all(id=re.compile("^m_\\\\d+$")):<br />        columns = [*row.find_all("td", recursive=False)]<br />        if len(columns) < min_columns:<br />            # Malformed row?  Skip it.<br />            continue<br /><br />        other_user = columns[user_column].get_text().strip()<br />        subject    = columns[subject_column].get_text().strip()<br />        date       = parse_date(columns[date_column].get_text().strip())<br />        msg_id     = int(row["id"][2:])<br /><br />        if in_inbox:<br />            yield MessageMetadata(msg_id, other_user, user, subject, date, None)<br />        else:<br />            yield MessageMetadata(msg_id, user, other_user, subject, date, None)<br /><br />def read_box(is_inbox):<br />    """<br />    A generator which extracts rows from a box.<br />    If is_inbox is True operates over the inbox, otherwise it operates<br />    over the sent items.<br />    """<br />    global latest_inbox, latest_sent, early_quit<br /><br />    # Set the variables for the box we will be reading.<br />    latest = latest_inbox if is_inbox else latest_sent<br />    end_of_new_messages = False<br /><br />    next_page = "<a href=\"https://inkbunny.net/privatemessages_process.php?mode="\" rel=\"nofollow\">https://inkbunny.net/privatemessages_process.php?mode=&...</a>;<br />    if is_inbox:<br />        log.info("Reading Inbox.")<br />        next_page += "inbox"<br />    else:<br />        log.info("Reading Sent.")<br />        next_page += "sent"<br /><br />    # Download and parse the next page of the index.<br />    while next_page is not None:<br />        next_page = urllib.parse.urljoin(base_url, next_page)<br />        soup = download(next_page)<br /><br />        # Read the rows in the index and yield them to the caller.<br />        for row in read_box_page(soup, is_inbox):<br />            if row.msg_id <= latest:<br />                end_of_new_messages = True<br />            yield row<br /><br />        # Exit the loop early if we've encountered a message on this page<br />        # older than one we've already downloaded.<br />        if early_quit and end_of_new_messages:<br />            log.info("End of new messages in this box.")<br />            break<br /><br />        next_page = get_next_page(soup)<br /><br />def unparse_html_to_bbcode(tag):<br />    """<br />    Given some HTML generated by Inkbunny's BBCode renderer attempt to return<br />    the BBCode which might have generated it.  Since the transformation is<br />    non-injective it's impossible to reverse with 100% accuracy.<br />    White-space will likely not be preserved either.<br />    """<br />    global simple_replacements, offsite_mapping, re_offsite_title, re_usericon, re_color, re_size, re_pool_url, re_bare<br /><br />    # [code]<br />    for string in tag.find_all(string=re_bare):<br />        string.replace_with(re_bare.sub(lambda m: "[code]"+m.group(0)+"[​/code]", string.string))<br /><br />    # [q] and [q=someone]<br />    for quote in tag.find_all(class_="bbcode_quote"):<br />        author = quote.find(class_="bbcode_quote_author")<br />        body = quote.find(class_="bbcode_quote_quote").extract()<br />        argument = ""<br />        if author is not None:<br />            argument = "=" + author.get_text().strip()[:-7]<br />        quote.insert_before(bs4.NavigableString("[q"+argument+"]"))<br />        quote.insert_before(body)<br /><br />        children = (*body.contents,)<br />        if len(children) == 1:<br />            if isinstance(children[0], bs4.NavigableString):<br />                children[0].replace_with(children[0].string.strip())<br />        elif len(children) > 1:<br />            if isinstance(children[0], bs4.NavigableString):<br />                children[0].replace_with(children[0].string.lstrip())<br />            if isinstance(children[-1], bs4.NavigableString):<br />                children[-1].replace_with(children[-1].string.rstrip())<br /><br />        body.unwrap()<br />        quote.insert_after(bs4.NavigableString("[/q]"))<br />        quote.extract()<br /><br />    # [smallpool], [mediumpool]<br />    for pool in tag.find_all(class_="widget_imageFromSubmission"):<br />        table = pool.find_parent("table")<br />        if table is None:<br />            continue<br />        table = table.find_parent("table")<br />        if table is None or table.parent is None:<br />            # There are three divs with widget_imageFromSubmissions as their<br />            # class per pool table, so it is possible we've already processed<br />            # this pool.<br />            # This also filters out thumbnails which get processed after.<br />            continue<br />        size = re_size.search(pool.find("img")["src"]).group(1)<br />        pool_id = table.find("a", href=re_pool_url)["href"][31:]<br />        table.replace_with(bs4.NavigableString("["+size+"pool]"+pool_id+"[/"+size+"pool]"))<br /><br />    # [smallthumb], [mediumthumb], [largethumb], [hugethumb]<br />    for thumb in tag.find_all(class_="widget_imageFromSubmission"):<br />        table = thumb.find_parent("table")<br />        size = re_size.search(thumb.find("img")["src"]).group(1)<br />        submission_id = thumb.find("a")["href"][3:]<br />        table.replace_with(bs4.NavigableString("["+size+"thumb]"+submission_id+"[/"+size+"thumb]"))<br /><br />    # [color]<br />    for color in tag.find_all("span", style=re_color):<br />        color.insert_before(bs4.NavigableString("[color="+color["style"][7:-1]+"]"))<br />        color.insert_after(bs4.NavigableString("[/color]"))<br />        color.unwrap()<br /><br />    # Newlines<br />    for br in tag.find_all("br"):<br />        br.replace_with(bs4.NavigableString("\\n"))<br /><br />    # [icon], [iconname]<br />    for icon in tag.find_all("img", src=re_usericon):<br />        table = icon.find_parent("table")<br />        link = icon.find_parent("a")<br />        name = link["href"][21:]<br />        if len((*table.find_all("a"),)) > 1:<br />            table.replace_with("[iconname]"+name+"[/iconname]")<br />        else:<br />            table.replace_with("[icon]"+name+"[/icon]")<br /><br />    # [name]<br />    for namelink in tag.find_all("span", class_="widget_userNameSmall"):<br />        name = namelink.find("a")["href"][1:]<br />        namelink.replace_with("[name]"+name+"[/name]")<br /><br />    # [da], [fa], [sf], [w]<br />    for link in tag.find_all("a", title=re_offsite_title):<br />        # Each off-site link generates two sibling elements in the output, we<br />        # can replace either, but we should remove the one we don't replace.<br />        if link.find("img") is None:<br />            # Replace the textual link.<br />            m = re_offsite_title.search(link["title"])<br />            site = offsite_mapping[m.group(1)]<br />            link.replace_with("["+site+"]"+link.get_text().strip()+"[/"+site+"]")<br />        else:<br />            # Remove the image link.<br />            link.extract()<br /><br />    # [url]<br />    for a in tag.find_all("a"):<br />        a.insert_before(bs4.NavigableString("[url="+a["href"]+"]"))<br />        a.insert_after(bs4.NavigableString("[/url]"))<br />        a.unwrap()<br /><br />    # simple replacements<br />    for (name, args) in simple_replacements.items():<br />        for target in tag.find_all(*args[0], **args[1]):<br />            target.insert_before(bs4.NavigableString("["+name+"]"))<br />            target.insert_after(bs4.NavigableString("[/"+name+"]"))<br />            target.unwrap()<br /><br />    # combine all text nodes and return the string representation of the child<br />    # nodes.<br />    tag.smooth()<br />    return "".join(str(n) for n in tag.contents)<br /><br />def read_thread(tag, other_user):<br />    """<br />    A generator which extracts messages from thread view.<br />    """<br />    global user, messages<br />    previous_subject = None<br />    previous_msg_id  = None<br /><br />    got_result = False<br />    for elem in tag.find_all(id=re.compile("^irt_message_\\\\d+$")):<br />        # Extract the next message's data from the page.<br />        msg_id = int(elem["id"][12:])<br /><br />        children = [e for e in elem.children if not isinstance(e, bs4.NavigableString)]<br /><br />        if len(children) < 5:<br />            # Malformed<br />            continue<br /><br />        date_field    = children[2].find(True)<br />        body_span     = children[2].find("span", style="word-wrap: break-word;")<br /><br />        if date_field is None or body_span is None:<br />            # Malformed<br />            continue<br /><br />        subject_div = children[2].find(style="margin-bottom: 5px;")<br />        left_link   = children[0].find(class_="widget_userNameSmall")<br />        right_link  = children[4].find(class_="widget_userNameSmall")<br /><br />        date        = parse_date(date_field.get_text().strip())<br />        subject     = subject_div.get_text().strip() if subject_div is not None else previous_subject<br />        in_reply_to = previous_msg_id<br />        sender      = None<br /><br />        if left_link is not None:<br />            sender = left_link.get_text().strip()<br />        elif right_link is not None:<br />            sender = right_link.get_text().strip()<br />        else:<br />            continue<br /><br />        receiver = other_user if sender == user else user<br /><br />        # Create, populate and yield an EmailMessage for this message.<br />        content = bs4.BeautifulSoup("<html><title/><body/>", features="lxml")<br />        content.title.string = subject<br />        content.body.append(body_span)<br />        body_span.unwrap()<br /><br />        metadata = MessageMetadata(msg_id, sender, receiver, subject, date, in_reply_to)<br />        msg = metadata.create_message()<br />        msg.add_alternative("<!DOCTYPE html>" + str(content), "html")<br />        msg.add_alternative(str(unparse_html_to_bbcode(content.body)))<br />        yield msg<br />        got_result = True<br /><br />        # Update information on the previous message (the one we just yielded)<br />        # prior to extracting next message.<br />        if subject is not None:<br />            previous_subject = subject<br />        previous_msg_id = msg_id<br /><br />    # This should only be reached if a page other than the one we were<br />    # expecting is returned, i.e. an error page or similar.<br />    # Since something has clearly gone wrong just inform the user and abort.<br />    if not got_result:<br />        log.critical("Failed to parse at least one message in thread.  Got: %s", str(tag))<br />        exit(1)<br /><br /># Main Program starts here.<br /><br /># Configure logging.<br />log.addHandler(logging.StreamHandler())<br />log.setLevel(logging.INFO)<br /><br /># Handle arguments.<br />arg_parser = argparse.ArgumentParser(<br />    prog = program_file,<br />    description = "".join((<br />        "Downloads messages from the Inkbunny.net website and stores them",<br />        " as emails in maildir format."<br />    )),<br />    epilog = "".join((<br />        "If no DIRECTORY is specified then the current directory is",<br />        " assumed to be the maildir, care should be taken to ensure that",<br />        " the program is run in the correct directory or it may read or",<br />        " write to files the user does not want it to.\\n",<br />        "\\n",<br />        "The SESSION may be passed using the PHPSESSID environment variable."<br />        " this may be useful if you wish to pass the same value to multiple"<br />        " programs, or if you wish to set the value in your shell's profile.\\n"<br />        "\\n",<br />        "Report bugs to: "+bug_reports+"\\n",<br />        program_name + " home page: <"+homepage+"\\n"<br />    )),<br />    formatter_class = argparse.RawDescriptionHelpFormatter<br />)<br />arg_parser.add_argument(<br />    "-s",<br />    "--session",<br />    nargs = 1,<br />    help = "the session id to send with requests"<br />)<br />arg_parser.add_argument(<br />    "-C",<br />    "--directory",<br />    nargs = 1,<br />    help = "change directory when program starts"<br />)<br />arg_parser.add_argument(<br />    "-q",<br />    "--quiet",<br />    action = "count",<br />    default = 2,<br />    help = "decrease verbosity"<br />)<br />arg_parser.add_argument(<br />    "-v",<br />    "--verbose",<br />    action = "count",<br />    default = 0,<br />    help = "increase verbosity"<br />)<br />arg_parser.add_argument(<br />    "-f",<br />    "--full-scan",<br />    action = "store_true",<br />    help = "read the full message index even if it seems unnecessary"<br />)<br />arg_parser.add_argument(<br />    "-V",<br />    "--version",<br />    action = "store_true"<br />)<br /><br />args = arg_parser.parse_args()<br /><br />if args.version:<br />    print(<br />        " ".join((program_name, version)),<br />        "Copyright (C) 2025 JustLurking<<a href=\"https://inkbunny.net/JustLurking>"\" rel=\"nofollow\">https://inkbunny.net/JustLurking>"</a>,<br />        "License GPLv3+: GNU GPL version 3 or later "+<br />            "<<a href=\"https://gnu.org/licenses/gpl.html>"\" rel=\"nofollow\">https://gnu.org/licenses/gpl.html>"</a>,<br />        "",<br />        "This is free software: you are free to change and redistribute it.",<br />        "There is NO WARRANTY, to the extent permitted by law.",<br />        sep="\\n"<br />    )<br />    exit(0)<br /><br />log.setLevel(max(1, min(5, args.quiet-args.verbose))*10)<br /><br />if args.directory is not None:<br />    log.info("Changing to directory %d.")<br />    os.chdir(args.directory)<br /><br />if args.full_scan:<br />    early_quit = False<br /><br />if args.session is not None:<br />    cookie = args.session<br /><br /># Open or create Maildir.<br />maildir = mailbox.Maildir(".")<br />inbox   = maildir.add_folder("inbox")<br />sent    = maildir.add_folder("sent")<br /><br /># Set the Session Cookie from the environment.<br />if cookie is None:<br />    cookie = os.getenv("PHPSESSID")<br /><br />if cookie is None:<br />    log.error("Please use the -s argument or set the PHPSESSID environment variable to set the session id.")<br />    exit(1)<br />cookies.set("PHPSESSID", cookie, domain="inkbunny.net", path="/")<br /><br /># Make a list of all already downloaded messages.<br />for msg in inbox:<br />        msg_id = int(msg["Message-Id"])<br />        downloaded.add(msg_id)<br />        latest_inbox = max(latest_inbox, msg_id)<br /><br />for msg in sent:<br />        msg_id = int(msg["Message-Id"])<br />        downloaded.add(msg_id)<br />        latest_sent = max(latest_sent, msg_id)<br /><br /># Fetch message indices for inbox and sent and for each message found fetch the<br /># appropriate thread view and download all new messages in that thread.<br />for row in read_box(True):<br />    row.download_messages_in_thread()<br /><br />for row in read_box(False):<br />    row.download_messages_in_thread()<br /></span>", "pools_count": 2, "title": "Inkbunny Maildir Fetch 0.1.0", "deleted": "f", "public": "t", "mimetype": "text/plain", "pagecount": "1", "rating_id": "0", "rating_name": "General", "ratings": [], "submission_type_id": "12", "type_name": "Writing - Document", "guest_block": "f", "friends_only": "f", "comments_count": "2", "views": "68" }