You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
21576 lines
1016 KiB
Python
21576 lines
1016 KiB
Python
__filename__ = "daemon.py"
|
|
__author__ = "Bob Mottram"
|
|
__license__ = "AGPL3+"
|
|
__version__ = "1.3.0"
|
|
__maintainer__ = "Bob Mottram"
|
|
__email__ = "bob@libreserver.org"
|
|
__status__ = "Production"
|
|
__module_group__ = "Core"
|
|
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer, HTTPServer
|
|
import copy
|
|
import sys
|
|
import json
|
|
import time
|
|
import urllib.parse
|
|
import datetime
|
|
from socket import error as SocketError
|
|
import errno
|
|
from functools import partial
|
|
# for saving images
|
|
from hashlib import sha256
|
|
from hashlib import md5
|
|
from shutil import copyfile
|
|
from session import create_session
|
|
from session import get_session_for_domain
|
|
from session import get_session_for_domains
|
|
from session import set_session_for_sender
|
|
from webfinger import webfinger_meta
|
|
from webfinger import webfinger_node_info
|
|
from webfinger import webfinger_lookup
|
|
from webfinger import webfinger_update
|
|
from mastoapiv1 import masto_api_v1_response
|
|
from metadata import meta_data_node_info
|
|
from metadata import metadata_custom_emoji
|
|
from enigma import get_enigma_pub_key
|
|
from enigma import set_enigma_pub_key
|
|
from pgp import actor_to_vcard
|
|
from pgp import actor_to_vcard_xml
|
|
from pgp import get_email_address
|
|
from pgp import set_email_address
|
|
from pgp import get_pgp_pub_key
|
|
from pgp import get_pgp_fingerprint
|
|
from pgp import set_pgp_pub_key
|
|
from pgp import set_pgp_fingerprint
|
|
from xmpp import get_xmpp_address
|
|
from xmpp import set_xmpp_address
|
|
from ssb import get_ssb_address
|
|
from ssb import set_ssb_address
|
|
from tox import get_tox_address
|
|
from tox import set_tox_address
|
|
from briar import get_briar_address
|
|
from briar import set_briar_address
|
|
from cwtch import get_cwtch_address
|
|
from cwtch import set_cwtch_address
|
|
from matrix import get_matrix_address
|
|
from matrix import set_matrix_address
|
|
from donate import get_donation_url
|
|
from donate import set_donation_url
|
|
from donate import get_website
|
|
from donate import set_website
|
|
from person import clear_person_qrcodes
|
|
from person import add_alternate_domains
|
|
from person import add_actor_update_timestamp
|
|
from person import set_person_notes
|
|
from person import get_default_person_context
|
|
from person import get_actor_update_json
|
|
from person import save_person_qrcode
|
|
from person import randomize_actor_images
|
|
from person import person_upgrade_actor
|
|
from person import activate_account
|
|
from person import deactivate_account
|
|
from person import register_account
|
|
from person import person_lookup
|
|
from person import person_box_json
|
|
from person import create_shared_inbox
|
|
from person import create_news_inbox
|
|
from person import suspend_account
|
|
from person import reenable_account
|
|
from person import remove_account
|
|
from person import can_remove_post
|
|
from person import person_snooze
|
|
from person import person_unsnooze
|
|
from posts import get_post_expiry_keep_dms
|
|
from posts import set_post_expiry_keep_dms
|
|
from posts import get_post_expiry_days
|
|
from posts import set_post_expiry_days
|
|
from posts import get_original_post_from_announce_url
|
|
from posts import save_post_to_box
|
|
from posts import get_instance_actor_key
|
|
from posts import remove_post_interactions
|
|
from posts import outbox_message_create_wrap
|
|
from posts import get_pinned_post_as_json
|
|
from posts import pin_post
|
|
from posts import json_pin_post
|
|
from posts import undo_pinned_post
|
|
from posts import is_moderator
|
|
from posts import create_question_post
|
|
from posts import create_public_post
|
|
from posts import create_blog_post
|
|
from posts import create_report_post
|
|
from posts import create_unlisted_post
|
|
from posts import create_followers_only_post
|
|
from posts import create_direct_message_post
|
|
from posts import populate_replies_json
|
|
from posts import add_to_field
|
|
from posts import expire_cache
|
|
from inbox import clear_queue_items
|
|
from inbox import inbox_permitted_message
|
|
from inbox import inbox_message_has_params
|
|
from inbox import run_inbox_queue
|
|
from inbox import run_inbox_queue_watchdog
|
|
from inbox import save_post_to_inbox_queue
|
|
from inbox import populate_replies
|
|
from follow import follower_approval_active
|
|
from follow import is_following_actor
|
|
from follow import get_following_feed
|
|
from follow import send_follow_request
|
|
from follow import unfollow_account
|
|
from follow import create_initial_last_seen
|
|
from skills import get_skills_from_list
|
|
from skills import no_of_actor_skills
|
|
from skills import actor_has_skill
|
|
from skills import actor_skill_value
|
|
from skills import set_actor_skill_level
|
|
from auth import record_login_failure
|
|
from auth import authorize
|
|
from auth import create_password
|
|
from auth import create_basic_auth_header
|
|
from auth import authorize_basic
|
|
from auth import store_basic_credentials
|
|
from threads import begin_thread
|
|
from threads import thread_with_trace
|
|
from threads import remove_dormant_threads
|
|
from media import process_meta_data
|
|
from media import convert_image_to_low_bandwidth
|
|
from media import replace_you_tube
|
|
from media import replace_twitter
|
|
from media import attach_media
|
|
from media import path_is_video
|
|
from media import path_is_audio
|
|
from blocking import get_cw_list_variable
|
|
from blocking import load_cw_lists
|
|
from blocking import update_blocked_cache
|
|
from blocking import mute_post
|
|
from blocking import unmute_post
|
|
from blocking import set_broch_mode
|
|
from blocking import broch_mode_is_active
|
|
from blocking import add_block
|
|
from blocking import remove_block
|
|
from blocking import add_global_block
|
|
from blocking import remove_global_block
|
|
from blocking import is_blocked_hashtag
|
|
from blocking import is_blocked_domain
|
|
from blocking import get_domain_blocklist
|
|
from roles import set_roles_from_list
|
|
from roles import get_actor_roles_list
|
|
from blog import path_contains_blog_link
|
|
from blog import html_blog_page_rss2
|
|
from blog import html_blog_page_rss3
|
|
from blog import html_blog_view
|
|
from blog import html_blog_page
|
|
from blog import html_blog_post
|
|
from blog import html_edit_blog
|
|
from blog import get_blog_address
|
|
from webapp_podcast import html_podcast_episode
|
|
from webapp_theme_designer import html_theme_designer
|
|
from webapp_minimalbutton import set_minimal
|
|
from webapp_minimalbutton import is_minimal
|
|
from webapp_utils import get_avatar_image_url
|
|
from webapp_utils import html_hashtag_blocked
|
|
from webapp_utils import html_following_list
|
|
from webapp_utils import csv_following_list
|
|
from webapp_utils import set_blog_address
|
|
from webapp_utils import html_show_share
|
|
from webapp_utils import get_pwa_theme_colors
|
|
from webapp_utils import text_mode_browser
|
|
from webapp_calendar import html_calendar_delete_confirm
|
|
from webapp_calendar import html_calendar
|
|
from webapp_about import html_about
|
|
from webapp_specification import html_specification
|
|
from webapp_accesskeys import html_access_keys
|
|
from webapp_accesskeys import load_access_keys_for_accounts
|
|
from webapp_confirm import html_confirm_delete
|
|
from webapp_confirm import html_confirm_remove_shared_item
|
|
from webapp_confirm import html_confirm_block
|
|
from webapp_confirm import html_confirm_unblock
|
|
from webapp_person_options import person_minimize_images
|
|
from webapp_person_options import person_undo_minimize_images
|
|
from webapp_person_options import html_person_options
|
|
from webapp_timeline import html_shares
|
|
from webapp_timeline import html_wanted
|
|
from webapp_timeline import html_inbox
|
|
from webapp_timeline import html_bookmarks
|
|
from webapp_timeline import html_inbox_dms
|
|
from webapp_timeline import html_inbox_replies
|
|
from webapp_timeline import html_inbox_media
|
|
from webapp_timeline import html_inbox_blogs
|
|
from webapp_timeline import html_inbox_news
|
|
from webapp_timeline import html_inbox_features
|
|
from webapp_timeline import html_outbox
|
|
from webapp_media import load_peertube_instances
|
|
from webapp_moderation import html_account_info
|
|
from webapp_moderation import html_moderation
|
|
from webapp_moderation import html_moderation_info
|
|
from webapp_create_post import html_new_post
|
|
from webapp_login import html_login
|
|
from webapp_login import html_get_login_credentials
|
|
from webapp_suspended import html_suspended
|
|
from webapp_tos import html_terms_of_service
|
|
from webapp_confirm import html_confirm_follow
|
|
from webapp_confirm import html_confirm_unfollow
|
|
from webapp_post import html_emoji_reaction_picker
|
|
from webapp_post import html_post_replies
|
|
from webapp_post import html_individual_post
|
|
from webapp_post import individual_post_as_html
|
|
from webapp_profile import html_edit_profile
|
|
from webapp_profile import html_profile_after_search
|
|
from webapp_profile import html_profile
|
|
from webapp_column_left import html_links_mobile
|
|
from webapp_column_left import html_edit_links
|
|
from webapp_column_right import html_newswire_mobile
|
|
from webapp_column_right import html_edit_newswire
|
|
from webapp_column_right import html_citations
|
|
from webapp_column_right import html_edit_news_post
|
|
from webapp_search import html_skills_search
|
|
from webapp_search import html_history_search
|
|
from webapp_search import html_hashtag_search
|
|
from webapp_search import rss_hashtag_search
|
|
from webapp_search import html_search_emoji
|
|
from webapp_search import html_search_shared_items
|
|
from webapp_search import html_search_emoji_text_entry
|
|
from webapp_search import html_search
|
|
from webapp_hashtagswarm import get_hashtag_categories_feed
|
|
from webapp_hashtagswarm import html_search_hashtag_category
|
|
from webapp_welcome import welcome_screen_is_complete
|
|
from webapp_welcome import html_welcome_screen
|
|
from webapp_welcome import is_welcome_screen_complete
|
|
from webapp_welcome_profile import html_welcome_profile
|
|
from webapp_welcome_final import html_welcome_final
|
|
from shares import merge_shared_item_tokens
|
|
from shares import run_federated_shares_daemon
|
|
from shares import run_federated_shares_watchdog
|
|
from shares import update_shared_item_federation_token
|
|
from shares import create_shared_item_federation_token
|
|
from shares import authorize_shared_items
|
|
from shares import generate_shared_item_federation_tokens
|
|
from shares import get_shares_feed_for_person
|
|
from shares import add_share
|
|
from shares import remove_shared_item
|
|
from shares import expire_shares
|
|
from shares import shares_catalog_endpoint
|
|
from shares import shares_catalog_account_endpoint
|
|
from shares import shares_catalog_csv_endpoint
|
|
from categories import set_hashtag_category
|
|
from categories import update_hashtag_categories
|
|
from languages import get_actor_languages
|
|
from languages import set_actor_languages
|
|
from languages import get_understood_languages
|
|
from like import update_likes_collection
|
|
from reaction import update_reaction_collection
|
|
from utils import get_json_content_from_accept
|
|
from utils import remove_eol
|
|
from utils import text_in_file
|
|
from utils import is_onion_request
|
|
from utils import is_i2p_request
|
|
from utils import get_account_timezone
|
|
from utils import set_account_timezone
|
|
from utils import load_account_timezones
|
|
from utils import local_network_host
|
|
from utils import undo_reaction_collection_entry
|
|
from utils import get_new_post_endpoints
|
|
from utils import has_actor
|
|
from utils import set_reply_interval_hours
|
|
from utils import can_reply_to
|
|
from utils import is_dm
|
|
from utils import replace_users_with_at
|
|
from utils import local_actor_url
|
|
from utils import is_float
|
|
from utils import valid_password
|
|
from utils import get_base_content_from_post
|
|
from utils import acct_dir
|
|
from utils import get_image_extension_from_mime_type
|
|
from utils import get_image_mime_type
|
|
from utils import has_object_dict
|
|
from utils import user_agent_domain
|
|
from utils import is_local_network_address
|
|
from utils import permitted_dir
|
|
from utils import is_account_dir
|
|
from utils import get_occupation_skills
|
|
from utils import get_occupation_name
|
|
from utils import set_occupation_name
|
|
from utils import load_translations_from_file
|
|
from utils import load_bold_reading
|
|
from utils import get_local_network_addresses
|
|
from utils import decoded_host
|
|
from utils import is_public_post
|
|
from utils import get_locked_account
|
|
from utils import has_users_path
|
|
from utils import get_full_domain
|
|
from utils import remove_html
|
|
from utils import is_editor
|
|
from utils import is_artist
|
|
from utils import get_image_extensions
|
|
from utils import media_file_mime_type
|
|
from utils import get_css
|
|
from utils import first_paragraph_from_string
|
|
from utils import clear_from_post_caches
|
|
from utils import contains_invalid_chars
|
|
from utils import is_system_account
|
|
from utils import set_config_param
|
|
from utils import get_config_param
|
|
from utils import remove_id_ending
|
|
from utils import undo_likes_collection_entry
|
|
from utils import delete_post
|
|
from utils import is_blog_post
|
|
from utils import remove_avatar_from_cache
|
|
from utils import locate_post
|
|
from utils import get_cached_post_filename
|
|
from utils import remove_post_from_cache
|
|
from utils import get_nickname_from_actor
|
|
from utils import get_domain_from_actor
|
|
from utils import get_status_number
|
|
from utils import url_permitted
|
|
from utils import load_json
|
|
from utils import save_json
|
|
from utils import is_suspended
|
|
from utils import dangerous_markup
|
|
from utils import refresh_newswire
|
|
from utils import is_image_file
|
|
from utils import has_group_type
|
|
from manualapprove import manual_deny_follow_request_thread
|
|
from manualapprove import manual_approve_follow_request_thread
|
|
from announce import create_announce
|
|
from content import load_dogwhistles
|
|
from content import valid_url_lengths
|
|
from content import contains_invalid_local_links
|
|
from content import get_price_from_string
|
|
from content import replace_emoji_from_tags
|
|
from content import add_html_tags
|
|
from content import extract_media_in_form_post
|
|
from content import save_media_in_form_post
|
|
from content import extract_text_fields_in_post
|
|
from cache import check_for_changed_actor
|
|
from cache import store_person_in_cache
|
|
from cache import get_person_from_cache
|
|
from cache import get_person_pub_key
|
|
from httpsig import verify_post_headers
|
|
from theme import reset_theme_designer_settings
|
|
from theme import set_theme_from_designer
|
|
from theme import scan_themes_for_scripts
|
|
from theme import import_theme
|
|
from theme import export_theme
|
|
from theme import is_news_theme_name
|
|
from theme import get_text_mode_banner
|
|
from theme import set_news_avatar
|
|
from theme import set_theme
|
|
from theme import get_theme
|
|
from theme import enable_grayscale
|
|
from theme import disable_grayscale
|
|
from schedule import run_post_schedule
|
|
from schedule import run_post_schedule_watchdog
|
|
from schedule import remove_scheduled_posts
|
|
from outbox import post_message_to_outbox
|
|
from happening import remove_calendar_event
|
|
from happening import dav_propfind_response
|
|
from happening import dav_put_response
|
|
from happening import dav_report_response
|
|
from happening import dav_delete_response
|
|
from bookmarks import bookmark_post
|
|
from bookmarks import undo_bookmark_post
|
|
from petnames import set_pet_name
|
|
from followingCalendar import add_person_to_calendar
|
|
from followingCalendar import remove_person_from_calendar
|
|
from notifyOnPost import add_notify_on_post
|
|
from notifyOnPost import remove_notify_on_post
|
|
from devices import e2e_edevices_collection
|
|
from devices import e2e_evalid_device
|
|
from devices import e2e_eadd_device
|
|
from newswire import get_rs_sfrom_dict
|
|
from newswire import rss2header
|
|
from newswire import rss2footer
|
|
from newswire import load_hashtag_categories
|
|
from newsdaemon import run_newswire_watchdog
|
|
from newsdaemon import run_newswire_daemon
|
|
from filters import is_filtered
|
|
from filters import add_global_filter
|
|
from filters import remove_global_filter
|
|
from context import has_valid_context
|
|
from context import get_individual_post_context
|
|
from speaker import get_ssml_box
|
|
from city import get_spoofed_city
|
|
from fitnessFunctions import fitness_performance
|
|
from fitnessFunctions import fitness_thread
|
|
from fitnessFunctions import sorted_watch_points
|
|
from fitnessFunctions import html_watch_points_graph
|
|
from siteactive import referer_is_active
|
|
from webapp_likers import html_likers_of_post
|
|
from crawlers import update_known_crawlers
|
|
from crawlers import blocked_user_agent
|
|
from crawlers import load_known_web_bots
|
|
from qrcode import save_domain_qrcode
|
|
from importFollowing import run_import_following_watchdog
|
|
from maps import map_format_from_tagmaps_path
|
|
import os
|
|
|
|
|
|
# maximum number of posts to list in outbox feed
|
|
MAX_POSTS_IN_FEED = 12
|
|
|
|
# maximum number of posts in a hashtag feed
|
|
MAX_POSTS_IN_HASHTAG_FEED = 6
|
|
|
|
# reduced posts for media feed because it can take a while
|
|
MAX_POSTS_IN_MEDIA_FEED = 6
|
|
|
|
# Blogs can be longer, so don't show many per page
|
|
MAX_POSTS_IN_BLOGS_FEED = 4
|
|
|
|
MAX_POSTS_IN_NEWS_FEED = 10
|
|
|
|
# Maximum number of entries in returned rss.xml
|
|
MAX_POSTS_IN_RSS_FEED = 10
|
|
|
|
# number of follows/followers per page
|
|
FOLLOWS_PER_PAGE = 6
|
|
|
|
# number of item shares per page
|
|
SHARES_PER_PAGE = 12
|
|
|
|
|
|
class PubServer(BaseHTTPRequestHandler):
|
|
protocol_version = 'HTTP/1.1'
|
|
|
|
def _convert_domains(self, calling_domain, referer_domain,
|
|
msg_str: str) -> str:
|
|
"""Convert domains to onion or i2p, depending upon who is asking
|
|
"""
|
|
curr_http_prefix = self.server.http_prefix + '://'
|
|
if is_onion_request(calling_domain, referer_domain,
|
|
self.server.domain,
|
|
self.server.onion_domain):
|
|
msg_str = msg_str.replace(curr_http_prefix +
|
|
self.server.domain,
|
|
'http://' +
|
|
self.server.onion_domain)
|
|
elif is_i2p_request(calling_domain, referer_domain,
|
|
self.server.domain,
|
|
self.server.i2p_domain):
|
|
msg_str = msg_str.replace(curr_http_prefix +
|
|
self.server.domain,
|
|
'http://' +
|
|
self.server.i2p_domain)
|
|
return msg_str
|
|
|
|
def _detect_mitm(self) -> bool:
|
|
"""Detect if a request contains a MiTM
|
|
"""
|
|
mitm_domains = ['cloudflare']
|
|
check_headers = (
|
|
'Server', 'Report-To', 'Report-to', 'report-to',
|
|
'Expect-CT', 'Expect-Ct', 'expect-ct'
|
|
)
|
|
for interloper in mitm_domains:
|
|
for header_name in check_headers:
|
|
if self.headers.get(header_name):
|
|
if interloper in self.headers[header_name]:
|
|
return True
|
|
# The presence if these headers on their own indicates a MiTM
|
|
mitm_headers = (
|
|
'CF-Connecting-IP', 'CF-RAY', 'CF-IPCountry', 'CF-Visitor',
|
|
'CDN-Loop', 'CF-Worker', 'CF-Cache-Status'
|
|
)
|
|
for header_name in mitm_headers:
|
|
if self.headers.get(header_name):
|
|
return True
|
|
if self.headers.get(header_name.lower()):
|
|
return True
|
|
return False
|
|
|
|
def _get_instance_url(self, calling_domain: str) -> str:
|
|
"""Returns the URL for this instance
|
|
"""
|
|
if calling_domain.endswith('.onion') and \
|
|
self.server.onion_domain:
|
|
instance_url = 'http://' + self.server.onion_domain
|
|
elif (calling_domain.endswith('.i2p') and
|
|
self.server.i2p_domain):
|
|
instance_url = 'http://' + self.server.i2p_domain
|
|
else:
|
|
instance_url = \
|
|
self.server.http_prefix + '://' + self.server.domain_full
|
|
return instance_url
|
|
|
|
def _getheader_signature_input(self):
|
|
"""There are different versions of http signatures with
|
|
different header styles
|
|
"""
|
|
if self.headers.get('Signature-Input'):
|
|
# https://tools.ietf.org/html/
|
|
# draft-ietf-httpbis-message-signatures-01
|
|
return self.headers['Signature-Input']
|
|
if self.headers.get('signature-input'):
|
|
return self.headers['signature-input']
|
|
if self.headers.get('signature'):
|
|
# Ye olde Masto http sig
|
|
return self.headers['signature']
|
|
return None
|
|
|
|
def handle_error(self, request, client_address):
|
|
print('ERROR: http server error: ' + str(request) + ', ' +
|
|
str(client_address))
|
|
|
|
def _send_reply_to_question(self, nickname: str, message_id: str,
|
|
answer: str,
|
|
curr_session, proxy_type: str) -> None:
|
|
"""Sends a reply to a question
|
|
"""
|
|
votes_filename = \
|
|
acct_dir(self.server.base_dir, nickname, self.server.domain) + \
|
|
'/questions.txt'
|
|
|
|
if os.path.isfile(votes_filename):
|
|
# have we already voted on this?
|
|
if text_in_file(message_id, votes_filename):
|
|
print('Already voted on message ' + message_id)
|
|
return
|
|
|
|
print('Voting on message ' + message_id)
|
|
print('Vote for: ' + answer)
|
|
comments_enabled = True
|
|
attach_image_filename = None
|
|
media_type = None
|
|
image_description = None
|
|
in_reply_to = message_id
|
|
in_reply_to_atom_uri = message_id
|
|
subject = None
|
|
schedule_post = False
|
|
event_date = None
|
|
event_time = None
|
|
event_end_time = None
|
|
location = None
|
|
conversation_id = None
|
|
city = get_spoofed_city(self.server.city,
|
|
self.server.base_dir,
|
|
nickname, self.server.domain)
|
|
languages_understood = \
|
|
get_understood_languages(self.server.base_dir,
|
|
self.server.http_prefix,
|
|
nickname,
|
|
self.server.domain_full,
|
|
self.server.person_cache)
|
|
|
|
message_json = \
|
|
create_public_post(self.server.base_dir,
|
|
nickname,
|
|
self.server.domain, self.server.port,
|
|
self.server.http_prefix,
|
|
answer, False, False,
|
|
comments_enabled,
|
|
attach_image_filename, media_type,
|
|
image_description, city,
|
|
in_reply_to,
|
|
in_reply_to_atom_uri,
|
|
subject,
|
|
schedule_post,
|
|
event_date,
|
|
event_time, event_end_time,
|
|
location, False,
|
|
self.server.system_language,
|
|
conversation_id,
|
|
self.server.low_bandwidth,
|
|
self.server.content_license_url,
|
|
languages_understood,
|
|
self.server.translate)
|
|
if message_json:
|
|
# name field contains the answer
|
|
message_json['object']['name'] = answer
|
|
if self._post_to_outbox(message_json,
|
|
self.server.project_version, nickname,
|
|
curr_session, proxy_type):
|
|
post_filename = \
|
|
locate_post(self.server.base_dir, nickname,
|
|
self.server.domain, message_id)
|
|
if post_filename:
|
|
post_json_object = load_json(post_filename)
|
|
if post_json_object:
|
|
populate_replies(self.server.base_dir,
|
|
self.server.http_prefix,
|
|
self.server.domain_full,
|
|
post_json_object,
|
|
self.server.max_replies,
|
|
self.server.debug)
|
|
# record the vote
|
|
try:
|
|
with open(votes_filename, 'a+',
|
|
encoding='utf-8') as votes_file:
|
|
votes_file.write(message_id + '\n')
|
|
except OSError:
|
|
print('EX: unable to write vote ' +
|
|
votes_filename)
|
|
|
|
# ensure that the cached post is removed if it exists,
|
|
# so that it then will be recreated
|
|
cached_post_filename = \
|
|
get_cached_post_filename(self.server.base_dir,
|
|
nickname,
|
|
self.server.domain,
|
|
post_json_object)
|
|
if cached_post_filename:
|
|
if os.path.isfile(cached_post_filename):
|
|
try:
|
|
os.remove(cached_post_filename)
|
|
except OSError:
|
|
print('EX: _send_reply_to_question ' +
|
|
'unable to delete ' +
|
|
cached_post_filename)
|
|
# remove from memory cache
|
|
remove_post_from_cache(post_json_object,
|
|
self.server.recent_posts_cache)
|
|
else:
|
|
print('ERROR: unable to post vote to outbox')
|
|
else:
|
|
print('ERROR: unable to create vote')
|
|
|
|
def _request_csv(self) -> bool:
|
|
"""Should a csv response be given?
|
|
"""
|
|
if not self.headers.get('Accept'):
|
|
return False
|
|
accept_str = self.headers['Accept']
|
|
if 'text/csv' in accept_str:
|
|
return True
|
|
return False
|
|
|
|
def _request_ssml(self) -> bool:
|
|
"""Should a ssml response be given?
|
|
"""
|
|
if not self.headers.get('Accept'):
|
|
return False
|
|
accept_str = self.headers['Accept']
|
|
if 'application/ssml' in accept_str:
|
|
if 'text/html' not in accept_str:
|
|
return True
|
|
return False
|
|
|
|
def _request_http(self) -> bool:
|
|
"""Should a http response be given?
|
|
"""
|
|
if not self.headers.get('Accept'):
|
|
return False
|
|
accept_str = self.headers['Accept']
|
|
if self.server.debug:
|
|
print('ACCEPT: ' + accept_str)
|
|
if 'application/ssml' in accept_str:
|
|
if 'text/html' not in accept_str:
|
|
return False
|
|
if 'image/' in accept_str:
|
|
if 'text/html' not in accept_str:
|
|
return False
|
|
if 'video/' in accept_str:
|
|
if 'text/html' not in accept_str:
|
|
return False
|
|
if 'audio/' in accept_str:
|
|
if 'text/html' not in accept_str:
|
|
return False
|
|
if accept_str.startswith('*') or 'text/html' in accept_str:
|
|
if self.headers.get('User-Agent'):
|
|
if text_mode_browser(self.headers['User-Agent']):
|
|
return True
|
|
if 'text/html' not in accept_str:
|
|
return False
|
|
if 'json' in accept_str:
|
|
return False
|
|
return True
|
|
|
|
def _request_icalendar(self) -> bool:
|
|
"""Should an icalendar response be given?
|
|
"""
|
|
if not self.headers.get('Accept'):
|
|
return False
|
|
accept_str = self.headers['Accept']
|
|
if 'text/calendar' in accept_str:
|
|
return True
|
|
return False
|
|
|
|
def _signed_get_key_id(self) -> str:
|
|
"""Returns the actor from the signed GET key_id
|
|
"""
|
|
signature = None
|
|
if self.headers.get('signature'):
|
|
signature = self.headers['signature']
|
|
elif self.headers.get('Signature'):
|
|
signature = self.headers['Signature']
|
|
|
|
# check that the headers are signed
|
|
if not signature:
|
|
if self.server.debug:
|
|
print('AUTH: secure mode actor, ' +
|
|
'GET has no signature in headers')
|
|
return None
|
|
|
|
# get the key_id, which is typically the instance actor
|
|
key_id = None
|
|
signature_params = signature.split(',')
|
|
for signature_item in signature_params:
|
|
if signature_item.startswith('keyId='):
|
|
if '"' in signature_item:
|
|
key_id = signature_item.split('"')[1]
|
|
# remove #/main-key or #main-key
|
|
if '#' in key_id:
|
|
key_id = key_id.split('#')[0]
|
|
return key_id
|
|
return None
|
|
|
|
def _establish_session(self,
|
|
calling_function: str,
|
|
curr_session,
|
|
proxy_type: str):
|
|
"""Recreates session if needed
|
|
"""
|
|
if curr_session:
|
|
return curr_session
|
|
print('DEBUG: creating new session during ' + calling_function)
|
|
curr_session = create_session(proxy_type)
|
|
if curr_session:
|
|
set_session_for_sender(self.server, proxy_type, curr_session)
|
|
return curr_session
|
|
print('ERROR: GET failed to create session during ' +
|
|
calling_function)
|
|
return None
|
|
|
|
def _secure_mode(self, curr_session, proxy_type: str,
|
|
force: bool = False) -> bool:
|
|
"""http authentication of GET requests for json
|
|
"""
|
|
if not self.server.secure_mode and not force:
|
|
return True
|
|
|
|
key_id = self._signed_get_key_id()
|
|
if not key_id:
|
|
if self.server.debug:
|
|
print('AUTH: secure mode, ' +
|
|
'failed to obtain key_id from signature')
|
|
return False
|
|
|
|
# is the key_id (actor) valid?
|
|
if not url_permitted(key_id, self.server.federation_list):
|
|
if self.server.debug:
|
|
print('AUTH: Secure mode GET request not permitted: ' + key_id)
|
|
return False
|
|
|
|
if self.server.onion_domain:
|
|
if '.onion/' in key_id:
|
|
curr_session = self.server.session_onion
|
|
proxy_type = 'tor'
|
|
if self.server.i2p_domain:
|
|
if '.i2p/' in key_id:
|
|
curr_session = self.server.session_i2p
|
|
proxy_type = 'i2p'
|
|
|
|
curr_session = \
|
|
self._establish_session("secure mode",
|
|
curr_session, proxy_type)
|
|
if not curr_session:
|
|
return False
|
|
|
|
# obtain the public key. key_id is the actor
|
|
pub_key = \
|
|
get_person_pub_key(self.server.base_dir,
|
|
curr_session, key_id,
|
|
self.server.person_cache, self.server.debug,
|
|
self.server.project_version,
|
|
self.server.http_prefix,
|
|
self.server.domain,
|
|
self.server.onion_domain,
|
|
self.server.i2p_domain,
|
|
self.server.signing_priv_key_pem)
|
|
if not pub_key:
|
|
if self.server.debug:
|
|
print('AUTH: secure mode failed to ' +
|
|
'obtain public key for ' + key_id)
|
|
return False
|
|
|
|
# verify the GET request without any digest
|
|
if verify_post_headers(self.server.http_prefix,
|
|
self.server.domain_full,
|
|
pub_key, self.headers,
|
|
self.path, True, None, '', self.server.debug):
|
|
return True
|
|
|
|
if self.server.debug:
|
|
print('AUTH: secure mode authorization failed for ' + key_id)
|
|
return False
|
|
|
|
def _get_account_pub_key(self, path: str, person_cache: {},
|
|
base_dir: str, http_prefix: str,
|
|
domain: str, onion_domain: str,
|
|
i2p_domain: str,
|
|
calling_domain: str) -> str:
|
|
"""Returns the public key for an account
|
|
"""
|
|
if '/users/' not in path:
|
|
return None
|
|
nickname = path.split('/users/')[1]
|
|
if '#main-key' in nickname:
|
|
nickname = nickname.split('#main-key')[0]
|
|
elif '/main-key' in nickname:
|
|
nickname = nickname.split('/main-key')[0]
|
|
elif '#/publicKey' in nickname:
|
|
nickname = nickname.split('#/publicKey')[0]
|
|
else:
|
|
return None
|
|
if calling_domain.endswith('.onion'):
|
|
actor = 'http://' + onion_domain + '/users/' + nickname
|
|
elif calling_domain.endswith('.i2p'):
|
|
actor = 'http://' + i2p_domain + '/users/' + nickname
|
|
else:
|
|
actor = http_prefix + '://' + domain + '/users/' + nickname
|
|
actor_json = get_person_from_cache(base_dir, actor, person_cache)
|
|
if not actor_json:
|
|
actor_filename = acct_dir(base_dir, nickname, domain) + '.json'
|
|
if not os.path.isfile(actor_filename):
|
|
return None
|
|
actor_json = load_json(actor_filename, 1, 1)
|
|
if not actor_json:
|
|
return None
|
|
store_person_in_cache(base_dir, actor, actor_json,
|
|
person_cache, False)
|
|
if not actor_json.get('publicKey'):
|
|
return None
|
|
return actor_json['publicKey']
|
|
|
|
def _login_headers(self, file_format: str, length: int,
|
|
calling_domain: str) -> None:
|
|
self.send_response(200)
|
|
self.send_header('Content-type', file_format)
|
|
self.send_header('Content-Length', str(length))
|
|
self.send_header('Host', calling_domain)
|
|
self.send_header('WWW-Authenticate',
|
|
'title="Login to Epicyon", Basic realm="epicyon"')
|
|
self.end_headers()
|
|
|
|
def _logout_headers(self, file_format: str, length: int,
|
|
calling_domain: str) -> None:
|
|
self.send_response(200)
|
|
self.send_header('Content-type', file_format)
|
|
self.send_header('Content-Length', str(length))
|
|
self.send_header('Set-Cookie', 'epicyon=; SameSite=Strict')
|
|
self.send_header('Host', calling_domain)
|
|
self.send_header('WWW-Authenticate',
|
|
'title="Login to Epicyon", Basic realm="epicyon"')
|
|
self.end_headers()
|
|
|
|
def _quoted_redirect(self, redirect: str) -> str:
|
|
"""hashtag screen urls sometimes contain non-ascii characters which
|
|
need to be url encoded
|
|
"""
|
|
if '/tags/' not in redirect:
|
|
return redirect
|
|
last_str = redirect.split('/')[-1]
|
|
return redirect.replace('/' + last_str, '/' +
|
|
urllib.parse.quote_plus(last_str))
|
|
|
|
def _logout_redirect(self, redirect: str, cookie: str,
|
|
calling_domain: str) -> None:
|
|
if '://' not in redirect:
|
|
if calling_domain.endswith('.onion') and self.server.onion_domain:
|
|
redirect = 'http://' + self.server.onion_domain + redirect
|
|
elif calling_domain.endswith('.i2p') and self.server.i2p_domain:
|
|
redirect = 'http://' + self.server.i2p_domain + redirect
|
|
else:
|
|
redirect = \
|
|
self.server.http_prefix + '://' + \
|
|
self.server.domain_full + redirect
|
|
print('WARN: redirect was not an absolute url, changed to ' +
|
|
redirect)
|
|
|
|
self.send_response(303)
|
|
self.send_header('Set-Cookie', 'epicyon=; SameSite=Strict')
|
|
self.send_header('Location', self._quoted_redirect(redirect))
|
|
self.send_header('Host', calling_domain)
|
|
self.send_header('X-AP-Instance-ID', self.server.instance_id)
|
|
self.send_header('Content-Length', '0')
|
|
self.end_headers()
|
|
|
|
def _set_headers_base(self, file_format: str, length: int, cookie: str,
|
|
calling_domain: str, permissive: bool) -> None:
|
|
self.send_response(200)
|
|
self.send_header('Content-type', file_format)
|
|
if 'image/' in file_format or \
|
|
'audio/' in file_format or \
|
|
'video/' in file_format:
|
|
cache_control = 'public, max-age=84600, immutable'
|
|
self.send_header('Cache-Control', cache_control)
|
|
else:
|
|
self.send_header('Cache-Control', 'public')
|
|
self.send_header('Origin', self.server.domain_full)
|
|
if length > -1:
|
|
self.send_header('Content-Length', str(length))
|
|
if calling_domain:
|
|
self.send_header('Host', calling_domain)
|
|
if permissive:
|
|
self.send_header('Access-Control-Allow-Origin', '*')
|
|
return
|
|
self.send_header('X-AP-Instance-ID', self.server.instance_id)
|
|
self.send_header('X-Clacks-Overhead', self.server.clacks)
|
|
self.send_header('User-Agent',
|
|
'Epicyon/' + __version__ +
|
|
'; +' + self.server.http_prefix + '://' +
|
|
self.server.domain_full + '/')
|
|
if cookie:
|
|
cookie_str = cookie
|
|
if 'HttpOnly;' not in cookie_str:
|
|
if self.server.http_prefix == 'https':
|
|
cookie_str += '; Secure'
|
|
cookie_str += '; HttpOnly; SameSite=Strict'
|
|
self.send_header('Cookie', cookie_str)
|
|
|
|
def _set_headers(self, file_format: str, length: int, cookie: str,
|
|
calling_domain: str, permissive: bool) -> None:
|
|
self._set_headers_base(file_format, length, cookie, calling_domain,
|
|
permissive)
|
|
self.end_headers()
|
|
|
|
def _set_headers_head(self, file_format: str, length: int, etag: str,
|
|
calling_domain: str, permissive: bool,
|
|
last_modified_time_str: str) -> None:
|
|
self._set_headers_base(file_format, length, None, calling_domain,
|
|
permissive)
|
|
if etag:
|
|
self.send_header('ETag', '"' + etag + '"')
|
|
if last_modified_time_str:
|
|
self.send_header('last-modified',
|
|
last_modified_time_str)
|
|
self.end_headers()
|
|
|
|
def _set_headers_etag(self, media_filename: str, file_format: str,
|
|
data, cookie: str, calling_domain: str,
|
|
permissive: bool, last_modified: str) -> None:
|
|
datalen = len(data)
|
|
self._set_headers_base(file_format, datalen, cookie, calling_domain,
|
|
permissive)
|
|
etag = None
|
|
if os.path.isfile(media_filename + '.etag'):
|
|
try:
|
|
with open(media_filename + '.etag', 'r',
|
|
encoding='utf-8') as efile:
|
|
etag = efile.read()
|
|
except OSError:
|
|
print('EX: _set_headers_etag ' +
|
|
'unable to read ' + media_filename + '.etag')
|
|
if not etag:
|
|
etag = md5(data).hexdigest() # nosec
|
|
try:
|
|
with open(media_filename + '.etag', 'w+',
|
|
encoding='utf-8') as efile:
|
|
efile.write(etag)
|
|
except OSError:
|
|
print('EX: _set_headers_etag ' +
|
|
'unable to write ' + media_filename + '.etag')
|
|
# if etag:
|
|
# self.send_header('ETag', '"' + etag + '"')
|
|
if last_modified:
|
|
self.send_header('last-modified', last_modified)
|
|
self.end_headers()
|
|
|
|
def _etag_exists(self, media_filename: str) -> bool:
|
|
"""Does an etag header exist for the given file?
|
|
"""
|
|
etag_header = 'If-None-Match'
|
|
if not self.headers.get(etag_header):
|
|
etag_header = 'if-none-match'
|
|
if not self.headers.get(etag_header):
|
|
etag_header = 'If-none-match'
|
|
|
|
if self.headers.get(etag_header):
|
|
old_etag = self.headers[etag_header].replace('"', '')
|
|
if os.path.isfile(media_filename + '.etag'):
|
|
# load the etag from file
|
|
curr_etag = ''
|
|
try:
|
|
with open(media_filename + '.etag', 'r',
|
|
encoding='utf-8') as efile:
|
|
curr_etag = efile.read()
|
|
except OSError:
|
|
print('EX: _etag_exists unable to read ' +
|
|
str(media_filename))
|
|
if curr_etag and old_etag == curr_etag:
|
|
# The file has not changed
|
|
return True
|
|
return False
|
|
|
|
def _redirect_headers(self, redirect: str, cookie: str,
|
|
calling_domain: str) -> None:
|
|
if '://' not in redirect:
|
|
if calling_domain.endswith('.onion') and self.server.onion_domain:
|
|
redirect = 'http://' + self.server.onion_domain + redirect
|
|
elif calling_domain.endswith('.i2p') and self.server.i2p_domain:
|
|
redirect = 'http://' + self.server.i2p_domain + redirect
|
|
else:
|
|
redirect = \
|
|
self.server.http_prefix + '://' + \
|
|
self.server.domain_full + redirect
|
|
print('WARN: redirect was not an absolute url, changed to ' +
|
|
redirect)
|
|
|
|
self.send_response(303)
|
|
|
|
if cookie:
|
|
cookie_str = cookie.replace('SET:', '').strip()
|
|
if 'HttpOnly;' not in cookie_str:
|
|
if self.server.http_prefix == 'https':
|
|
cookie_str += '; Secure'
|
|
cookie_str += '; HttpOnly; SameSite=Strict'
|
|
if not cookie.startswith('SET:'):
|
|
self.send_header('Cookie', cookie_str)
|
|
else:
|
|
self.send_header('Set-Cookie', cookie_str)
|
|
self.send_header('Location', self._quoted_redirect(redirect))
|
|
self.send_header('Host', calling_domain)
|
|
self.send_header('X-AP-Instance-ID', self.server.instance_id)
|
|
self.send_header('Content-Length', '0')
|
|
self.end_headers()
|
|
|
|
def _http_return_code(self, http_code: int, http_description: str,
|
|
long_description: str, etag: str) -> None:
|
|
msg = \
|
|
'<html><head><title>' + str(http_code) + '</title></head>' \
|
|
'<body bgcolor="linen" text="black">' \
|
|
'<div style="font-size: 400px; ' \
|
|
'text-align: center;">' + str(http_code) + '</div>' \
|
|
'<div style="font-size: 128px; ' \
|
|
'text-align: center; font-variant: ' \
|
|
'small-caps;"><p role="alert">' + http_description + '</p></div>' \
|
|
'<div style="text-align: center;">' + long_description + '</div>' \
|
|
'</body></html>'
|
|
msg = msg.encode('utf-8')
|
|
self.send_response(http_code)
|
|
self.send_header('Content-Type', 'text/html; charset=utf-8')
|
|
msg_len_str = str(len(msg))
|
|
self.send_header('Content-Length', msg_len_str)
|
|
if etag:
|
|
self.send_header('ETag', etag)
|
|
self.end_headers()
|
|
if not self._write(msg):
|
|
print('Error when showing ' + str(http_code))
|
|
|
|
def _200(self) -> None:
|
|
if self.server.translate:
|
|
ok_str = self.server.translate['This is nothing ' +
|
|
'less than an utter triumph']
|
|
self._http_return_code(200, self.server.translate['Ok'],
|
|
ok_str, None)
|
|
else:
|
|
self._http_return_code(200, 'Ok',
|
|
'This is nothing less ' +
|
|
'than an utter triumph', None)
|
|
|
|
def _401(self, post_msg: str) -> None:
|
|
if self.server.translate:
|
|
ok_str = self.server.translate[post_msg]
|
|
self._http_return_code(401, self.server.translate['Unauthorized'],
|
|
ok_str, None)
|
|
else:
|
|
self._http_return_code(401, 'Unauthorized',
|
|
post_msg, None)
|
|
|
|
def _201(self, etag: str) -> None:
|
|
if self.server.translate:
|
|
done_str = self.server.translate['It is done']
|
|
self._http_return_code(201,
|
|
self.server.translate['Created'], done_str,
|
|
etag)
|
|
else:
|
|
self._http_return_code(201, 'Created', 'It is done', etag)
|
|
|
|
def _207(self) -> None:
|
|
if self.server.translate:
|
|
multi_str = self.server.translate['Lots of things']
|
|
self._http_return_code(207,
|
|
self.server.translate['Multi Status'],
|
|
multi_str, None)
|
|
else:
|
|
self._http_return_code(207, 'Multi Status',
|
|
'Lots of things', None)
|
|
|
|
def _403(self) -> None:
|
|
if self.server.translate:
|
|
self._http_return_code(403, self.server.translate['Forbidden'],
|
|
self.server.translate["You're not allowed"],
|
|
None)
|
|
else:
|
|
self._http_return_code(403, 'Forbidden',
|
|
"You're not allowed", None)
|
|
|
|
def _404(self) -> None:
|
|
if self.server.translate:
|
|
self._http_return_code(404, self.server.translate['Not Found'],
|
|
self.server.translate['These are not the ' +
|
|
'droids you are ' +
|
|
'looking for'],
|
|
None)
|
|
else:
|
|
self._http_return_code(404, 'Not Found',
|
|
'These are not the ' +
|
|
'droids you are ' +
|
|
'looking for', None)
|
|
|
|
def _304(self) -> None:
|
|
if self.server.translate:
|
|
self._http_return_code(304, self.server.translate['Not changed'],
|
|
self.server.translate['The contents of ' +
|
|
'your local cache ' +
|
|
'are up to date'],
|
|
None)
|
|
else:
|
|
self._http_return_code(304, 'Not changed',
|
|
'The contents of ' +
|
|
'your local cache ' +
|
|
'are up to date',
|
|
None)
|
|
|
|
def _400(self) -> None:
|
|
if self.server.translate:
|
|
self._http_return_code(400, self.server.translate['Bad Request'],
|
|
self.server.translate['Better luck ' +
|
|
'next time'],
|
|
None)
|
|
else:
|
|
self._http_return_code(400, 'Bad Request',
|
|
'Better luck next time', None)
|
|
|
|
def _503(self) -> None:
|
|
if self.server.translate:
|
|
busy_str = \
|
|
self.server.translate['The server is busy. ' +
|
|
'Please try again later']
|
|
self._http_return_code(503, self.server.translate['Unavailable'],
|
|
busy_str, None)
|
|
else:
|
|
self._http_return_code(503, 'Unavailable',
|
|
'The server is busy. Please try again ' +
|
|
'later', None)
|
|
|
|
def _write(self, msg) -> bool:
|
|
tries = 0
|
|
while tries < 5:
|
|
try:
|
|
self.wfile.write(msg)
|
|
return True
|
|
except BrokenPipeError as ex:
|
|
if self.server.debug:
|
|
print('EX: _write error ' + str(tries) + ' ' + str(ex))
|
|
break
|
|
except BaseException as ex:
|
|
print('EX: _write error ' + str(tries) + ' ' + str(ex))
|
|
time.sleep(0.5)
|
|
tries += 1
|
|
return False
|
|
|
|
def _has_accept(self, calling_domain: str) -> bool:
|
|
"""Do the http headers have an Accept field?
|
|
"""
|
|
if not self.headers.get('Accept'):
|
|
if self.headers.get('accept'):
|
|
print('Upper case Accept')
|
|
self.headers['Accept'] = self.headers['accept']
|
|
|
|
if self.headers.get('Accept') or calling_domain.endswith('.b32.i2p'):
|
|
if not self.headers.get('Accept'):
|
|
self.headers['Accept'] = \
|
|
'text/html,application/xhtml+xml,' \
|
|
'application/xml;q=0.9,image/webp,*/*;q=0.8'
|
|
return True
|
|
return False
|
|
|
|
def _masto_api_v1(self, path: str, calling_domain: str,
|
|
ua_str: str,
|
|
authorized: bool,
|
|
http_prefix: str,
|
|
base_dir: str, nickname: str, domain: str,
|
|
domain_full: str,
|
|
onion_domain: str, i2p_domain: str,
|
|
translate: {},
|
|
registration: bool,
|
|
system_language: str,
|
|
project_version: str,
|
|
custom_emoji: [],
|
|
show_node_info_accounts: bool,
|
|
referer_domain: str,
|
|
debug: bool,
|
|
calling_site_timeout: int,
|
|
known_crawlers: {}) -> bool:
|
|
"""This is a vestigil mastodon API for the purpose
|
|
of returning an empty result to sites like
|
|
https://mastopeek.app-dist.eu
|
|
"""
|
|
if not path.startswith('/api/v1/'):
|
|
return False
|
|
|
|
if not referer_domain:
|
|
if not (debug and self.server.unit_test):
|
|
print('mastodon api request has no referer domain ' +
|
|
str(ua_str))
|
|
self._400()
|
|
return True
|
|
if referer_domain == self.server.domain_full:
|
|
print('mastodon api request from self')
|
|
self._400()
|
|
return True
|
|
if self.server.masto_api_is_active:
|
|
print('mastodon api is busy during request from ' +
|
|
referer_domain)
|
|
self._503()
|
|
return True
|
|
self.server.masto_api_is_active = True
|
|
# is this a real website making the call ?
|
|
if not debug and not self.server.unit_test and referer_domain:
|
|
# Does calling_domain look like a domain?
|
|
if ' ' in referer_domain or \
|
|
';' in referer_domain or \
|
|
'.' not in referer_domain:
|
|
print('mastodon api ' +
|
|
'referer does not look like a domain ' +
|
|
referer_domain)
|
|
self._400()
|
|
self.server.masto_api_is_active = False
|
|
return True
|
|
if not self.server.allow_local_network_access:
|
|
if local_network_host(referer_domain):
|
|
print('mastodon api referer domain is from the ' +
|
|
'local network ' + referer_domain)
|
|
self._400()
|
|
self.server.masto_api_is_active = False
|
|
return True
|
|
if not referer_is_active(http_prefix,
|
|
referer_domain, ua_str,
|
|
calling_site_timeout):
|
|
print('mastodon api referer url is not active ' +
|
|
referer_domain)
|
|
self._400()
|
|
self.server.masto_api_is_active = False
|
|
return True
|
|
|
|
print('mastodon api v1: ' + path)
|
|
print('mastodon api v1: authorized ' + str(authorized))
|
|
print('mastodon api v1: nickname ' + str(nickname))
|
|
print('mastodon api v1: referer ' + str(referer_domain))
|
|
crawl_time = \
|
|
update_known_crawlers(ua_str, base_dir,
|
|
self.server.known_crawlers,
|
|
self.server.last_known_crawler)
|
|
if crawl_time is not None:
|
|
self.server.last_known_crawler = crawl_time
|
|
|
|
broch_mode = broch_mode_is_active(base_dir)
|
|
send_json, send_json_str = \
|
|
masto_api_v1_response(path,
|
|
calling_domain,
|
|
ua_str,
|
|
authorized,
|
|
http_prefix,
|
|
base_dir,
|
|
nickname, domain,
|
|
domain_full,
|
|
onion_domain,
|
|
i2p_domain,
|
|
translate,
|
|
registration,
|
|
system_language,
|
|
project_version,
|
|
custom_emoji,
|
|
show_node_info_accounts,
|
|
broch_mode)
|
|
|
|
if send_json is not None:
|
|
msg_str = json.dumps(send_json)
|
|
msg_str = self._convert_domains(calling_domain, referer_domain,
|
|
msg_str)
|
|
msg = msg_str.encode('utf-8')
|
|
msglen = len(msg)
|
|
if self._has_accept(calling_domain):
|
|
protocol_str = \
|
|
get_json_content_from_accept(self.headers.get('Accept'))
|
|
self._set_headers(protocol_str, msglen,
|
|
None, calling_domain, True)
|
|
else:
|
|
self._set_headers('application/ld+json', msglen,
|
|
None, calling_domain, True)
|
|
self._write(msg)
|
|
if send_json_str:
|
|
print(send_json_str)
|
|
self.server.masto_api_is_active = False
|
|
return True
|
|
|
|
# no api endpoints were matched
|
|
self._404()
|
|
self.server.masto_api_is_active = False
|
|
return True
|
|
|
|
def _masto_api(self, path: str, calling_domain: str,
|
|
ua_str: str,
|
|
authorized: bool, http_prefix: str,
|
|
base_dir: str, nickname: str, domain: str,
|
|
domain_full: str,
|
|
onion_domain: str, i2p_domain: str,
|
|
translate: {},
|
|
registration: bool,
|
|
system_language: str,
|
|
project_version: str,
|
|
custom_emoji: [],
|
|
show_node_info_accounts: bool,
|
|
referer_domain: str, debug: bool,
|
|
known_crawlers: {}) -> bool:
|
|
return self._masto_api_v1(path, calling_domain, ua_str, authorized,
|
|
http_prefix, base_dir, nickname, domain,
|
|
domain_full, onion_domain, i2p_domain,
|
|
translate, registration, system_language,
|
|
project_version, custom_emoji,
|
|
show_node_info_accounts,
|
|
referer_domain, debug, 5,
|
|
known_crawlers)
|
|
|
|
def _show_vcard(self, base_dir: str, path: str, calling_domain: str,
|
|
referer_domain: str, domain: str) -> bool:
|
|
"""Returns a vcard for the given account
|
|
"""
|
|
if not self._has_accept(calling_domain):
|
|
return False
|
|
if path.endswith('.vcf'):
|
|
path = path.split('.vcf')[0]
|
|
accept_str = 'text/vcard'
|
|
else:
|
|
accept_str = self.headers['Accept']
|
|
if 'text/vcard' not in accept_str and \
|
|
'application/vcard+xml' not in accept_str:
|
|
return False
|
|
if path.startswith('/@'):
|
|
path = path.replace('/@', '/users/', 1)
|
|
if not path.startswith('/users/'):
|
|
self._400()
|
|
return True
|
|
nickname = path.split('/users/')[1]
|
|
if '/' in nickname:
|
|
nickname = nickname.split('/')[0]
|
|
if '?' in nickname:
|
|
nickname = nickname.split('?')[0]
|
|
if self.server.vcard_is_active:
|
|
print('vcard is busy during request from ' + str(referer_domain))
|
|
self._503()
|
|
return True
|
|
self.server.vcard_is_active = True
|
|
actor_json = None
|
|
actor_filename = \
|
|
acct_dir(base_dir, nickname, domain) + '.json'
|
|
if os.path.isfile(actor_filename):
|
|
actor_json = load_json(actor_filename)
|
|
if not actor_json:
|
|
print('WARN: vcard actor not found ' + actor_filename)
|
|
self._404()
|
|
self.server.vcard_is_active = False
|
|
return True
|
|
if 'application/vcard+xml' in accept_str:
|
|
vcard_str = actor_to_vcard_xml(actor_json, domain)
|
|
header_type = 'application/vcard+xml; charset=utf-8'
|
|
else:
|
|
vcard_str = actor_to_vcard(actor_json, domain)
|
|
header_type = 'text/vcard; charset=utf-8'
|
|
if vcard_str:
|
|
msg = vcard_str.encode('utf-8')
|
|
msglen = len(msg)
|
|
self._set_headers(header_type, msglen,
|
|
None, calling_domain, True)
|
|
self._write(msg)
|
|
print('vcard sent to ' + str(referer_domain))
|
|
self.server.vcard_is_active = False
|
|
return True
|
|
print('WARN: vcard string not returned')
|
|
self._404()
|
|
self.server.vcard_is_active = False
|
|
return True
|
|
|
|
def _nodeinfo(self, ua_str: str, calling_domain: str,
|
|
referer_domain: str,
|
|
http_prefix: str, calling_site_timeout: int,
|
|
debug: bool) -> bool:
|
|
if self.path.startswith('/nodeinfo/1.0'):
|
|
self._400()
|
|
return True
|
|
if not self.path.startswith('/nodeinfo/2.0'):
|
|
return False
|
|
if not referer_domain:
|
|
if not debug and not self.server.unit_test:
|
|
print('nodeinfo request has no referer domain ' + str(ua_str))
|
|
self._400()
|
|
return True
|
|
if referer_domain == self.server.domain_full:
|
|
print('nodeinfo request from self')
|
|
self._400()
|
|
return True
|
|
if self.server.nodeinfo_is_active:
|
|
if not referer_domain:
|
|
print('nodeinfo is busy during request without referer domain')
|
|
else:
|
|
print('nodeinfo is busy during request from ' + referer_domain)
|
|
self._503()
|
|
return True
|
|
self.server.nodeinfo_is_active = True
|
|
# is this a real website making the call ?
|
|
if not debug and not self.server.unit_test and referer_domain:
|
|
# Does calling_domain look like a domain?
|
|
if ' ' in referer_domain or \
|
|
';' in referer_domain or \
|
|
'.' not in referer_domain:
|
|
print('nodeinfo referer domain does not look like a domain ' +
|
|
referer_domain)
|
|
self._400()
|
|
self.server.nodeinfo_is_active = False
|
|
return True
|
|
if not self.server.allow_local_network_access:
|
|
if local_network_host(referer_domain):
|
|
print('nodeinfo referer domain is from the ' +
|
|
'local network ' + referer_domain)
|
|
self._400()
|
|
self.server.nodeinfo_is_active = False
|
|
return True
|
|
|
|
if not referer_is_active(http_prefix,
|
|
referer_domain, ua_str,
|
|
calling_site_timeout):
|
|
print('nodeinfo referer url is not active ' +
|
|
referer_domain)
|
|
self._400()
|
|
self.server.nodeinfo_is_active = False
|
|
return True
|
|
if self.server.debug:
|
|
print('DEBUG: nodeinfo ' + self.path)
|
|
crawl_time = \
|
|
update_known_crawlers(ua_str,
|
|
self.server.base_dir,
|
|
self.server.known_crawlers,
|
|
self.server.last_known_crawler)
|
|
if crawl_time is not None:
|
|
self.server.last_known_crawler = crawl_time
|
|
|
|
# If we are in broch mode then don't show potentially
|
|
# sensitive metadata.
|
|
# For example, if this or allied instances are being attacked
|
|
# then numbers of accounts may be changing as people
|
|
# migrate, and that information may be useful to an adversary
|
|
broch_mode = broch_mode_is_active(self.server.base_dir)
|
|
|
|
node_info_version = self.server.project_version
|
|
if not self.server.show_node_info_version or broch_mode:
|
|
node_info_version = '0.0.0'
|
|
|
|
show_node_info_accounts = self.server.show_node_info_accounts
|
|
if broch_mode:
|
|
show_node_info_accounts = False
|
|
|
|
instance_url = self._get_instance_url(calling_domain)
|
|
about_url = instance_url + '/about'
|
|
terms_of_service_url = instance_url + '/terms'
|
|
info = meta_data_node_info(self.server.base_dir,
|
|
about_url, terms_of_service_url,
|
|
self.server.registration,
|
|
node_info_version,
|
|
show_node_info_accounts)
|
|
if info:
|
|
msg_str = json.dumps(info)
|
|
msg_str = self._convert_domains(calling_domain, referer_domain,
|
|
msg_str)
|
|
msg = msg_str.encode('utf-8')
|
|
msglen = len(msg)
|
|
if self._has_accept(calling_domain):
|
|
protocol_str = \
|
|
get_json_content_from_accept(self.headers.get('Accept'))
|
|
self._set_headers(protocol_str, msglen,
|
|
None, calling_domain, True)
|
|
else:
|
|
self._set_headers('application/ld+json', msglen,
|
|
None, calling_domain, True)
|
|
self._write(msg)
|
|
if referer_domain:
|
|
print('nodeinfo sent to ' + referer_domain)
|
|
else:
|
|
print('nodeinfo sent to unknown referer')
|
|
self.server.nodeinfo_is_active = False
|
|
return True
|
|
self._404()
|
|
self.server.nodeinfo_is_active = False
|
|
return True
|
|
|
|
def _webfinger(self, calling_domain: str, referer_domain: str) -> bool:
|
|
if not self.path.startswith('/.well-known'):
|
|
return False
|
|
if self.server.debug:
|
|
print('DEBUG: WEBFINGER well-known')
|
|
|
|
if self.server.debug:
|
|
print('DEBUG: WEBFINGER host-meta')
|
|
if self.path.startswith('/.well-known/host-meta'):
|
|
if calling_domain.endswith('.onion') and \
|
|
self.server.onion_domain:
|
|
wf_result = \
|
|
webfinger_meta('http', self.server.onion_domain)
|
|
elif (calling_domain.endswith('.i2p') and
|
|
self.server.i2p_domain):
|
|
wf_result = \
|
|
webfinger_meta('http', self.server.i2p_domain)
|
|
else:
|
|
wf_result = \
|
|
webfinger_meta(self.server.http_prefix,
|
|
self.server.domain_full)
|
|
if wf_result:
|
|
msg = wf_result.encode('utf-8')
|
|
msglen = len(msg)
|
|
self._set_headers('application/xrd+xml', msglen,
|
|
None, calling_domain, True)
|
|
self._write(msg)
|
|
return True
|
|
self._404()
|
|
return True
|
|
if self.path.startswith('/api/statusnet') or \
|
|
self.path.startswith('/api/gnusocial') or \
|
|
self.path.startswith('/siteinfo') or \
|
|
self.path.startswith('/poco') or \
|
|
self.path.startswith('/friendi'):
|
|
self._404()
|
|
return True
|
|
if self.path.startswith('/.well-known/nodeinfo') or \
|
|
self.path.startswith('/.well-known/x-nodeinfo'):
|
|
if calling_domain.endswith('.onion') and \
|
|
self.server.onion_domain:
|
|
wf_result = \
|
|
webfinger_node_info('http', self.server.onion_domain)
|
|
elif (calling_domain.endswith('.i2p') and
|
|
self.server.i2p_domain):
|
|
wf_result = \
|
|
webfinger_node_info('http', self.server.i2p_domain)
|
|
else:
|
|
wf_result = \
|
|
webfinger_node_info(self.server.http_prefix,
|
|
self.server.domain_full)
|
|
if wf_result:
|
|
msg_str = json.dumps(wf_result)
|
|
msg_str = self._convert_domains(calling_domain,
|
|
referer_domain,
|
|
msg_str)
|
|
msg = msg_str.encode('utf-8')
|
|
msglen = len(msg)
|
|
if self._has_accept(calling_domain):
|
|
accept_str = self.headers.get('Accept')
|
|
protocol_str = \
|
|
get_json_content_from_accept(accept_str)
|
|
self._set_headers(protocol_str, msglen,
|
|
None, calling_domain, True)
|
|
else:
|
|
self._set_headers('application/ld+json', msglen,
|
|
None, calling_domain, True)
|
|
self._write(msg)
|
|
return True
|
|
self._404()
|
|
return True
|
|
|
|
if self.server.debug:
|
|
print('DEBUG: WEBFINGER lookup ' + self.path + ' ' +
|
|
str(self.server.base_dir))
|
|
wf_result = \
|
|
webfinger_lookup(self.path, self.server.base_dir,
|
|
self.server.domain,
|
|
self.server.onion_domain,
|
|
self.server.i2p_domain,
|
|
self.server.port, self.server.debug)
|
|
if wf_result:
|
|
msg_str = json.dumps(wf_result)
|
|
msg_str = self._convert_domains(calling_domain,
|
|
referer_domain,
|
|
msg_str)
|
|
msg = msg_str.encode('utf-8')
|
|
msglen = len(msg)
|
|
self._set_headers('application/jrd+json', msglen,
|
|
None, calling_domain, True)
|
|
self._write(msg)
|
|
else:
|
|
if self.server.debug:
|
|
print('DEBUG: WEBFINGER lookup 404 ' + self.path)
|
|
self._404()
|
|
return True
|
|
|
|
def _post_to_outbox(self, message_json: {}, version: str,
|
|
post_to_nickname: str,
|
|
curr_session, proxy_type: str) -> bool:
|
|
"""post is received by the outbox
|
|
Client to server message post
|
|
https://www.w3.org/TR/activitypub/#client-to-server-outbox-delivery
|
|
"""
|
|
if not curr_session:
|
|
return False
|
|
|
|
city = self.server.city
|
|
|
|
if post_to_nickname:
|
|
print('Posting to nickname ' + post_to_nickname)
|
|
self.post_to_nickname = post_to_nickname
|
|
city = get_spoofed_city(self.server.city,
|
|
self.server.base_dir,
|
|
post_to_nickname, self.server.domain)
|
|
|
|
shared_items_federated_domains = \
|
|
self.server.shared_items_federated_domains
|
|
shared_item_federation_tokens = \
|
|
self.server.shared_item_federation_tokens
|
|
return post_message_to_outbox(curr_session,
|
|
self.server.translate,
|
|
message_json, self.post_to_nickname,
|
|
self.server, self.server.base_dir,
|
|
self.server.http_prefix,
|
|
self.server.domain,
|
|
self.server.domain_full,
|
|
self.server.onion_domain,
|
|
self.server.i2p_domain,
|
|
self.server.port,
|
|
self.server.recent_posts_cache,
|
|
self.server.followers_threads,
|
|
self.server.federation_list,
|
|
self.server.send_threads,
|
|
self.server.postLog,
|
|
self.server.cached_webfingers,
|
|
self.server.person_cache,
|
|
self.server.allow_deletion,
|
|
proxy_type, version,
|
|
self.server.debug,
|
|
self.server.yt_replace_domain,
|
|
self.server.twitter_replacement_domain,
|
|
self.server.show_published_date_only,
|
|
self.server.allow_local_network_access,
|
|
city, self.server.system_language,
|
|
shared_items_federated_domains,
|
|
shared_item_federation_tokens,
|
|
self.server.low_bandwidth,
|
|
self.server.signing_priv_key_pem,
|
|
self.server.peertube_instances,
|
|
self.server.theme_name,
|
|
self.server.max_like_count,
|
|
self.server.max_recent_posts,
|
|
self.server.cw_lists,
|
|
self.server.lists_enabled,
|
|
self.server.content_license_url,
|
|
self.server.dogwhistles)
|
|
|
|
def _get_outbox_thread_index(self, nickname: str,
|
|
max_outbox_threads_per_account: int) -> int:
|
|
"""Returns the outbox thread index for the given account
|
|
This is a ring buffer used to store the thread objects which
|
|
are sending out posts
|
|
"""
|
|
account_outbox_thread_name = nickname
|
|
if not account_outbox_thread_name:
|
|
account_outbox_thread_name = '*'
|
|
|
|
# create the buffer for the given account
|
|
if not self.server.outboxThread.get(account_outbox_thread_name):
|
|
self.server.outboxThread[account_outbox_thread_name] = \
|
|
[None] * max_outbox_threads_per_account
|
|
self.server.outbox_thread_index[account_outbox_thread_name] = 0
|
|
return 0
|
|
|
|
# increment the ring buffer index
|
|
index = self.server.outbox_thread_index[account_outbox_thread_name] + 1
|
|
if index >= max_outbox_threads_per_account:
|
|
index = 0
|
|
|
|
self.server.outbox_thread_index[account_outbox_thread_name] = index
|
|
|
|
# remove any existing thread from the current index in the buffer
|
|
acct = account_outbox_thread_name
|
|
if self.server.outboxThread.get(acct):
|
|
if len(self.server.outboxThread[acct]) > index:
|
|
try:
|
|
if self.server.outboxThread[acct][index].is_alive():
|
|
self.server.outboxThread[acct][index].kill()
|
|
except BaseException:
|
|
pass
|
|
return index
|
|
|
|
def _post_to_outbox_thread(self, message_json: {},
|
|
curr_session, proxy_type: str) -> bool:
|
|
"""Creates a thread to send a post
|
|
"""
|
|
account_outbox_thread_name = self.post_to_nickname
|
|
if not account_outbox_thread_name:
|
|
account_outbox_thread_name = '*'
|
|
|
|
index = self._get_outbox_thread_index(account_outbox_thread_name, 8)
|
|
|
|
print('Creating outbox thread ' +
|
|
account_outbox_thread_name + '/' +
|
|
str(self.server.outbox_thread_index[account_outbox_thread_name]))
|
|
print('THREAD: _post_to_outbox')
|
|
self.server.outboxThread[account_outbox_thread_name][index] = \
|
|
thread_with_trace(target=self._post_to_outbox,
|
|
args=(message_json.copy(),
|
|
self.server.project_version, None,
|
|
curr_session, proxy_type),
|
|
daemon=True)
|
|
print('Starting outbox thread')
|
|
outbox_thread = \
|
|
self.server.outboxThread[account_outbox_thread_name][index]
|
|
begin_thread(outbox_thread, '_post_to_outbox_thread')
|
|
return True
|
|
|
|
def _update_inbox_queue(self, nickname: str, message_json: {},
|
|
message_bytes: str, debug: bool) -> int:
|
|
"""Update the inbox queue
|
|
"""
|
|
if debug:
|
|
print('INBOX: checking inbox queue restart')
|
|
if self.server.restart_inbox_queue_in_progress:
|
|
self._503()
|
|
print('INBOX: ' +
|
|
'message arrived but currently restarting inbox queue')
|
|
self.server.postreq_busy = False
|
|
return 2
|
|
|
|
# check that the incoming message has a fully recognized
|
|
# linked data context
|
|
if debug:
|
|
print('INBOX: checking valid context')
|
|
if not has_valid_context(message_json):
|
|
print('INBOX: ' +
|
|
'message arriving at inbox queue has no valid context')
|
|
self._400()
|
|
self.server.postreq_busy = False
|
|
return 3
|
|
|
|
# check for blocked domains so that they can be rejected early
|
|
if debug:
|
|
print('INBOX: checking for actor')
|
|
message_domain = None
|
|
if not has_actor(message_json, self.server.debug):
|
|
print('INBOX: message arriving at inbox queue has no actor')
|
|
self._400()
|
|
self.server.postreq_busy = False
|
|
return 3
|
|
|
|
# actor should be a string
|
|
if debug:
|
|
print('INBOX: checking that actor is string')
|
|
if not isinstance(message_json['actor'], str):
|
|
print('INBOX: ' +
|
|
'actor should be a string ' + str(message_json['actor']))
|
|
self._400()
|
|
self.server.postreq_busy = False
|
|
return 3
|
|
|
|
# check that some additional fields are strings
|
|
if debug:
|
|
print('INBOX: checking fields 1')
|
|
string_fields = ('id', 'type', 'published')
|
|
for check_field in string_fields:
|
|
if not message_json.get(check_field):
|
|
continue
|
|
if not isinstance(message_json[check_field], str):
|
|
print('INBOX: ' +
|
|
'id, type and published fields should be strings ' +
|
|
check_field + ' ' + str(message_json[check_field]))
|
|
self._400()
|
|
self.server.postreq_busy = False
|
|
return 3
|
|
|
|
# check that to/cc fields are lists
|
|
if debug:
|
|
print('INBOX: checking to and cc fields')
|
|
list_fields = ('to', 'cc')
|
|
for check_field in list_fields:
|
|
if not message_json.get(check_field):
|
|
continue
|
|
if not isinstance(message_json[check_field], list):
|
|
print('INBOX: To and Cc fields should be strings ' +
|
|
check_field + ' ' + str(message_json[check_field]))
|
|
self._400()
|
|
self.server.postreq_busy = False
|
|
return 3
|
|
|
|
if has_object_dict(message_json):
|
|
if debug:
|
|
print('INBOX: checking object fields')
|
|
string_fields = (
|
|
'id', 'actor', 'type', 'content', 'published',
|
|
'summary', 'url', 'attributedTo'
|
|
)
|
|
for check_field in string_fields:
|
|
if not message_json['object'].get(check_field):
|
|
continue
|
|
if not isinstance(message_json['object'][check_field], str):
|
|
print('INBOX: ' +
|
|
check_field + ' should be a string ' +
|
|
str(message_json['object'][check_field]))
|
|
self._400()
|
|
self.server.postreq_busy = False
|
|
return 3
|
|
# check that some fields are lists
|
|
if debug:
|
|
print('INBOX: checking object to and cc fields')
|
|
list_fields = ('to', 'cc', 'attachment')
|
|
for check_field in list_fields:
|
|
if not message_json['object'].get(check_field):
|
|
continue
|
|
if not isinstance(message_json['object'][check_field], list):
|
|
print('INBOX: ' +
|
|
check_field + ' should be a list ' +
|
|
str(message_json['object'][check_field]))
|
|
self._400()
|
|
self.server.postreq_busy = False
|
|
return 3
|
|
# check that the content does not contain impossibly long urls
|
|
if message_json['object'].get('content'):
|
|
content_str = message_json['object']['content']
|
|
if not valid_url_lengths(content_str, 2048):
|
|
print('INBOX: content contains urls which are too long ' +
|
|
message_json['actor'])
|
|
self._400()
|
|
self.server.postreq_busy = False
|
|
return 3
|
|
# check that the summary does not contain links
|
|
if message_json['object'].get('summary'):
|
|
if len(message_json['object']['summary']) > 1024:
|
|
print('INBOX: summary is too long ' +
|
|
message_json['actor'] + ' ' +
|
|
message_json['object']['summary'])
|
|
self._400()
|
|
self.server.postreq_busy = False
|
|
return 3
|
|
if '://' in message_json['object']['summary']:
|
|
print('INBOX: summary should not contain links ' +
|
|
message_json['actor'] + ' ' +
|
|
message_json['object']['summary'])
|
|
self._400()
|
|
self.server.postreq_busy = False
|
|
return 3
|
|
|
|
# actor should look like a url
|
|
if debug:
|
|
print('INBOX: checking that actor looks like a url')
|
|
if '://' not in message_json['actor'] or \
|
|
'.' not in message_json['actor']:
|
|
print('INBOX: POST actor does not look like a url ' +
|
|
message_json['actor'])
|
|
self._400()
|
|
self.server.postreq_busy = False
|
|
return 3
|
|
|
|
# sent by an actor on a local network address?
|
|
if debug:
|
|
print('INBOX: checking for local network access')
|
|
if not self.server.allow_local_network_access:
|
|
local_network_pattern_list = get_local_network_addresses()
|
|
for local_network_pattern in local_network_pattern_list:
|
|
if local_network_pattern in message_json['actor']:
|
|
print('INBOX: POST actor contains local network address ' +
|
|
message_json['actor'])
|
|
self._400()
|
|
self.server.postreq_busy = False
|
|
return 3
|
|
|
|
message_domain, _ = \
|
|
get_domain_from_actor(message_json['actor'])
|
|
|
|
self.server.blocked_cache_last_updated = \
|
|
update_blocked_cache(self.server.base_dir,
|
|
self.server.blocked_cache,
|
|
self.server.blocked_cache_last_updated,
|
|
self.server.blocked_cache_update_secs)
|
|
|
|
if debug:
|
|
print('INBOX: checking for blocked domain ' + message_domain)
|
|
if is_blocked_domain(self.server.base_dir, message_domain,
|
|
self.server.blocked_cache):
|
|
print('INBOX: POST from blocked domain ' + message_domain)
|
|
self._400()
|
|
self.server.postreq_busy = False
|
|
return 3
|
|
|
|
# if the inbox queue is full then return a busy code
|
|
if debug:
|
|
print('INBOX: checking for full queue')
|
|
if len(self.server.inbox_queue) >= self.server.max_queue_length:
|
|
if message_domain:
|
|
print('INBOX: Queue: ' +
|
|
'Inbox queue is full. Incoming post from ' +
|
|
message_json['actor'])
|
|
else:
|
|
print('INBOX: Queue: Inbox queue is full')
|
|
self._503()
|
|
clear_queue_items(self.server.base_dir, self.server.inbox_queue)
|
|
if not self.server.restart_inbox_queue_in_progress:
|
|
self.server.restart_inbox_queue = True
|
|
self.server.postreq_busy = False
|
|
return 2
|
|
|
|
# Convert the headers needed for signature verification to dict
|
|
headers_dict = {}
|
|
headers_dict['host'] = self.headers['host']
|
|
headers_dict['signature'] = self.headers['signature']
|
|
if self.headers.get('Date'):
|
|
headers_dict['Date'] = self.headers['Date']
|
|
elif self.headers.get('date'):
|
|
headers_dict['Date'] = self.headers['date']
|
|
if self.headers.get('digest'):
|
|
headers_dict['digest'] = self.headers['digest']
|
|
if self.headers.get('Collection-Synchronization'):
|
|
headers_dict['Collection-Synchronization'] = \
|
|
self.headers['Collection-Synchronization']
|
|
if self.headers.get('Content-type'):
|
|
headers_dict['Content-type'] = self.headers['Content-type']
|
|
if self.headers.get('Content-Length'):
|
|
headers_dict['Content-Length'] = self.headers['Content-Length']
|
|
elif self.headers.get('content-length'):
|
|
headers_dict['content-length'] = self.headers['content-length']
|
|
|
|
original_message_json = message_json.copy()
|
|
|
|
# whether to add a 'to' field to the message
|
|
add_to_field_types = (
|
|
'Follow', 'Like', 'EmojiReact', 'Add', 'Remove', 'Ignore'
|
|
)
|
|
for add_to_type in add_to_field_types:
|
|
message_json, _ = \
|
|
add_to_field(add_to_type, message_json, self.server.debug)
|
|
|
|
begin_save_time = time.time()
|
|
# save the json for later queue processing
|
|
message_bytes_decoded = message_bytes.decode('utf-8')
|
|
|
|
if debug:
|
|
print('INBOX: checking for invalid links')
|
|
if contains_invalid_local_links(message_bytes_decoded):
|
|
print('INBOX: post contains invalid local links ' +
|
|
str(original_message_json))
|
|
return 5
|
|
|
|
self.server.blocked_cache_last_updated = \
|
|
update_blocked_cache(self.server.base_dir,
|
|
self.server.blocked_cache,
|
|
self.server.blocked_cache_last_updated,
|
|
self.server.blocked_cache_update_secs)
|
|
|
|
mitm = self._detect_mitm()
|
|
|
|
if debug:
|
|
print('INBOX: saving post to queue')
|
|
queue_filename = \
|
|
save_post_to_inbox_queue(self.server.base_dir,
|
|
self.server.http_prefix,
|
|
nickname,
|
|
self.server.domain_full,
|
|
message_json, original_message_json,
|
|
message_bytes_decoded,
|
|
headers_dict,
|
|
self.path,
|
|
self.server.debug,
|
|
self.server.blocked_cache,
|
|
self.server.system_language,
|
|
mitm)
|
|
if queue_filename:
|
|
# add json to the queue
|
|
if queue_filename not in self.server.inbox_queue:
|
|
self.server.inbox_queue.append(queue_filename)
|
|
if self.server.debug:
|
|
time_diff = int((time.time() - begin_save_time) * 1000)
|
|
if time_diff > 200:
|
|
print('SLOW: slow save of inbox queue item ' +
|
|
queue_filename + ' took ' + str(time_diff) + ' mS')
|
|
self.send_response(201)
|
|
self.end_headers()
|
|
self.server.postreq_busy = False
|
|
return 0
|
|
self._503()
|
|
self.server.postreq_busy = False
|
|
return 1
|
|
|
|
def _is_authorized(self) -> bool:
|
|
self.authorized_nickname = None
|
|
|
|
not_auth_paths = (
|
|
'/icons/', '/avatars/', '/favicons/',
|
|
'/system/accounts/avatars/',
|
|
'/system/accounts/headers/',
|
|
'/system/media_attachments/files/',
|
|
'/accounts/avatars/', '/accounts/headers/',
|
|
'/favicon.ico', '/newswire.xml',
|
|
'/newswire_favicon.ico', '/categories.xml'
|
|
)
|
|
for not_auth_str in not_auth_paths:
|
|
if self.path.startswith(not_auth_str):
|
|
return False
|
|
|
|
# token based authenticated used by the web interface
|
|
if self.headers.get('Cookie'):
|
|
if self.headers['Cookie'].startswith('epicyon='):
|
|
token_str = self.headers['Cookie'].split('=', 1)[1].strip()
|
|
if ';' in token_str:
|
|
token_str = token_str.split(';')[0].strip()
|
|
if self.server.tokens_lookup.get(token_str):
|
|
nickname = self.server.tokens_lookup[token_str]
|
|
if not is_system_account(nickname):
|
|
self.authorized_nickname = nickname
|
|
# default to the inbox of the person
|
|
if self.path == '/':
|
|
self.path = '/users/' + nickname + '/inbox'
|
|
# check that the path contains the same nickname
|
|
# as the cookie otherwise it would be possible
|
|
# to be authorized to use an account you don't own
|
|
if '/' + nickname + '/' in self.path:
|
|
return True
|
|
if '/' + nickname + '?' in self.path:
|
|
return True
|
|
if self.path.endswith('/' + nickname):
|
|
return True
|
|
if self.server.debug:
|
|
print('AUTH: nickname ' + nickname +
|
|
' was not found in path ' + self.path)
|
|
return False
|
|
print('AUTH: epicyon cookie ' +
|
|
'authorization failed, header=' +
|
|
self.headers['Cookie'].replace('epicyon=', '') +
|
|
' token_str=' + token_str)
|
|
return False
|
|
print('AUTH: Header cookie was not authorized')
|
|
return False
|
|
# basic auth for c2s
|
|
if self.headers.get('Authorization'):
|
|
if authorize(self.server.base_dir, self.path,
|
|
self.headers['Authorization'],
|
|
self.server.debug):
|
|
return True
|
|
print('AUTH: C2S Basic auth did not authorize ' +
|
|
self.headers['Authorization'])
|
|
return False
|
|
|
|
def _clear_login_details(self, nickname: str, calling_domain: str) -> None:
|
|
"""Clears login details for the given account
|
|
"""
|
|
# remove any token
|
|
if self.server.tokens.get(nickname):
|
|
del self.server.tokens_lookup[self.server.tokens[nickname]]
|
|
del self.server.tokens[nickname]
|
|
self._redirect_headers(self.server.http_prefix + '://' +
|
|
self.server.domain_full + '/login',
|
|
'epicyon=; SameSite=Strict',
|
|
calling_domain)
|
|
|
|
def _post_login_screen(self, calling_domain: str, cookie: str,
|
|
base_dir: str, http_prefix: str,
|
|
domain: str, domain_full: str, port: int,
|
|
onion_domain: str, i2p_domain: str,
|
|
ua_str: str, debug: bool) -> None:
|
|
"""POST to login screen, containing credentials
|
|
"""
|
|
# ensure that there is a minimum delay between failed login
|
|
# attempts, to mitigate brute force
|
|
if int(time.time()) - self.server.last_login_failure < 5:
|
|
self._503()
|
|
self.server.postreq_busy = False
|
|
return
|
|
|
|
# get the contents of POST containing login credentials
|
|
length = int(self.headers['Content-length'])
|
|
if length > 512:
|
|
print('Login failed - credentials too long')
|
|
self._401('Credentials are too long')
|
|
self.server.postreq_busy = False
|
|
return
|
|
|
|
try:
|
|
login_params = self.rfile.read(length).decode('utf-8')
|
|
except SocketError as ex:
|
|
if ex.errno == errno.ECONNRESET:
|
|
print('EX: POST login read ' +
|
|
'connection reset by peer')
|
|
else:
|
|
print('EX: POST login read socket error')
|
|
self.send_response(400)
|
|
self.end_headers()
|
|
self.server.postreq_busy = False
|
|
return
|
|
except ValueError as ex:
|
|
print('EX: POST login read failed, ' + str(ex))
|
|
self.send_response(400)
|
|
self.end_headers()
|
|
self.server.postreq_busy = False
|
|
return
|
|
|
|
login_nickname, login_password, register = \
|
|
html_get_login_credentials(login_params,
|
|
self.server.last_login_time,
|
|
domain)
|
|
if login_nickname and login_password:
|
|
if is_system_account(login_nickname):
|
|
print('Invalid username login: ' + login_nickname +
|
|
' (system account)')
|
|
self._clear_login_details(login_nickname, calling_domain)
|
|
self.server.postreq_busy = False
|
|
return
|
|
self.server.last_login_time = int(time.time())
|
|
if register:
|
|
if not valid_password(login_password):
|
|
self.server.postreq_busy = False
|
|
if calling_domain.endswith('.onion') and onion_domain:
|
|
self._redirect_headers('http://' + onion_domain +
|
|
'/login', cookie,
|
|
calling_domain)
|
|
elif (calling_domain.endswith('.i2p') and i2p_domain):
|
|
self._redirect_headers('http://' + i2p_domain +
|
|
'/login', cookie,
|
|
calling_domain)
|
|
else:
|
|
self._redirect_headers(http_prefix + '://' +
|
|
domain_full + '/login',
|
|
cookie, calling_domain)
|
|
return
|
|
|
|
if not register_account(base_dir, http_prefix, domain, port,
|
|
login_nickname, login_password,
|
|
self.server.manual_follower_approval):
|
|
self.server.postreq_busy = False
|
|
if calling_domain.endswith('.onion') and onion_domain:
|
|
self._redirect_headers('http://' + onion_domain +
|
|
'/login', cookie,
|
|
calling_domain)
|
|
elif (calling_domain.endswith('.i2p') and i2p_domain):
|
|
self._redirect_headers('http://' + i2p_domain +
|
|
'/login', cookie,
|
|
calling_domain)
|
|
else:
|
|
self._redirect_headers(http_prefix + '://' +
|
|
domain_full + '/login',
|
|
cookie, calling_domain)
|
|
return
|
|
auth_header = \
|
|
create_basic_auth_header(login_nickname, login_password)
|
|
if self.headers.get('X-Forward-For'):
|
|
ip_address = self.headers['X-Forward-For']
|
|
elif self.headers.get('X-Forwarded-For'):
|
|
ip_address = self.headers['X-Forwarded-For']
|
|
else:
|
|
ip_address = self.client_address[0]
|
|
if not domain.endswith('.onion'):
|
|
if not is_local_network_address(ip_address):
|
|
print('Login attempt from IP: ' + str(ip_address))
|
|
if not authorize_basic(base_dir, '/users/' +
|
|
login_nickname + '/outbox',
|
|
auth_header, False):
|
|
print('Login failed: ' + login_nickname)
|
|
self._clear_login_details(login_nickname, calling_domain)
|
|
fail_time = int(time.time())
|
|
self.server.last_login_failure = fail_time
|
|
if not domain.endswith('.onion'):
|
|
if not is_local_network_address(ip_address):
|
|
record_login_failure(base_dir, ip_address,
|
|
self.server.login_failure_count,
|
|
fail_time,
|
|
self.server.log_login_failures)
|
|
self.server.postreq_busy = False
|
|
return
|
|
else:
|
|
if self.server.login_failure_count.get(ip_address):
|
|
del self.server.login_failure_count[ip_address]
|
|
if is_suspended(base_dir, login_nickname):
|
|
msg = \
|
|
html_suspended(base_dir).encode('utf-8')
|
|
msglen = len(msg)
|
|
self._login_headers('text/html',
|
|
msglen, calling_domain)
|
|
self._write(msg)
|
|
self.server.postreq_busy = False
|
|
return
|
|
# login success - redirect with authorization
|
|
print('====== Login success: ' + login_nickname +
|
|
' ' + ua_str)
|
|
# re-activate account if needed
|
|
activate_account(base_dir, login_nickname, domain)
|
|
# This produces a deterministic token based
|
|
# on nick+password+salt
|
|
salt_filename = \
|
|
acct_dir(base_dir, login_nickname, domain) + '/.salt'
|
|
salt = create_password(32)
|
|
if os.path.isfile(salt_filename):
|
|
try:
|
|
with open(salt_filename, 'r',
|
|
encoding='utf-8') as fp_salt:
|
|
salt = fp_salt.read()
|
|
except OSError as ex:
|
|
print('EX: Unable to read salt for ' +
|
|
login_nickname + ' ' + str(ex))
|
|
else:
|
|
try:
|
|
with open(salt_filename, 'w+',
|
|
encoding='utf-8') as fp_salt:
|
|
fp_salt.write(salt)
|
|
except OSError as ex:
|
|
print('EX: Unable to save salt for ' +
|
|
login_nickname + ' ' + str(ex))
|
|
|
|
token_text = login_nickname + login_password + salt
|
|
token = sha256(token_text.encode('utf-8')).hexdigest()
|
|
self.server.tokens[login_nickname] = token
|
|
login_handle = login_nickname + '@' + domain
|
|
token_filename = \
|
|
base_dir + '/accounts/' + \
|
|
login_handle + '/.token'
|
|
try:
|
|
with open(token_filename, 'w+',
|
|
encoding='utf-8') as fp_tok:
|
|
fp_tok.write(token)
|
|
except OSError as ex:
|
|
print('EX: Unable to save token for ' +
|
|
login_nickname + ' ' + str(ex))
|
|
|
|
person_upgrade_actor(base_dir, None,
|
|
base_dir + '/accounts/' +
|
|
login_handle + '.json')
|
|
|
|
index = self.server.tokens[login_nickname]
|
|
self.server.tokens_lookup[index] = login_nickname
|
|
cookie_str = 'SET:epicyon=' + \
|
|
self.server.tokens[login_nickname] + '; SameSite=Strict'
|
|
if calling_domain.endswith('.onion') and onion_domain:
|
|
self._redirect_headers('http://' +
|
|
onion_domain +
|
|
'/users/' +
|
|
login_nickname + '/' +
|
|
self.server.default_timeline,
|
|
cookie_str, calling_domain)
|
|
elif (calling_domain.endswith('.i2p') and i2p_domain):
|
|
self._redirect_headers('http://' +
|
|
i2p_domain +
|
|
'/users/' +
|
|
login_nickname + '/' +
|
|
self.server.default_timeline,
|
|
cookie_str, calling_domain)
|
|
else:
|
|
self._redirect_headers(http_prefix + '://' +
|
|
domain_full + '/users/' +
|
|
login_nickname + '/' +
|
|
self.server.default_timeline,
|
|
cookie_str, calling_domain)
|
|
self.server.postreq_busy = False
|
|
return
|
|
else:
|
|
print('WARN: No login credentials presented to /login')
|
|
if debug:
|
|
# be careful to avoid logging the password
|
|
login_str = login_params
|
|
if '=' in login_params:
|
|
login_params_list = login_params.split('=')
|
|
login_str = ''
|
|
skip_param = False
|
|
for login_prm in login_params_list:
|
|
if not skip_param:
|
|
login_str += login_prm + '='
|
|
else:
|
|
len_str = login_prm.split('&')[0]
|
|
if len(len_str) > 0:
|
|
login_str += login_prm + '*'
|
|
len_str = ''
|
|
if '&' in login_prm:
|
|
login_str += \
|
|
'&' + login_prm.split('&')[1] + '='
|
|
skip_param = False
|
|
if 'password' in login_prm:
|
|
skip_param = True
|
|
login_str = login_str[:len(login_str) - 1]
|
|
print(login_str)
|
|
self._401('No login credentials were posted')
|
|
self.server.postreq_busy = False
|
|
self._200()
|
|
self.server.postreq_busy = False
|
|
|
|
def _moderator_actions(self, path: str, calling_domain: str, cookie: str,
|
|
base_dir: str, http_prefix: str,
|
|
domain: str, port: int, debug: bool) -> None:
|
|
"""Actions on the moderator screen
|
|
"""
|
|
users_path = path.replace('/moderationaction', '')
|
|
nickname = users_path.replace('/users/', '')
|
|
actor_str = self._get_instance_url(calling_domain) + users_path
|
|
if not is_moderator(self.server.base_dir, nickname):
|
|
self._redirect_headers(actor_str + '/moderation',
|
|
cookie, calling_domain)
|
|
self.server.postreq_busy = False
|
|
return
|
|
|
|
length = int(self.headers['Content-length'])
|
|
|
|
try:
|
|
moderation_params = self.rfile.read(length).decode('utf-8')
|
|
except SocketError as ex:
|
|
if ex.errno == errno.ECONNRESET:
|
|
print('EX: POST moderation_params connection was reset')
|
|
else:
|
|
print('EX: POST moderation_params ' +
|
|
'rfile.read socket error')
|
|
self.send_response(400)
|
|
self.end_headers()
|
|
self.server.postreq_busy = False
|
|
return
|
|
except ValueError as ex:
|
|
print('EX: POST moderation_params rfile.read failed, ' +
|
|
str(ex))
|
|
self.send_response(400)
|
|
self.end_headers()
|
|
self.server.postreq_busy = False
|
|
return
|
|
|
|
if '&' in moderation_params:
|
|
moderation_text = None
|
|
moderation_button = None
|
|
# get the moderation text first
|
|
act_str = 'moderationAction='
|
|
for moderation_str in moderation_params.split('&'):
|
|
if moderation_str.startswith(act_str):
|
|
if act_str in moderation_str:
|
|
moderation_text = \
|
|
moderation_str.split(act_str)[1].strip()
|
|
mod_text = moderation_text.replace('+', ' ')
|
|
moderation_text = \
|
|
urllib.parse.unquote_plus(mod_text.strip())
|
|
# which button was pressed?
|
|
for moderation_str in moderation_params.split('&'):
|
|
if moderation_str.startswith('submitInfo='):
|
|
if not moderation_text and \
|
|
'submitInfo=' in moderation_str:
|
|
moderation_text = \
|
|
moderation_str.split('submitInfo=')[1].strip()
|
|
mod_text = moderation_text.replace('+', ' ')
|
|
moderation_text = \
|
|
urllib.parse.unquote_plus(mod_text.strip())
|
|
search_handle = moderation_text
|
|
if search_handle:
|
|
if '/@' in search_handle:
|
|
search_nickname = \
|
|
get_nickname_from_actor(search_handle)
|
|
if search_nickname:
|
|
search_domain, _ = \
|
|
get_domain_from_actor(search_handle)
|
|
search_handle = \
|
|
search_nickname + '@' + search_domain
|
|
else:
|
|
search_handle = ''
|
|
if '@' not in search_handle:
|
|
if search_handle.startswith('http') or \
|
|
search_handle.startswith('ipfs') or \
|
|
search_handle.startswith('ipns'):
|
|
search_nickname = \
|
|
get_nickname_from_actor(search_handle)
|
|
if search_nickname:
|
|
search_domain, _ = \
|
|
get_domain_from_actor(search_handle)
|
|
search_handle = \
|
|
search_nickname + '@' + search_domain
|
|
else:
|
|
search_handle = ''
|
|
if '@' not in search_handle:
|
|
# is this a local nickname on this instance?
|
|
local_handle = \
|
|
search_handle + '@' + self.server.domain
|
|
if os.path.isdir(self.server.base_dir +
|
|
'/accounts/' + local_handle):
|
|
search_handle = local_handle
|
|
else:
|
|
search_handle = ''
|
|
if search_handle is None:
|
|
search_handle = ''
|
|
if '@' in search_handle:
|
|
msg = \
|
|
html_account_info(self.server.translate,
|
|
base_dir, http_prefix,
|
|
nickname,
|
|
self.server.domain,
|
|
self.server.port,
|
|
search_handle,
|
|
self.server.debug,
|
|
self.server.system_language,
|
|
self.server.signing_priv_key_pem)
|
|
else:
|
|
msg = \
|
|
html_moderation_info(self.server.translate,
|
|
base_dir, nickname,
|
|
self.server.domain,
|
|
self.server.theme_name,
|
|
self.server.access_keys)
|
|
if msg:
|
|
msg = msg.encode('utf-8')
|
|
msglen = len(msg)
|
|
self._login_headers('text/html',
|
|
msglen, calling_domain)
|
|
self._write(msg)
|
|
self.server.postreq_busy = False
|
|
return
|
|
if moderation_str.startswith('submitBlock'):
|
|
moderation_button = 'block'
|
|
elif moderation_str.startswith('submitUnblock'):
|
|
moderation_button = 'unblock'
|
|
elif moderation_str.startswith('submitFilter'):
|
|
moderation_button = 'filter'
|
|
elif moderation_str.startswith('submitUnfilter'):
|
|
moderation_button = 'unfilter'
|
|
elif moderation_str.startswith('submitSuspend'):
|
|
moderation_button = 'suspend'
|
|
elif moderation_str.startswith('submitUnsuspend'):
|
|
moderation_button = 'unsuspend'
|
|
elif moderation_str.startswith('submitRemove'):
|
|
moderation_button = 'remove'
|
|
if moderation_button and moderation_text:
|
|
if debug:
|
|
print('moderation_button: ' + moderation_button)
|
|
print('moderation_text: ' + moderation_text)
|
|
nickname = moderation_text
|
|
if nickname.startswith('http') or \
|
|
nickname.startswith('ipfs') or \
|
|
nickname.startswith('ipns') or \
|
|
nickname.startswith('hyper'):
|
|
nickname = get_nickname_from_actor(nickname)
|
|
if '@' in nickname:
|
|
nickname = nickname.split('@')[0]
|
|
if moderation_button == 'suspend':
|
|
suspend_account(base_dir, nickname, domain)
|
|
if moderation_button == 'unsuspend':
|
|
reenable_account(base_dir, nickname)
|
|
if moderation_button == 'filter':
|
|
add_global_filter(base_dir, moderation_text)
|
|
if moderation_button == 'unfilter':
|
|
remove_global_filter(base_dir, moderation_text)
|
|
if moderation_button == 'block':
|
|
full_block_domain = None
|
|
if moderation_text.startswith('http') or \
|
|
moderation_text.startswith('ipfs') or \
|
|
moderation_text.startswith('ipns') or \
|
|
moderation_text.startswith('hyper'):
|
|
# https://domain
|
|
block_domain, block_port = \
|
|
get_domain_from_actor(moderation_text)
|
|
full_block_domain = \
|
|
get_full_domain(block_domain, block_port)
|
|
if '@' in moderation_text:
|
|
# nick@domain or *@domain
|
|
full_block_domain = moderation_text.split('@')[1]
|
|
else:
|
|
# assume the text is a domain name
|
|
if not full_block_domain and '.' in moderation_text:
|
|
nickname = '*'
|
|
full_block_domain = moderation_text.strip()
|
|
if full_block_domain or nickname.startswith('#'):
|
|
add_global_block(base_dir, nickname, full_block_domain)
|
|
if moderation_button == 'unblock':
|
|
full_block_domain = None
|
|
if moderation_text.startswith('http') or \
|
|
moderation_text.startswith('ipfs') or \
|
|
moderation_text.startswith('ipns') or \
|
|
moderation_text.startswith('hyper'):
|
|
# https://domain
|
|
block_domain, block_port = \
|
|
get_domain_from_actor(moderation_text)
|
|
full_block_domain = \
|
|
get_full_domain(block_domain, block_port)
|
|
if '@' in moderation_text:
|
|
# nick@domain or *@domain
|
|
full_block_domain = moderation_text.split('@')[1]
|
|
else:
|
|
# assume the text is a domain name
|
|
if not full_block_domain and '.' in moderation_text:
|
|
nickname = '*'
|
|
full_block_domain = moderation_text.strip()
|
|
if full_block_domain or nickname.startswith('#'):
|