Function for line ending characters

auth.py

@@ -16,6 +16,7 @@ import datetime
 from utils import is_system_account
 from utils import has_users_path
 from utils import text_in_file
+from utils import remove_eol
 
 
 def _hash_password(password: str) -> str:
@@ -78,9 +79,9 @@ def create_basic_auth_header(nickname: str, password: str) -> str:
     """This is only used by tests
     """
     auth_str = \
-        nickname.replace('\n', '').replace('\r', '') + \
+        remove_eol(nickname) + \
         ':' + \
-        password.replace('\n', '').replace('\r', '')
+        remove_eol(password)
     return 'Basic ' + \
         base64.b64encode(auth_str.encode('utf-8')).decode('utf-8')
 
@@ -118,8 +119,8 @@ def authorize_basic(base_dir: str, path: str, auth_header: str,
         print('basic auth - attempted login using system account ' +
               nickname_from_path + ' in path')
         return False
-    base64_str = \
-        auth_header.split(' ')[1].replace('\n', '').replace('\r', '')
+    base64_str1 = auth_header.split(' ')[1]
+    base64_str = remove_eol(base64_str1)
     plain = base64.b64decode(base64_str).decode('utf-8')
     if ':' not in plain:
         if debug:
@@ -148,8 +149,8 @@ def authorize_basic(base_dir: str, path: str, auth_header: str,
             for line in passfile:
                 if not line.startswith(nickname + ':'):
                     continue
-                stored_password = \
-                    line.split(':')[1].replace('\n', '').replace('\r', '')
+                stored_password_base = line.split(':')[1]
+                stored_password = remove_eol(stored_password_base)
                 success = _verify_password(stored_password, provided_password)
                 if not success:
                     if debug:
@@ -169,8 +170,8 @@ def store_basic_credentials(base_dir: str,
     """
     if ':' in nickname or ':' in password:
         return False
-    nickname = nickname.replace('\n', '').replace('\r', '').strip()
-    password = password.replace('\n', '').replace('\r', '').strip()
+    nickname = remove_eol(nickname).strip()
+    password = remove_eol(password).strip()
 
     if not os.path.isdir(base_dir + '/accounts'):
         os.mkdir(base_dir + '/accounts')

blocking.py

@@ -11,6 +11,7 @@ import os
 import json
 import time
 from datetime import datetime
+from utils import remove_eol
 from utils import has_object_string
 from utils import has_object_string_object
 from utils import has_object_string_type
@@ -162,8 +163,7 @@ def remove_global_block(base_dir: str,
                         with open(unblocking_filename + '.new', 'w+',
                                   encoding='utf-8') as fpnew:
                             for line in fp_unblock:
-                                handle = \
-                                    line.replace('\n', '').replace('\r', '')
+                                handle = remove_eol(line)
                                 if unblock_handle not in line:
                                     fpnew.write(handle + '\n')
                 except OSError as ex:
@@ -189,8 +189,7 @@ def remove_global_block(base_dir: str,
                         with open(unblocking_filename + '.new', 'w+',
                                   encoding='utf-8') as fpnew:
                             for line in fp_unblock:
-                                block_line = \
-                                    line.replace('\n', '').replace('\r', '')
+                                block_line = remove_eol(line)
                                 if unblock_hashtag not in line:
                                     fpnew.write(block_line + '\n')
                 except OSError as ex:
@@ -225,7 +224,7 @@ def remove_block(base_dir: str, nickname: str, domain: str,
                     with open(unblocking_filename + '.new', 'w+',
                               encoding='utf-8') as fpnew:
                         for line in fp_unblock:
-                            handle = line.replace('\n', '').replace('\r', '')
+                            handle = remove_eol(line)
                             if unblock_handle not in line:
                                 fpnew.write(handle + '\n')
             except OSError as ex:
@@ -304,7 +303,7 @@ def update_blocked_cache(base_dir: str,
             blocked_lines = fp_blocked.readlines()
             # remove newlines
             for index, _ in enumerate(blocked_lines):
-                blocked_lines[index] = blocked_lines[index].replace('\n', '')
+                blocked_lines[index] = remove_eol(blocked_lines[index])
             # update the cache
             blocked_cache.clear()
             blocked_cache += blocked_lines
@@ -946,7 +945,7 @@ def set_broch_mode(base_dir: str, domain_full: str, enabled: bool) -> None:
                             for handle in follow_list:
                                 if '@' not in handle:
                                     continue
-                                handle = handle.replace('\n', '')
+                                handle = remove_eol(handle)
                                 handle_domain = handle.split('@')[1]
                                 if handle_domain not in allowed_domains:
                                     allowed_domains.append(handle_domain)

blog.py

@@ -17,6 +17,7 @@ from webapp_utils import html_footer
 from webapp_utils import get_post_attachments_as_html
 from webapp_utils import edit_text_area
 from webapp_media import add_embedded_elements
+from utils import remove_eol
 from utils import text_in_file
 from utils import local_actor_url
 from utils import get_actor_languages_list
@@ -80,7 +81,7 @@ def _no_of_blog_replies(base_dir: str, http_prefix: str, translate: {},
         print('EX: failed to read blog ' + post_filename)
 
     for reply_post_id in lines:
-        reply_post_id = reply_post_id.replace('\n', '').replace('\r', '')
+        reply_post_id = remove_eol(reply_post_id)
         reply_post_id = reply_post_id.replace('.json', '')
         if locate_post(base_dir, nickname, domain, reply_post_id):
             reply_post_id = reply_post_id.replace('.replies', '')
@@ -99,8 +100,7 @@ def _no_of_blog_replies(base_dir: str, http_prefix: str, translate: {},
         try:
             with open(post_filename, 'w+', encoding='utf-8') as post_file:
                 for reply_post_id in lines:
-                    reply_post_id = \
-                        reply_post_id.replace('\n', '').replace('\r', '')
+                    reply_post_id = remove_eol(reply_post_id)
                     if reply_post_id not in removals:
                         post_file.write(reply_post_id + '\n')
         except OSError as ex:
@@ -158,7 +158,7 @@ def _get_blog_replies(base_dir: str, http_prefix: str, translate: {},
     if lines:
         replies_str = ''
         for reply_post_id in lines:
-            reply_post_id = reply_post_id.replace('\n', '').replace('\r', '')
+            reply_post_id = remove_eol(reply_post_id)
             reply_post_id = reply_post_id.replace('.json', '')
             reply_post_id = reply_post_id.replace('.replies', '')
             post_filename = acct_dir(base_dir, nickname, domain) + \

bookmarks.py

@@ -29,6 +29,7 @@ from utils import local_actor_url
 from utils import has_actor
 from utils import has_object_string_type
 from utils import text_in_file
+from utils import remove_eol
 from posts import get_person_box
 from session import post_json
 
@@ -71,7 +72,7 @@ def undo_bookmarks_collection_entry(recent_posts_cache: {},
         bookmark_index = post_filename.split('/')[-1].strip()
     else:
         bookmark_index = post_filename.strip()
-    bookmark_index = bookmark_index.replace('\n', '').replace('\r', '')
+    bookmark_index = remove_eol(bookmark_index)
     if not text_in_file(bookmark_index, bookmarks_index_filename):
         return
     index_str = ''

city.py

@@ -13,6 +13,7 @@ import random
 import math
 from random import randint
 from utils import acct_dir
+from utils import remove_eol
 
 # states which the simulated city dweller can be in
 PERSON_SLEEP = 0
@@ -145,7 +146,7 @@ def _get_city_pulse(curr_time_of_day, decoy_seed: int) -> (float, float):
 def parse_nogo_string(nogo_line: str) -> []:
     """Parses a line from locations_nogo.txt and returns the polygon
     """
-    nogo_line = nogo_line.replace('\n', '').replace('\r', '')
+    nogo_line = remove_eol(nogo_line)
     polygon_str = nogo_line.split(':', 1)[1]
     if ';' in polygon_str:
         pts = polygon_str.split(';')
@@ -320,7 +321,8 @@ def get_spoofed_city(city: str, base_dir: str,
     if os.path.isfile(city_filename):
         try:
             with open(city_filename, 'r', encoding='utf-8') as city_file:
-                city = city_file.read().replace('\n', '')
+                city1 = city_file.read()
+                city = remove_eol(city1)
         except OSError:
             print('EX: unable to read ' + city_filename)
     return city

content.py

@@ -32,6 +32,7 @@ from utils import acct_dir
 from utils import is_float
 from utils import get_currencies
 from utils import remove_html
+from utils import remove_eol
 from petnames import get_pet_name
 from session import download_image
 
@@ -288,7 +289,7 @@ def switch_words(base_dir: str, nickname: str, domain: str, content: str,
             print('EX: unable to read switches ' + switch_words_filename)
 
     for line in rules:
-        replace_str = line.replace('\n', '').replace('\r', '')
+        replace_str = remove_eol(line)
         splitters = ('->', ':', ',', ';', '-')
         word_transform = None
         for split_str in splitters:
@@ -397,7 +398,8 @@ def _update_common_emoji(base_dir: str, emoji_content: str) -> None:
                     line = count_str + ' ' + emoji_content
                     new_common_emoji.append(line)
             else:
-                new_common_emoji.append(line.replace('\n', ''))
+                line1 = remove_eol(line)
+                new_common_emoji.append(line1)
         if not emoji_found:
             new_common_emoji.append(str(1).zfill(16) + ' ' + emoji_content)
         new_common_emoji.sort(reverse=True)
@@ -757,7 +759,7 @@ def _add_mention(word_str: str, http_prefix: str, following: str,
                 continue
             follow_nick = follow.split('@')[0]
             if possible_nickname == follow_nick:
-                follow_str = follow.replace('\n', '').replace('\r', '')
+                follow_str = remove_eol(follow)
                 replace_domain = follow_str.split('@')[1]
                 recipient_actor = http_prefix + "://" + \
                     replace_domain + "/@" + possible_nickname
@@ -780,10 +782,10 @@ def _add_mention(word_str: str, http_prefix: str, following: str,
             if '@' not in follow:
                 follow_ctr += 1
                 continue
-            pet = petnames[follow_ctr].replace('\n', '')
+            pet = remove_eol(petnames[follow_ctr])
             if pet:
                 if possible_nickname == pet:
-                    follow_str = follow.replace('\n', '').replace('\r', '')
+                    follow_str = remove_eol(follow)
                     replace_nickname = follow_str.split('@')[0]
                     replace_domain = follow_str.split('@')[1]
                     recipient_actor = http_prefix + "://" + \
@@ -817,7 +819,7 @@ def _add_mention(word_str: str, http_prefix: str, following: str,
         return False
     if following:
         for follow in following:
-            if follow.replace('\n', '').replace('\r', '') != possible_handle:
+            if remove_eol(follow) != possible_handle:
                 continue
             recipient_actor = http_prefix + "://" + \
                 possible_domain + "/@" + possible_nickname

crawlers.py

@@ -11,6 +11,7 @@ import os
 import time
 from utils import save_json
 from utils import user_agent_domain
+from utils import remove_eol
 from blocking import update_blocked_cache
 from blocking import is_blocked_domain
 
@@ -72,7 +73,7 @@ def load_known_web_bots(base_dir: str) -> []:
     for crawler in crawlers_list:
         if not crawler:
             continue
-        crawler = crawler.replace('\n', '').strip()
+        crawler = remove_eol(crawler).strip()
         if not crawler:
             continue
         if crawler not in known_bots:

daemon.py

@@ -253,6 +253,7 @@ from languages import set_actor_languages
 from languages import get_understood_languages
 from like import update_likes_collection
 from reaction import update_reaction_collection
+from utils import remove_eol
 from utils import text_in_file
 from utils import is_onion_request
 from utils import is_i2p_request
@@ -4185,7 +4186,7 @@ class PubServer(BaseHTTPRequestHandler):
                         timezone = \
                             self.server.account_timezone.get(nickname)
 
-                    profile_handle = search_str.replace('\n', '').strip()
+                    profile_handle = remove_eol(search_str).strip()
 
                     # establish the session
                     curr_proxy_type = proxy_type

epicyon.py

@@ -68,6 +68,7 @@ from tests import test_update_actor
 from tests import run_all_tests
 from auth import store_basic_credentials
 from auth import create_password
+from utils import remove_eol
 from utils import text_in_file
 from utils import remove_domain_port
 from utils import get_port_from_domain
@@ -1457,7 +1458,8 @@ def _command_options() -> None:
             with open(approve_follows_filename, 'r',
                       encoding='utf-8') as approvefile:
                 for approve in approvefile:
-                    print(approve.replace('\n', '').replace('\r', ''))
+                    approve1 = remove_eol(approve)
+                    print(approve1)
                     approve_ctr += 1
         if approve_ctr == 0:
             print('There are no follow requests pending approval.')
@@ -1488,7 +1490,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         session = create_session(proxy_type)
         if not argb.sendto:
@@ -1504,7 +1506,7 @@ def _command_options() -> None:
         if '@' in argb.sendto:
             to_nickname = argb.sendto.split('@')[0]
             to_domain = argb.sendto.split('@')[1]
-            to_domain = to_domain.replace('\n', '').replace('\r', '')
+            to_domain = remove_eol(to_domain)
             to_port = 443
             if ':' in to_domain:
                 to_port = get_port_from_domain(to_domain)
@@ -1583,7 +1585,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
         proxy_type = None
         if argb.tor or domain.endswith('.onion'):
             proxy_type = 'tor'
@@ -1624,7 +1626,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         session = create_session(proxy_type)
         person_cache = {}
@@ -1661,7 +1663,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         proxy_type = None
         if argb.tor or domain.endswith('.onion'):
@@ -1697,7 +1699,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         if not argb.nickname:
             print('Specify a nickname with the --nickname option')
@@ -1769,7 +1771,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         if not argb.nickname:
             print('Specify a nickname with the --nickname option')
@@ -1803,7 +1805,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         if not argb.nickname:
             print('Specify a nickname with the --nickname option')
@@ -1875,7 +1877,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         if not argb.nickname:
             print('Specify a nickname with the --nickname option')
@@ -1913,7 +1915,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         session = create_session(proxy_type)
         person_cache = {}
@@ -1952,7 +1954,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         session = create_session(proxy_type)
         person_cache = {}
@@ -1985,7 +1987,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         session = create_session(proxy_type)
         person_cache = {}
@@ -2025,7 +2027,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         session = create_session(proxy_type)
         person_cache = {}
@@ -2060,7 +2062,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         session = create_session(proxy_type)
         person_cache = {}
@@ -2094,7 +2096,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         session = create_session(proxy_type)
         person_cache = {}
@@ -2127,7 +2129,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         session = create_session(proxy_type)
         person_cache = {}
@@ -2164,7 +2166,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         follow_nickname = get_nickname_from_actor(argb.follow)
         if not follow_nickname:
@@ -2212,7 +2214,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         follow_nickname = get_nickname_from_actor(argb.unfollow)
         if not follow_nickname:
@@ -2257,7 +2259,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         session = create_session(proxy_type)
         person_cache = {}
@@ -2290,7 +2292,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         session = create_session(proxy_type)
         person_cache = {}
@@ -2324,7 +2326,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         session = create_session(proxy_type)
         person_cache = {}
@@ -2442,31 +2444,31 @@ def _command_options() -> None:
                 sys.exit()
             if '/users/' in argb.followers:
                 nickname = argb.followers.split('/users/')[1]
-                nickname = nickname.replace('\n', '').replace('\r', '')
+                nickname = remove_eol(nickname)
                 domain = argb.followers.split('/users/')[0]
             elif '/profile/' in argb.followers:
                 nickname = argb.followers.split('/profile/')[1]
-                nickname = nickname.replace('\n', '').replace('\r', '')
+                nickname = remove_eol(nickname)
                 domain = argb.followers.split('/profile/')[0]
             elif '/author/' in argb.followers:
                 nickname = argb.followers.split('/author/')[1]
-                nickname = nickname.replace('\n', '').replace('\r', '')
+                nickname = remove_eol(nickname)
                 domain = argb.followers.split('/author/')[0]
             elif '/channel/' in argb.followers:
                 nickname = argb.followers.split('/channel/')[1]
-                nickname = nickname.replace('\n', '').replace('\r', '')
+                nickname = remove_eol(nickname)
                 domain = argb.followers.split('/channel/')[0]
             elif '/accounts/' in argb.followers:
                 nickname = argb.followers.split('/accounts/')[1]
-                nickname = nickname.replace('\n', '').replace('\r', '')
+                nickname = remove_eol(nickname)
                 domain = argb.followers.split('/accounts/')[0]
             elif '/u/' in argb.followers:
                 nickname = argb.followers.split('/u/')[1]
-                nickname = nickname.replace('\n', '').replace('\r', '')
+                nickname = remove_eol(nickname)
                 domain = argb.followers.split('/u/')[0]
             elif '/c/' in argb.followers:
                 nickname = argb.followers.split('/c/')[1]
-                nickname = nickname.replace('\n', '').replace('\r', '')
+                nickname = remove_eol(nickname)
                 domain = argb.followers.split('/c/')[0]
         else:
             # format: @nick@domain
@@ -2480,7 +2482,7 @@ def _command_options() -> None:
                 sys.exit()
             nickname = argb.followers.split('@')[0]
             domain = argb.followers.split('@')[1]
-            domain = domain.replace('\n', '').replace('\r', '')
+            domain = remove_eol(domain)
         cached_webfingers = {}
         if argb.http or domain.endswith('.onion'):
             http_prefix = 'http'
@@ -2603,7 +2605,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
         if len(argb.password.strip()) < 8:
             print('Password should be at least 8 characters')
             sys.exit()
@@ -2652,7 +2654,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
         if len(argb.password.strip()) < 8:
             print('Password should be at least 8 characters')
             sys.exit()
@@ -2818,7 +2820,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         if not argb.skillLevelPercent:
             print('Specify a skill level in the range 0-100')
@@ -2862,7 +2864,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         session = create_session(proxy_type)
         person_cache = {}
@@ -2906,7 +2908,7 @@ def _command_options() -> None:
                 print('Specify a password with the --password option')
                 sys.exit()
 
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         proxy_type = None
         if argb.tor or domain.endswith('.onion'):
@@ -2965,11 +2967,11 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         if '@' in argb.block:
             blocked_domain = argb.block.split('@')[1]
-            blocked_domain = blocked_domain.replace('\n', '').replace('\r', '')
+            blocked_domain = remove_eol(blocked_domain)
             blocked_nickname = argb.block.split('@')[0]
             blocked_actor = http_prefix + '://' + blocked_domain + \
                 '/users/' + blocked_nickname
@@ -3009,7 +3011,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         session = create_session(proxy_type)
         person_cache = {}
@@ -3041,7 +3043,7 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         session = create_session(proxy_type)
         person_cache = {}
@@ -3073,11 +3075,11 @@ def _command_options() -> None:
             if not argb.password:
                 print('Specify a password with the --password option')
                 sys.exit()
-        argb.password = argb.password.replace('\n', '')
+        argb.password = remove_eol(argb.password)
 
         if '@' in argb.unblock:
             blocked_domain = argb.unblock.split('@')[1]
-            blocked_domain = blocked_domain.replace('\n', '').replace('\r', '')
+            blocked_domain = remove_eol(blocked_domain)
             blocked_nickname = argb.unblock.split('@')[0]
             blocked_actor = http_prefix + '://' + blocked_domain + \
                 '/users/' + blocked_nickname

filters.py

@@ -10,6 +10,7 @@ __module_group__ = "Moderation"
 import os
 from utils import acct_dir
 from utils import text_in_file
+from utils import remove_eol
 
 
 def add_filter(base_dir: str, nickname: str, domain: str, words: str) -> bool:
@@ -62,7 +63,7 @@ def remove_filter(base_dir: str, nickname: str, domain: str,
         with open(filters_filename, 'r', encoding='utf-8') as fp_filt:
             with open(new_filters_filename, 'w+', encoding='utf-8') as fpnew:
                 for line in fp_filt:
-                    line = line.replace('\n', '')
+                    line = remove_eol(line)
                     if line != words:
                         fpnew.write(line + '\n')
     except OSError as ex:
@@ -87,7 +88,7 @@ def remove_global_filter(base_dir: str, words: str) -> bool:
         with open(filters_filename, 'r', encoding='utf-8') as fp_filt:
             with open(new_filters_filename, 'w+', encoding='utf-8') as fpnew:
                 for line in fp_filt:
-                    line = line.replace('\n', '')
+                    line = remove_eol(line)
                     if line != words:
                         fpnew.write(line + '\n')
     except OSError as ex:
@@ -122,7 +123,7 @@ def _is_filtered_base(filename: str, content: str) -> bool:
     try:
         with open(filename, 'r', encoding='utf-8') as fp_filt:
             for line in fp_filt:
-                filter_str = line.replace('\n', '').replace('\r', '')
+                filter_str = remove_eol(line)
                 if not filter_str:
                     continue
                 if len(filter_str) < 2:

follow.py

@@ -31,6 +31,7 @@ from utils import acct_dir
 from utils import has_group_type
 from utils import local_actor_url
 from utils import text_in_file
+from utils import remove_eol
 from acceptreject import create_accept
 from acceptreject import create_reject
 from webfinger import webfinger_handle
@@ -67,7 +68,7 @@ def create_initial_last_seen(base_dir: str, http_prefix: str) -> None:
                     continue
                 if '@' not in handle:
                     continue
-                handle = handle.replace('\n', '')
+                handle = remove_eol(handle)
                 nickname = handle.split('@')[0]
                 domain = handle.split('@')[1]
                 if nickname.startswith('!'):
@@ -244,7 +245,7 @@ def get_follower_domains(base_dir: str, nickname: str, domain: str) -> []:
 
     domains_list = []
     for handle in lines:
-        handle = handle.replace('\n', '')
+        handle = remove_eol(handle)
         follower_domain, _ = get_domain_from_actor(handle)
         if not follower_domain:
             continue
@@ -535,8 +536,8 @@ def get_following_feed(base_dir: str, domain: str, port: int, path: str,
                 page_ctr += 1
                 total_ctr += 1
                 if curr_page == page_number:
-                    line2 = \
-                        line.lower().replace('\n', '').replace('\r', '')
+                    line2_lower = line.lower()
+                    line2 = remove_eol(line2_lower)
                     nick = line2.split('@')[0]
                     dom = line2.split('@')[1]
                     if not nick.startswith('!'):
@@ -555,8 +556,8 @@ def get_following_feed(base_dir: str, domain: str, port: int, path: str,
                 page_ctr += 1
                 total_ctr += 1
                 if curr_page == page_number:
-                    append_str = \
-                        line.lower().replace('\n', '').replace('\r', '')
+                    append_str1 = line.lower()
+                    append_str = remove_eol(append_str1)
                     following['orderedItems'].append(append_str)
         if page_ctr >= follows_per_page:
             page_ctr = 0

happening.py

@@ -25,6 +25,7 @@ from utils import delete_post
 from utils import get_status_number
 from utils import get_full_domain
 from utils import text_in_file
+from utils import remove_eol
 from filters import is_filtered
 from context import get_individual_post_context
 from session import get_method
@@ -262,7 +263,7 @@ def get_todays_events(base_dir: str, nickname: str, domain: str,
     recreate_events_file = False
     with open(calendar_filename, 'r', encoding='utf-8') as events_file:
         for post_id in events_file:
-            post_id = post_id.replace('\n', '').replace('\r', '')
+            post_id = remove_eol(post_id)
             post_filename = locate_post(base_dir, nickname, domain, post_id)
             if not post_filename:
                 recreate_events_file = True
@@ -556,7 +557,7 @@ def day_events_check(base_dir: str, nickname: str, domain: str,
     events_exist = False
     with open(calendar_filename, 'r', encoding='utf-8') as events_file:
         for post_id in events_file:
-            post_id = post_id.replace('\n', '').replace('\r', '')
+            post_id = remove_eol(post_id)
             post_filename = locate_post(base_dir, nickname, domain, post_id)
             if not post_filename:
                 continue
@@ -612,7 +613,7 @@ def get_this_weeks_events(base_dir: str, nickname: str, domain: str) -> {}:
     recreate_events_file = False
     with open(calendar_filename, 'r', encoding='utf-8') as events_file:
         for post_id in events_file:
-            post_id = post_id.replace('\n', '').replace('\r', '')
+            post_id = remove_eol(post_id)
             post_filename = locate_post(base_dir, nickname, domain, post_id)
             if not post_filename:
                 recreate_events_file = True
@@ -679,7 +680,7 @@ def get_calendar_events(base_dir: str, nickname: str, domain: str,
     recreate_events_file = False
     with open(calendar_filename, 'r', encoding='utf-8') as events_file:
         for post_id in events_file:
-            post_id = post_id.replace('\n', '').replace('\r', '')
+            post_id = remove_eol(post_id)
             post_filename = locate_post(base_dir, nickname, domain, post_id)
             if not post_filename:
                 recreate_events_file = True

inbox.py

@@ -18,6 +18,7 @@ from languages import understood_post_language
 from like import update_likes_collection
 from reaction import update_reaction_collection
 from reaction import valid_emoji_content
+from utils import remove_eol
 from utils import text_in_file
 from utils import get_media_descriptions_from_post
 from utils import get_summary_from_post
@@ -2314,7 +2315,8 @@ def _receive_announce(recent_posts_cache: {},
               str(message_json))
     else:
         if debug:
-            print('Generated announce html ' + announce_html.replace('\n', ''))
+            announce_html2 = remove_eol(announce_html)
+            print('Generated announce html ' + announce_html2)
 
     post_json_object = download_announce(session, base_dir,
                                          http_prefix,

manualapprove.py

@@ -17,6 +17,7 @@ from utils import get_port_from_domain
 from utils import get_user_paths
 from utils import acct_dir
 from utils import text_in_file
+from utils import remove_eol
 from threads import thread_with_trace
 from session import create_session
 
@@ -57,8 +58,7 @@ def manual_deny_follow_request(session, session_onion, session_i2p,
         print('EX: unable to append ' + rejected_follows_filename)
 
     deny_nickname = deny_handle.split('@')[0]
-    deny_domain = \
-        deny_handle.split('@')[1].replace('\n', '').replace('\r', '')
+    deny_domain = remove_eol(deny_handle.split('@')[1])
     deny_port = port
     if ':' in deny_domain:
         deny_port = get_port_from_domain(deny_domain)
@@ -195,7 +195,7 @@ def manual_approve_follow_request(session, session_onion, session_i2p,
                 # is this the approved follow?
                 if handle_of_follow_requester.startswith(approve_handle_full):
                     handle_of_follow_requester = \
-                        handle_of_follow_requester.replace('\n', '')
+                        remove_eol(handle_of_follow_requester)
                     handle_of_follow_requester = \
                         handle_of_follow_requester.replace('\r', '')
                     port2 = port
@@ -212,7 +212,7 @@ def manual_approve_follow_request(session, session_onion, session_i2p,
                             approve_nickname = approve_handle.split('@')[0]
                             approve_domain = approve_handle.split('@')[1]
                             approve_domain = \
-                                approve_domain.replace('\n', '')
+                                remove_eol(approve_domain)
                             approve_domain = \
                                 approve_domain.replace('\r', '')
                             approve_port = port2

newswire.py

@@ -20,6 +20,7 @@ from datetime import timezone
 from collections import OrderedDict
 from utils import valid_post_date
 from categories import set_hashtag_category
+from utils import remove_eol
 from utils import get_domain_from_actor
 from utils import valid_hash_tag
 from utils import dangerous_svg
@@ -1458,8 +1459,7 @@ def _add_account_blogs_to_newswire(base_dir: str, nickname: str, domain: str,
                 # filename of the post without any extension or path
                 # This should also correspond to any index entry in
                 # the posts cache
-                post_url = \
-                    post_filename.replace('\n', '').replace('\r', '')
+                post_url = remove_eol(post_filename)
                 post_url = post_url.replace('.json', '').strip()
 
                 # read the post from file

person.py

@@ -43,7 +43,7 @@ from utils import get_nickname_from_actor
 from utils import remove_html
 from utils import contains_invalid_chars
 from utils import replace_users_with_at
-from utils import remove_line_endings
+from utils import remove_eol
 from utils import remove_domain_port
 from utils import get_status_number
 from utils import get_full_domain
@@ -63,6 +63,7 @@ from utils import get_group_paths
 from utils import local_actor_url
 from utils import dangerous_svg
 from utils import text_in_file
+from utils import remove_line_endings
 from session import create_session
 from session import get_json
 from webfinger import webfinger_handle
@@ -104,7 +105,7 @@ def set_profile_image(base_dir: str, http_prefix: str,
     """Saves the given image file as an avatar or background
     image for the given person
     """
-    image_filename = image_filename.replace('\n', '').replace('\r', '')
+    image_filename = remove_eol(image_filename)
     if not is_image_file(image_filename):
         print('Profile image must be png, jpg, gif or svg format')
         return False
@@ -1371,8 +1372,8 @@ def is_person_snoozed(base_dir: str, nickname: str, domain: str,
         for line in snoozed_file:
             # is this the entry for the actor?
             if line.startswith(snooze_actor + ' '):
-                snoozed_time_str = \
-                    line.split(' ')[1].replace('\n', '').replace('\r', '')
+                snoozed_time_str1 = line.split(' ')[1]
+                snoozed_time_str = remove_eol(snoozed_time_str1)
                 # is there a time appended?
                 if snoozed_time_str.isdigit():
                     snoozed_time = int(snoozed_time_str)
@@ -1525,7 +1526,7 @@ def get_actor_json(host_domain: str, handle: str, http: bool, gnunet: bool,
         for user_path in paths:
             if user_path in handle:
                 nickname = handle.split(user_path)[1]
-                nickname = nickname.replace('\n', '').replace('\r', '')
+                nickname = remove_eol(nickname)
                 domain = handle.split(user_path)[0]
                 user_path_found = True
                 break
@@ -1556,7 +1557,7 @@ def get_actor_json(host_domain: str, handle: str, http: bool, gnunet: bool,
             return None, None
         nickname = handle.split('@')[0]
         domain = handle.split('@')[1]
-        domain = domain.replace('\n', '').replace('\r', '')
+        domain = remove_eol(domain)
 
     cached_webfingers = {}
     proxy_type = None

posts.py

@@ -32,6 +32,7 @@ from webfinger import webfinger_handle
 from httpsig import create_signed_header
 from siteactive import site_is_active
 from languages import understood_post_language
+from utils import remove_eol
 from utils import text_in_file
 from utils import get_media_descriptions_from_post
 from utils import valid_hash_tag
@@ -139,8 +140,7 @@ def no_of_followers_on_domain(base_dir: str, handle: str,
         for follower_handle in followers_file:
             if '@' in follower_handle:
                 follower_domain = follower_handle.split('@')[1]
-                follower_domain = follower_domain.replace('\n', '')
-                follower_domain = follower_domain.replace('\r', '')
+                follower_domain = remove_eol(follower_domain)
                 if domain == follower_domain:
                     ctr += 1
     return ctr
@@ -2707,8 +2707,8 @@ def group_followers_by_domain(base_dir: str, nickname: str, domain: str) -> {}:
         for follower_handle in foll_file:
             if '@' not in follower_handle:
                 continue
-            fhandle = \
-                follower_handle.strip().replace('\n', '').replace('\r', '')
+            fhandle1 = follower_handle.strip()
+            fhandle = remove_eol(fhandle1)
             follower_domain = fhandle.split('@')[1]
             if not grouped.get(follower_domain):
                 grouped[follower_domain] = [fhandle]
@@ -4041,8 +4041,7 @@ def _create_box_indexed(recent_posts_cache: {},
                 # filename of the post without any extension or path
                 # This should also correspond to any index entry in
                 # the posts cache
-                post_url = \
-                    post_filename.replace('\n', '').replace('\r', '')
+                post_url = remove_eol(post_filename)
                 post_url = post_url.replace('.json', '').strip()
 
                 if post_url in post_urls_in_box:
@@ -4737,7 +4736,7 @@ def populate_replies_json(base_dir: str, nickname: str, domain: str,
             reply_found = False
             # examine inbox and outbox
             for boxname in replies_boxes:
-                message_id2 = message_id.replace('\n', '').replace('\r', '')
+                message_id2 = remove_eol(message_id)
                 search_filename = \
                     acct_dir(base_dir, nickname, domain) + '/' + \
                     boxname + '/' + \
@@ -4763,7 +4762,7 @@ def populate_replies_json(base_dir: str, nickname: str, domain: str,
                     break
             # if not in either inbox or outbox then examine the shared inbox
             if not reply_found:
-                message_id2 = message_id.replace('\n', '').replace('\r', '')
+                message_id2 = remove_eol(message_id)
                 search_filename = \
                     base_dir + \
                     '/accounts/inbox@' + \

reaction.py

@@ -31,6 +31,7 @@ from utils import save_json
 from utils import remove_post_from_cache
 from utils import get_cached_post_filename
 from utils import contains_invalid_chars
+from utils import remove_eol
 from posts import send_signed_json
 from session import post_json
 from webfinger import webfinger_handle
@@ -470,7 +471,8 @@ def _update_common_reactions(base_dir: str, emoji_content: str) -> None:
                     line = count_str + ' ' + emoji_content
                     new_common_reactions.append(line)
             else:
-                new_common_reactions.append(line.replace('\n', ''))
+                line1 = remove_eol(line)
+                new_common_reactions.append(line1)
         if not reaction_found:
             new_common_reactions.append(str(1).zfill(16) + ' ' + emoji_content)
         new_common_reactions.sort(reverse=True)

schedule.py

@@ -15,6 +15,7 @@ from utils import get_status_number
 from utils import load_json
 from utils import is_account_dir
 from utils import acct_dir
+from utils import remove_eol
 from outbox import post_message_to_outbox
 from session import create_session
 
@@ -43,7 +44,8 @@ def _update_post_schedule(base_dir: str, handle: str, httpd,
             date_str = line.split(' ')[0]
             if 'T' not in date_str:
                 continue
-            post_id = line.split(' ', 1)[1].replace('\n', '').replace('\r', '')
+            post_id1 = line.split(' ', 1)[1]
+            post_id = remove_eol(post_id1)
             post_filename = schedule_dir + post_id + '.json'
             if delete_schedule_post:
                 # delete extraneous scheduled posts

shares.py

@@ -22,6 +22,7 @@ from posts import get_person_box
 from session import post_json
 from session import post_image
 from session import create_session
+from utils import remove_eol
 from utils import has_object_string_type
 from utils import date_string_to_seconds
 from utils import date_seconds_to_string
@@ -1502,7 +1503,7 @@ def authorize_shared_items(shared_items_federated_domains: [],
         if debug:
             print('DEBUG: shared item federation should not use basic auth')
         return False
-    provided_token = auth_header.replace('\n', '').replace('\r', '').strip()
+    provided_token = remove_eol(auth_header).strip()
     if not provided_token:
         if debug:
             print('DEBUG: shared item federation token is empty')

tests.py

@@ -54,6 +54,7 @@ from follow import clear_followers
 from follow import send_follow_request_via_server
 from follow import send_unfollow_request_via_server
 from siteactive import site_is_active
+from utils import remove_eol
 from utils import text_in_file
 from utils import convert_published_to_local_timezone
 from utils import convert_to_snake_case
@@ -2741,7 +2742,7 @@ def _test_follows(base_dir: str) -> None:
         domain_found = False
         for following_domain in fp_foll:
             test_domain = following_domain.split('@')[1]
-            test_domain = test_domain.replace('\n', '').replace('\r', '')
+            test_domain = remove_eol(test_domain)
             if test_domain == 'mesh.com':
                 domain_found = True
             if test_domain not in federation_list:
@@ -2755,7 +2756,7 @@ def _test_follows(base_dir: str) -> None:
         domain_found = False
         for following_domain in fp_foll:
             test_domain = following_domain.split('@')[1]
-            test_domain = test_domain.replace('\n', '').replace('\r', '')
+            test_domain = remove_eol(test_domain)
             if test_domain == 'mesh.com':
                 domain_found = True
         assert domain_found is False
@@ -2779,7 +2780,7 @@ def _test_follows(base_dir: str) -> None:
               encoding='utf-8') as fp_foll:
         for follower_domain in fp_foll:
             test_domain = follower_domain.split('@')[1]
-            test_domain = test_domain.replace('\n', '').replace('\r', '')
+            test_domain = remove_eol(test_domain)
             if test_domain not in federation_list:
                 print(test_domain)
                 assert False
@@ -4715,7 +4716,7 @@ def _get_function_call_args(name: str, lines: [], start_line_ctr: int) -> []:
             continue
         args_str += lines[line_ctr].split(')')[0]
         break
-    return args_str.replace('\n', '').replace(' ', '').split(',')
+    return remove_eol(args_str).replace(' ', '').split(',')