Applications réelles de nettoyage de texte
Maintenant que nous avons appris différentes méthodes pour filtrer les caractères non alphanumériques, appliquons ces techniques à des scénarios du monde réel.
Nettoyage de l'entrée utilisateur
L'entrée utilisateur contient souvent des caractères inattendus qui doivent être nettoyés. Créons un fichier appelé text_cleaning_app.py pour démontrer cela :
## Text cleaning application for user input
import re
def clean_username(username):
"""Cleans a username by removing special characters and spaces"""
return re.sub(r'[^a-zA-Z0-9_]', '', username)
def clean_search_query(query):
"""Preserves alphanumeric chars and spaces, replaces multiple spaces with one"""
## First, replace non-alphanumeric chars (except spaces) with empty string
cleaned = re.sub(r'[^a-zA-Z0-9\s]', '', query)
## Then, replace multiple spaces with a single space
cleaned = re.sub(r'\s+', ' ', cleaned)
## Finally, strip leading and trailing spaces
return cleaned.strip()
## Simulate user input
usernames = [
"john.doe",
"user@example",
"my username!",
"admin_123"
]
search_queries = [
"python programming",
"how to filter?! special chars",
"$ regex examples $",
" string methods "
]
## Clean and display usernames
print("Username Cleaning:")
print("-" * 40)
for username in usernames:
cleaned = clean_username(username)
print(f"Original: {username}")
print(f"Cleaned: {cleaned}")
print("-" * 40)
## Clean and display search queries
print("\nSearch Query Cleaning:")
print("-" * 40)
for query in search_queries:
cleaned = clean_search_query(query)
print(f"Original: '{query}'")
print(f"Cleaned: '{cleaned}'")
print("-" * 40)
Exécutez ce fichier :
python3 /home/labex/project/text_cleaning_app.py
Gestion des données de fichier
Créons un fichier texte d'exemple et nettoyons-le. Tout d'abord, créez un fichier appelé sample_data.txt avec le contenu suivant :
User1: [email protected] (Active: Yes)
User2: [email protected] (Active: No)
User3: admin#[email protected] (Active: Yes)
Notes: Users should change their passwords every 90 days!
Vous pouvez créer ce fichier à l'aide de l'éditeur WebIDE. Maintenant, créons un fichier appelé file_cleaner.py pour nettoyer ces données :
## File cleaning application
import re
def extract_emails(text):
"""Extract email addresses from text"""
## Simple regex for email extraction
email_pattern = r'[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+'
return re.findall(email_pattern, text)
def extract_usernames(text):
"""Extract the username part from email addresses"""
emails = extract_emails(text)
usernames = [email.split('@')[0] for email in emails]
return usernames
def clean_usernames(usernames):
"""Clean usernames by removing non-alphanumeric characters"""
return [re.sub(r'[^a-zA-Z0-9]', '', username) for username in usernames]
## Read the sample data file
try:
with open('/home/labex/project/sample_data.txt', 'r') as file:
data = file.read()
except FileNotFoundError:
print("Error: sample_data.txt file not found!")
exit(1)
## Process the data
print("File Cleaning Results:")
print("-" * 50)
print("Original data:")
print(data)
print("-" * 50)
## Extract emails
emails = extract_emails(data)
print(f"Extracted {len(emails)} email addresses:")
for email in emails:
print(f" - {email}")
## Extract and clean usernames
usernames = extract_usernames(data)
cleaned_usernames = clean_usernames(usernames)
print("\nUsername extraction and cleaning:")
for i, (original, cleaned) in enumerate(zip(usernames, cleaned_usernames)):
print(f" - User {i+1}: {original} → {cleaned}")
print("-" * 50)
Exécutez ce fichier :
python3 /home/labex/project/file_cleaner.py
Différentes méthodes de filtrage peuvent avoir des caractéristiques de performance différentes. Créons un fichier appelé performance_test.py pour les comparer :
## Performance comparison of different filtering methods
import re
import time
def filter_with_loop(text):
"""Filter using a simple loop"""
result = ""
for char in text:
if char.isalnum():
result += char
return result
def filter_with_comprehension(text):
"""Filter using list comprehension"""
return ''.join(char for char in text if char.isalnum())
def filter_with_filter_function(text):
"""Filter using the built-in filter function"""
return ''.join(filter(str.isalnum, text))
def filter_with_regex(text):
"""Filter using regular expressions"""
return re.sub(r'[^a-zA-Z0-9]', '', text)
def filter_with_translate(text):
"""Filter using string.translate"""
## Create a translation table that maps all non-alphanumeric chars to None
from string import ascii_letters, digits
allowed = ascii_letters + digits
translation_table = str.maketrans('', '', ''.join(c for c in map(chr, range(128)) if c not in allowed))
return text.translate(translation_table)
## Generate test data (a string with a mix of alphanumeric and other characters)
test_data = "".join(chr(i) for i in range(33, 127)) * 1000 ## ASCII printable characters repeated
## Define the filtering methods to test
methods = [
("Simple Loop", filter_with_loop),
("List Comprehension", filter_with_comprehension),
("Filter Function", filter_with_filter_function),
("Regular Expression", filter_with_regex),
("String Translate", filter_with_translate)
]
print("Performance Comparison:")
print("-" * 60)
print(f"Test data length: {len(test_data)} characters")
print("-" * 60)
print(f"{'Method':<20} | {'Time (seconds)':<15} | {'Characters Removed':<20}")
print("-" * 60)
## Test each method
for name, func in methods:
start_time = time.time()
result = func(test_data)
end_time = time.time()
execution_time = end_time - start_time
chars_removed = len(test_data) - len(result)
print(f"{name:<20} | {execution_time:<15.6f} | {chars_removed:<20}")
print("-" * 60)
Exécutez ce fichier :
python3 /home/labex/project/performance_test.py
La sortie vous montrera quelle méthode est la plus efficace pour filtrer les caractères non alphanumériques, ce qui peut être important lors du traitement de grandes quantités de données textuelles.
