diff --git a/hydroshare/hydroshare.py b/hydroshare/hydroshare.py index 37a8aa33..46795c4e 100644 --- a/hydroshare/hydroshare.py +++ b/hydroshare/hydroshare.py @@ -52,6 +52,7 @@ GITHUB_ORG, GITHUB_REPO, ) +from spam_patterns.patterns_re import patterns SPAM_DATA_STREAM_NAME = "cuahsi-quality-spam-data-stream" SPAM_DATA_STREAM_CONFIG = Config( @@ -1814,59 +1815,7 @@ def print_formatted_result(detected_resources, classification): f"\nThere are {len(links)} resources at HydroShare " f'"Site Map" page.\n' ) - spam_patterns = [ - "amazing", - "business", - "cheap[est]?", - "credit[s]?", - "customer[s]?", - r"\bdeal[s]?\b", - "phone number", - "price", - "4free", - # US phone number format (1-[3 digits]-[3 digits]-[4 digits] - # r'' is a 'raw' string (backslash symbol is treated as a literal - # backslash). - r"\d-[\d]{3}-[\d]{3}-[\d]{4}", - "airline[s]?", - "baggage", - "booking", - "flight[s]?", - r"\breservation\b", - "vacation[al]?", - "ticket[s]?", - r"\baccount\b", - "antivirus", - "cleaner", - "cookies", - "[e]?mail", - "laptop", - "password", - "sign up", - "sign in", - "wi[-]?fi", - # r'' is a 'raw' string (backslash symbol is treated as a literal - # backslash). - # '\b' stands for 'word boundary'. - r"\bgoogle\b", - "android", - r"\bchrome\b", - r"\bapple\b", - "icloud", - r"\bios\b", - "iphone", - r"\bmac\b", - "macbook", - "macos", - "facebook", - "microsoft", - "internet explorer", - "adult", - "escort", - "porn", - "xxx", - ] - spam_resources = check_links_against_patterns(links, spam_patterns) + spam_resources = check_links_against_patterns(links, patterns) print_formatted_result(spam_resources, classification="potential spam") # Not related to spam, but these are potentially useless resources. diff --git a/hydroshare/spam_patterns.py b/hydroshare/spam_patterns.py deleted file mode 100644 index 1689c1ac..00000000 --- a/hydroshare/spam_patterns.py +++ /dev/null @@ -1,429 +0,0 @@ -patterns = [ - "Act Now", - "Buy", - "Call now", - "Click", - "Click below", - "Click here", - "Compare", - "Do it today", - "Don't delete", - "Free", - "Get", - "Get", - "Give it away", - "Open", - "Opt in", - "Order", - "Order now", - "Print from signature", - "Print out and fax", - "See for yourself", - "Sign up free today", - "Stop", - "Unsubscribe", - "Visit our website", - "Congratulations", - "Giving away", - "Prize", - "Prizes", - "Talks about prizes", - "Who really wins?", - "Win", - "Winner", - "Won", - "You are a winner!", - "You have been selected", - "You're a Winner!", - "Confidentially on all orders", - "Order today", - "Orders shipped by", - "Outstanding values", - "Sample", - "Wants credit card", - "What are you waiting for?", - "Hidden assets", - "$$$", - "Additional income", - "Apply Online", - "Avoid bankruptcy", - "Be your own boss", - "Beneficiary", - "Big bucks", - "Billion", - "Billion dollars", - "Calling creditors", - "Cash", - "Cash bonus", - "Cashcashcash", - "Casino", - "Cents on the dollar", - "Check", - "Check or money order", - "Collect child support", - "Consolidate debt and credit", - "Consolidate your debt", - "Cost", - "Costs", - "Credit", - "Credit bureaus", - "Credit card offers", - "Discount", - "Double your", - "Double your income", - "Earn", - "Earn $", - "Earn extra cash", - "Earn per week", - "Easy terms", - "Eliminate bad credit", - "Eliminate debt", - "Expect to earn", - "Extra income", - "Fast cash", - "Financial freedom", - "Financially independent", - "For free", - "For just $xxx", - "Free grant money", - "Free investment", - "Free money", - "Get out of debt", - "Get paid", - "Hidden assets", - "Hidden charges", - "Home based", - "Home employment", - "Homebased business", - "Income", - "Income from home", - "Investment", - "Investment decision", - "Leave", - "Loans", - "Lower interest rate", - "Lower interest rates", - "Lower monthly payment", - "Lower your mortgage rate", - "Lowest insurance rates", - "Make $", - "Make money", - "Million", - "Million dollars", - "Money", - "Money making", - "Mortgage", - "Mortgage rates", - "Nigerian", - "No credit check", - "No fees", - "No investment", - "Off shore", - "One time", - "Online biz opportunity", - "Only $", - "Pennies a day", - "Potential earnings", - "Pre-approved", - "Price", - "Prices", - "Print form signature", - "Profits", - "Pure Profits", - "Refinance", - "Refinance home", - "Refinanced home", - "Risk free", - "Save up to", - "Serious cash", - "Social security number", - "Stock alert", - "Stock disclaimer statement", - "Stock pick", - "Subject to cash", - "Subject to credit", - "They keep your money - no refund", - "Thousands", - "Unsecured credit", - "Unsecured credit/debt", - "Unsecured debt", - "US dollars", - "Why pay more?", - "Winner", - "Winning", - "Work at home", - "Work from home", - "Your income", - "All natural", - "Cell phone cancer scam", - "Cures baldness", - "Diagnostics", - "Don't hesitate", - "Fast Viagra delivery", - "Human growth hormone", - "Insurance", - "Life", - "Life Insurance", - "Lose", - "Lose weight", - "Lose weight spam", - "Medicine", - "No claim forms", - "No medical exams", - "Online pharmacy", - "Removes wrinkles", - "Reverses", - "Reverses aging", - "Stop snoring", - "Valium", - "Viagra", - "Viagra and other drugs", - "Vicodin", - "Weight loss", - "Claims to be in accordance with some spam law", - "Claims to be legal", - "In accordance with laws", - "100% free", - "Accept credit cards", - "Access", - "Accordingly", - "Ad", - "Amazing", - "Auto email removal", - "Being a member", - "Best price", - "Billing address", - "Bulk email", - "Cancel at any time", - "Cannot be combined with any other offer", - "Cards accepted", - "Dear [email/friend/somebody]", - "Dig up dirt on friends", - "Direct email", - "Direct marketing", - "Email harvest", - "Email marketing", - "Free gift", - "Free leads", - "Free preview", - "Free sample", - "Free trial", - "Full refund", - "Get started now", - "Gift certificate", - "Give it away", - "Great offer", - "If only it were that easy", - "Important information regarding", - "Increase traffic", - "Info you requested", - "Information you requested", - "Internet market", - "Internet marketing", - "Maintained", - "Marketing", - "Marketing solutions", - "Mass email", - "Member", - "Member stuff", - "Message contains", - "Message contains disclaimer", - "MLM", - "Multi level marketing", - "Multi-level marketing", - "Name brand", - "No catch", - "No disappointment", - "No hidden", - "Not intended", - "Obligation", - "Offer", - "Online marketing", - "Order status", - "Please read", - "Safeguard notice", - "Satisfaction", - "Satisfaction guaranteed", - "Search engine listings", - "Search engines", - "Sent in compliance", - "Solution", - "Special promotion", - "Supplies are limited", - "Talks about hidden charges", - "Tells you it's an ad", - "Terms and conditions", - "The following form", - "This isn't junk", - "This isn't spam", - "Trial", - "We hate spam", - "Web traffic", - "Weekend getaway", - "Bill 1618", - "Copy DVDs", - "Laser printer", - "Online degree", - "Rolex", - "S 1618", - "Section 301", - "Stainless steel", - "#1", - "4U", - "Amazing stuff", - "Brand new pager", - "Drastically reduced", - "For instant access", - "For you", - "Free", - "Free info", - "Free installation", - "Free Instant", - "Free membership", - "Free offer", - "Free priority mail", - "Free quote", - "Free website", - "Limited time only", - "Lowest Price", - "Month trial offer", - "More Internet Traffic", - "New customers only", - "No age restrictions", - "No experience", - "No purchase necessary", - "Offer expires", - "Only", - "Reserves the right", - "Time limited", - "Vacation offers", - "100% Satisfied", - "50% off", - "Act Now!", - "Act now!", - "Affordable", - "All new", - "As seen on", - "Bargain", - "Be amazed", - "Bonus", - "Buy direct", - "Buying judgments", - "Call", - "Call free", - "Can't live without", - "Cheap", - "Claims not to be selling anything", - "Clearance", - "Click to remove", - "Collect", - "Compare rates", - "Compete for your business", - "Deal", - "Don't hesitate!", - "Explode your business", - "F r e e", - "Fantastic deal", - "For just $(insert whatever amount)", - "For Only", - "Free consultation", - "Free consultation", - "Free DVD", - "Free hosting", - "Get it now", - "Guarantee", - "Guaranteed", - "Have you been turned down?", - "Increase sales", - "Increase your sales", - "Incredible deal", - "It's effective", - "Join millions", - "Join millions of Americans", - "Lifetime", - "limited time", - "Limited time offer", - "Long distance phone offer", - "Mail in order form", - "Money back", - "New domain extensions", - "No cost", - "No gimmick", - "No inventory", - "No middleman", - "No obligation", - "No questions asked", - "No selling", - "No strings attached", - "No-obligation", - "Now", - "Now only", - "One hundred percent free", - "One hundred percent guaranteed", - "Opportunity", - "Order shipped by", - "Problem", - "Promise you", - "Quote", - "Quotes", - "Sale", - "Sales", - "Save $", - "Save big money", - "See for yourself", - "Serious only", - "Sign up free today", - "Strong buy", - "Stuff on sale", - "Success", - "The best rates", - "They're just giving it away", - "This isn't a scam", - "This won't last", - "Unlimited", - "Urgent", - "Warranty", - "We honor all", - "While supplies last", - "While you sleep", - "Will not believe your eyes", - "Acceptance", - "Addresses on CD", - "Beverage", - "Celebrity", - "Chance", - "Congratulations", - "Copy accurately", - "Dormant", - "Form", - "Freedom", - "Friend", - "Hello", - "Here", - "Hidden", - "Home", - "Legal", - "Luxury car", - "Medium", - "Meet singles", - "Miracle", - "Never", - "Not spam", - "Passwords", - "Performance", - "Phone", - "Print out and fax", - "Priority mail", - "Produced and sent out", - "Real thing", - "Removal instructions", - "Remove", - "Round the world", - "Score with babes", - "Shopper", - "Teen", - "Undisclosed recipient", - "Unsolicited", - "Vacation", - "Wife", -] diff --git a/requirements.txt b/requirements.txt index fddbd4d4..e08dfa15 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,5 @@ boto3>=1.24.21,<2.0.0 python-dotenv>=0.20.0<=0.21.0 beautifulsoup4>=4.11.1<=4.12.0 webdriver-manager>=3.8.2<=3.9.0 -chromedriver-autoinstaller>=0.4.0<=1.0.0 \ No newline at end of file +chromedriver-autoinstaller>=0.4.0<=1.0.0 +spam_patterns @ git+https://github.com/CUAHSI/spam_patterns.git@0.0.1 \ No newline at end of file