From 4b8b71c392cd4381ac8de9826e7ca764ea0eda7f Mon Sep 17 00:00:00 2001
From: blankie <blankie@nixnetmail.com>
Date: Fri, 19 Jan 2024 14:31:21 +1100
Subject: [PATCH 1/3] [photos18] add support

---
 docs/supportedsites.md           |   6 ++
 gallery_dl/extractor/__init__.py |   1 +
 gallery_dl/extractor/photos18.py | 103 +++++++++++++++++++++++++++++++
 test/results/photos18.py         |  19 ++++++
 4 files changed, 129 insertions(+)
 create mode 100644 gallery_dl/extractor/photos18.py
 create mode 100644 test/results/photos18.py

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 4a6d8bd249..9c8cf0d4fb 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -619,6 +619,12 @@ Consider all listed sites to potentially be NSFW.
     <td>Albums, individual Images</td>
     <td></td>
 </tr>
+<tr>
+    <td>Photos18</td>
+    <td>https://photos18.com</td>
+    <td>Albums, Lists</td>
+    <td></td>
+</tr>
 <tr>
     <td>PhotoVogue</td>
     <td>https://www.vogue.com/photovogue/</td>
diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py
index 863089176a..c8721f202b 100644
--- a/gallery_dl/extractor/__init__.py
+++ b/gallery_dl/extractor/__init__.py
@@ -114,6 +114,7 @@
     "patreon",
     "philomena",
     "photobucket",
+    "photos18",
     "photovogue",
     "picarto",
     "piczel",
diff --git a/gallery_dl/extractor/photos18.py b/gallery_dl/extractor/photos18.py
new file mode 100644
index 0000000000..11415ee6e4
--- /dev/null
+++ b/gallery_dl/extractor/photos18.py
@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Extractors for https://photos18.com"""
+
+from .common import Extractor, Message
+from .. import text
+
+
+BASE_PATTERN = r"(?:https?://)(?:www\.)?photos18\.com"
+SORTING_METHODS = "(created|hits|views|score|likes)"
+
+
+class Photos18Extractor(Extractor):
+    """Base class for Photos18 extractors"""
+    category = "photos18"
+    directory_fmt = ("{category}", "{category_name}")
+    filename_fmt = "{category}_{title}_{num:>02}.{extension}"
+    archive_fmt = "{filename}"
+    root = "https://www.photos18.com"
+
+    def items(self):
+        for post_id in self.posts():
+            url = self.root + "/v/" + post_id
+            page = self.request(url).text
+            extr = text.extract_from(page)
+
+            title = text.unescape(extr(
+                '<meta property="og:title" content="', '"'))
+            category_id = int(extr(
+                '<li class="breadcrumb-item"><a href="/cat/', '"'))
+            category_name = text.unescape(extr('>', '<'))
+            date = text.parse_datetime(extr('"datePublished":"', '"'))
+
+            urls = []
+            while True:
+                url = text.unescape(extr(
+                    '<div class="my-2 imgHolder"><a href="', '"'))
+                if not url:
+                    break
+
+                urls.append(url)
+
+            data = {
+                "post_id": post_id,
+                "title": title,
+                "category_id": category_id,
+                "category_name": category_name,
+                "date": date,
+                "count": len(urls),
+                "_http_headers": {"Referer": self.root},
+            }
+            yield Message.Directory, data
+            for data["num"], url in enumerate(urls, 1):
+                yield Message.Url, url, text.nameext_from_url(url, data)
+
+
+class Photos18AlbumExtractor(Photos18Extractor):
+    """Extractor for a single album URL"""
+    subcategory = "album"
+    pattern = BASE_PATTERN + r"/v/(\w+)"
+    example = "https://www.photos18.com/v/ID"
+
+    def __init__(self, match):
+        Photos18Extractor.__init__(self, match)
+        self.post_id = match.group(1)
+
+    def posts(self):
+        return (self.post_id,)
+
+
+class Photos18ListExtractor(Photos18Extractor):
+    """Extractor for a list of posts"""
+    subcategory = "list"
+    pattern = (BASE_PATTERN + r"(?:/|/cat/(\d+)(?:/" + SORTING_METHODS +
+               r")?|/sort/" + SORTING_METHODS + r"|/q/([^/?#]+))?"
+               r"(?:\?([^#]*))?(?:#.*)?$")
+    example = "https://www.photos18.com/cat/1"
+
+    def __init__(self, match):
+        Photos18Extractor.__init__(self, match)
+        query = text.parse_query(match.group(5))
+        self.q = text.unquote(match.group(4) or "") or query.get("q")
+        self.category_id = match.group(1) or query.get("category_id")
+        self.sort = match.group(2) or match.group(3) or query.get("sort")
+        self.page = query.get("page")
+
+    def posts(self):
+        query = {}
+        if self.q:
+            query["q"] = self.q
+        if self.category_id:
+            query["category_id"] = self.category_id
+        if self.sort:
+            query["sort"] = self.sort
+        if self.page:
+            query["page"] = self.page
+
+        page = self.request(self.root, params=query).text
+        return text.extract_iter(page, '<a class="visited" href="/v/', '"')
diff --git a/test/results/photos18.py b/test/results/photos18.py
new file mode 100644
index 0000000000..5f208aa70b
--- /dev/null
+++ b/test/results/photos18.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+from gallery_dl.extractor import photos18
+
+
+__tests__ = (
+{
+    "#url"     : "https://www.photos18.com/v/3BXQy",
+    "#category": ("", "photos18", "album"),
+    "#class"   : photos18.Photos18AlbumExtractor,
+    "#count"   : 12,
+    "#sha1_url": "2f9442f34f31bafdd6d57f4954674348b38ef284",
+},
+
+)

From 2b3b08b2b8108ec1103530192ce38ff5db895b19 Mon Sep 17 00:00:00 2001
From: blankie <blankie@nixnetmail.com>
Date: Sat, 20 Jan 2024 01:50:17 +1100
Subject: [PATCH 2/3] [photos18] fix getting post titles

---
 gallery_dl/extractor/photos18.py |  4 ++--
 test/results/photos18.py         | 15 +++++++++++++++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/gallery_dl/extractor/photos18.py b/gallery_dl/extractor/photos18.py
index 11415ee6e4..043b702557 100644
--- a/gallery_dl/extractor/photos18.py
+++ b/gallery_dl/extractor/photos18.py
@@ -28,12 +28,12 @@ def items(self):
             page = self.request(url).text
             extr = text.extract_from(page)
 
-            title = text.unescape(extr(
-                '<meta property="og:title" content="', '"'))
             category_id = int(extr(
                 '<li class="breadcrumb-item"><a href="/cat/', '"'))
             category_name = text.unescape(extr('>', '<'))
             date = text.parse_datetime(extr('"datePublished":"', '"'))
+            title = text.unescape(extr(
+                '<h1 class="title py-1">', '</h1>')).strip()
 
             urls = []
             while True:
diff --git a/test/results/photos18.py b/test/results/photos18.py
index 5f208aa70b..8b91a218c2 100644
--- a/test/results/photos18.py
+++ b/test/results/photos18.py
@@ -14,6 +14,21 @@
     "#class"   : photos18.Photos18AlbumExtractor,
     "#count"   : 12,
     "#sha1_url": "2f9442f34f31bafdd6d57f4954674348b38ef284",
+
+    "title"        : "Peachmilky Nanami, Mami Rent-a-Girlfriend",
+    "category_id"  : 8,
+    "category_name": "COSPLAY",
+},
+
+{
+    "#url"     : "https://www.photos18.com/v/jMMn2",
+    "#category": ("", "photos18", "album"),
+    "#class"   : photos18.Photos18AlbumExtractor,
+    "#count"   : 36,
+
+    "title"        : "姐姐說兒童節幫我\"轉大人\"Kenna James - Stepbro Accidentally Cums In Stepsister's Pussy",
+    "category_id"  : 1,
+    "category_name": "歐美寫真",
 },
 
 )

From 3f82852650e1106eee2aef5ad2a3bb1e2300373c Mon Sep 17 00:00:00 2001
From: blankie <blankie@nixnetmail.com>
Date: Sat, 20 Jan 2024 02:03:02 +1100
Subject: [PATCH 3/3] [photos18] do pagination

Photos18ListExtractor bails if there are no posts within a page, since
(as of this commit), some pages might have improper navigation bars
(for example, https://www.photos18.com/cat/2?page=75 still has a link
to the next page despite the list of posts being far outside that range)
---
 gallery_dl/extractor/photos18.py | 90 +++++++++++++++++---------------
 test/results/photos18.py         |  8 +++
 2 files changed, 57 insertions(+), 41 deletions(-)

diff --git a/gallery_dl/extractor/photos18.py b/gallery_dl/extractor/photos18.py
index 043b702557..b562b13ec0 100644
--- a/gallery_dl/extractor/photos18.py
+++ b/gallery_dl/extractor/photos18.py
@@ -22,41 +22,6 @@ class Photos18Extractor(Extractor):
     archive_fmt = "{filename}"
     root = "https://www.photos18.com"
 
-    def items(self):
-        for post_id in self.posts():
-            url = self.root + "/v/" + post_id
-            page = self.request(url).text
-            extr = text.extract_from(page)
-
-            category_id = int(extr(
-                '<li class="breadcrumb-item"><a href="/cat/', '"'))
-            category_name = text.unescape(extr('>', '<'))
-            date = text.parse_datetime(extr('"datePublished":"', '"'))
-            title = text.unescape(extr(
-                '<h1 class="title py-1">', '</h1>')).strip()
-
-            urls = []
-            while True:
-                url = text.unescape(extr(
-                    '<div class="my-2 imgHolder"><a href="', '"'))
-                if not url:
-                    break
-
-                urls.append(url)
-
-            data = {
-                "post_id": post_id,
-                "title": title,
-                "category_id": category_id,
-                "category_name": category_name,
-                "date": date,
-                "count": len(urls),
-                "_http_headers": {"Referer": self.root},
-            }
-            yield Message.Directory, data
-            for data["num"], url in enumerate(urls, 1):
-                yield Message.Url, url, text.nameext_from_url(url, data)
-
 
 class Photos18AlbumExtractor(Photos18Extractor):
     """Extractor for a single album URL"""
@@ -68,8 +33,39 @@ def __init__(self, match):
         Photos18Extractor.__init__(self, match)
         self.post_id = match.group(1)
 
-    def posts(self):
-        return (self.post_id,)
+    def items(self):
+        url = self.root + "/v/" + self.post_id
+        page = self.request(url).text
+        extr = text.extract_from(page)
+
+        category_id = int(extr(
+            '<li class="breadcrumb-item"><a href="/cat/', '"'))
+        category_name = text.unescape(extr('>', '<'))
+        date = text.parse_datetime(extr('"datePublished":"', '"'))
+        title = text.unescape(extr(
+            '<h1 class="title py-1">', '</h1>')).strip()
+
+        urls = []
+        while True:
+            url = text.unescape(extr(
+                '<div class="my-2 imgHolder"><a href="', '"'))
+            if not url:
+                break
+
+            urls.append(url)
+
+        data = {
+            "post_id": self.post_id,
+            "title": title,
+            "category_id": category_id,
+            "category_name": category_name,
+            "date": date,
+            "count": len(urls),
+            "_http_headers": {"Referer": self.root},
+        }
+        yield Message.Directory, data
+        for data["num"], url in enumerate(urls, 1):
+            yield Message.Url, url, text.nameext_from_url(url, data)
 
 
 class Photos18ListExtractor(Photos18Extractor):
@@ -86,9 +82,9 @@ def __init__(self, match):
         self.q = text.unquote(match.group(4) or "") or query.get("q")
         self.category_id = match.group(1) or query.get("category_id")
         self.sort = match.group(2) or match.group(3) or query.get("sort")
-        self.page = query.get("page")
+        self.page = int(query.get("page") or 1)
 
-    def posts(self):
+    def items(self):
         query = {}
         if self.q:
             query["q"] = self.q
@@ -99,5 +95,17 @@ def posts(self):
         if self.page:
             query["page"] = self.page
 
-        page = self.request(self.root, params=query).text
-        return text.extract_iter(page, '<a class="visited" href="/v/', '"')
+        while True:
+            has_post = False
+            page = self.request(self.root, params=query).text
+
+            for i in text.extract_iter(
+                    page, '<a class="visited" href="/v/', '"'):
+                has_post = True
+                url = self.root + "/v/" + i
+                data = {"_extractor": Photos18AlbumExtractor}
+                yield Message.Queue, url, data
+
+            if not has_post or '<li class="page-item next">' not in page:
+                break
+            query["page"] += 1
diff --git a/test/results/photos18.py b/test/results/photos18.py
index 8b91a218c2..72bbf7ca75 100644
--- a/test/results/photos18.py
+++ b/test/results/photos18.py
@@ -31,4 +31,12 @@
     "category_name": "歐美寫真",
 },
 
+{
+    "#url"     : "https://www.photos18.com",
+    "#category": ("", "photos18", "list"),
+    "#class"   : photos18.Photos18ListExtractor,
+    "#range"   : "1-200",
+    "#count"   : 200,
+},
+
 )