✨ New commands bookmarks and favourites (and sundry) (#5)

* Support for new command "bookmarks"
* Add columns "bookmarked", "favourited", and "reblogged" to the statuses table
* Remove duplicated method save_bookmarks
* Support for new command "favourites"
* Update progress bar with total number of rows processed instead of API round trips
* Update README to include favourites command
* linters make good neighbours!
* update tests to reflect changes in 6cc141a
* remove "reblogged" column from "statuses"
* remove redundant tests
* add extraneous fields for transformer_status to strip in the test
* Integrate review feedback from @myles

README.md

@@ -52,3 +52,22 @@ statuses.
 ```console
 foo@bar:~$ mastodon-to-sqlite statuses mastodon.db
 ```
+
+## Retrieving Mastodon bookmarks
+
+The `bookmarks` command will retrieve all the details about your Mastodon
+bookmarks.
+
+```console
+foo@bar:~$ mastodon-to-sqlite bookmarks mastodon.db
+```
+
+
+## Retrieving Mastodon favourites
+
+The `favourites` command will retrieve all the details about your Mastodon
+favourites.
+
+```console
+foo@bar:~$ mastodon-to-sqlite favourites mastodon.db
+```

mastodon_to_sqlite/cli.py

@@ -110,6 +110,7 @@ def followers(db_path, auth):
     ) as bar:
         for followers in bar:
             service.save_accounts(db, followers, follower_id=account_id)
+            bar.pos = bar.pos + len(followers) - 1
 
 
 @cli.command()
@@ -146,6 +147,7 @@ def followings(db_path, auth):
     ) as bar:
         for followers in bar:
             service.save_accounts(db, followers, followed_id=account_id)
+            bar.pos = bar.pos + len(followers) - 1
 
 
 @cli.command()
@@ -182,3 +184,82 @@ def statuses(db_path, auth):
     ) as bar:
         for statuses in bar:
             service.save_statuses(db, statuses)
+            bar.pos = bar.pos + len(statuses) - 1
+
+
+@cli.command()
+@click.argument(
+    "db_path",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
+    required=True,
+)
+@click.option(
+    "-a",
+    "--auth",
+    type=click.Path(
+        file_okay=True, dir_okay=False, allow_dash=True, exists=True
+    ),
+    default="auth.json",
+    help="Path to auth.json token file",
+)
+def bookmarks(db_path, auth):
+    """
+    Save bookmarks for the authenticated user.
+    """
+    db = service.open_database(db_path)
+    client = service.get_client(auth)
+
+    authenticated_account = service.get_authenticated_account(client)
+    account_id = authenticated_account["id"]
+
+    service.save_accounts(db, [authenticated_account])
+
+    with click.progressbar(
+        service.get_bookmarks(account_id, client),
+        label="Importing bookmarks",
+        show_pos=True,
+    ) as bar:
+        for bookmarks in bar:
+            accounts = [d["account"] for d in bookmarks]
+            service.save_accounts(db, accounts)
+            service.save_activities("bookmarked", db, bookmarks)
+            bar.pos = bar.pos + len(bookmarks) - 1
+
+
+@cli.command()
+@click.argument(
+    "db_path",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
+    required=True,
+)
+@click.option(
+    "-a",
+    "--auth",
+    type=click.Path(
+        file_okay=True, dir_okay=False, allow_dash=True, exists=True
+    ),
+    default="auth.json",
+    help="Path to auth.json token file",
+)
+def favourites(db_path, auth):
+    """
+    Save favourites for the authenticated user.
+    """
+    db = service.open_database(db_path)
+    client = service.get_client(auth)
+
+    authenticated_account = service.get_authenticated_account(client)
+    account_id = authenticated_account["id"]
+
+    service.save_accounts(db, [authenticated_account])
+
+    with click.progressbar(
+        service.get_favourites(account_id, client),
+        label="Importing favourites",
+        show_pos=True,
+    ) as bar:
+        for favourites in bar:
+            accounts = [d["account"] for d in favourites]
+            service.save_accounts(db, accounts)
+            service.save_activities("favourited", db, favourites)
+            bar.pos = bar.pos + len(favourites) - 1

mastodon_to_sqlite/client.py

@@ -81,3 +81,13 @@ def accounts_statuses(
         self, account_id: str
     ) -> Generator[Tuple[PreparedRequest, Response], None, None]:
         return self.request_paginated("GET", f"accounts/{account_id}/statuses")
+
+    def bookmarks(
+        self,
+    ) -> Generator[Tuple[PreparedRequest, Response], None, None]:
+        return self.request_paginated("GET", "bookmarks")
+
+    def favourites(
+        self,
+    ) -> Generator[Tuple[PreparedRequest, Response], None, None]:
+        return self.request_paginated("GET", "favourites")

mastodon_to_sqlite/service.py

@@ -78,6 +78,29 @@ def build_database(db: Database):
     if ("account_id",) not in statuses_indexes:
         statuses_table.create_index(["account_id"])
 
+    status_activities_table = get_table("status_activities", db=db)
+    if status_activities_table.exists() is False:
+        status_activities_table.create(
+            columns={
+                "account_id": int,
+                "activity": str,  # favourited, bookmarked
+                "status_id": int,
+            },
+            pk=("account_id", "activity", "status_id"),
+            foreign_keys=(
+                ("account_id", "accounts", "id"),
+                ("status_id", "statuses", "id"),
+            ),
+        )
+
+    status_activities_indexes = {
+        tuple(i.columns) for i in status_activities_table.indexes
+    }
+    if ("account_id", "activity") not in status_activities_indexes:
+        status_activities_table.create_index(["account_id", "activity"])
+    if ("status_id", "activity") not in status_activities_indexes:
+        status_activities_table.create_index(["status_id", "activity"])
+
 
 def get_client(auth_file_path: str) -> MastodonClient:
     """
@@ -206,9 +229,12 @@ def transformer_status(status: Dict[str, Any]):
     """
     account = status.pop("account")
 
-    to_remove = [
-        k for k in status.keys() if k not in ("id", "created_at", "content")
-    ]
+    to_keep = (
+        "id",
+        "created_at",
+        "content",
+    )
+    to_remove = [k for k in status.keys() if k not in to_keep]
     for key in to_remove:
         del status[key]
 
@@ -226,3 +252,49 @@ def save_statuses(db: Database, statuses: List[Dict[str, Any]]):
         transformer_status(status)
 
     statuses_table.upsert_all(statuses, pk="id")
+
+
+def get_bookmarks(
+    account_id: str, client: MastodonClient
+) -> Generator[List[Dict[str, Any]], None, None]:
+    """
+    Get authenticated account's bookmarks.
+    """
+    for request, response in client.bookmarks():
+        yield response.json()
+
+
+def get_favourites(
+    account_id: str, client: MastodonClient
+) -> Generator[List[Dict[str, Any]], None, None]:
+    """
+    Get authenticated account's favourites.
+    """
+    for request, response in client.favourites():
+        yield response.json()
+
+
+def save_activities(type: str, db: Database, statuses: List[Dict[str, Any]]):
+    """
+    Save Mastodon activities to the SQLite database.
+    """
+    build_database(db)
+    statuses_table = get_table("statuses", db=db)
+    status_activities_table = get_table("status_activities", db=db)
+
+    for status in statuses:
+        transformer_status(status)
+
+    statuses_table.upsert_all(statuses, pk="id")
+
+    status_activities_table.upsert_all(
+        (
+            {
+                "account_id": status["account_id"],
+                "activity": type,
+                "status_id": status["id"],
+            }
+            for status in statuses
+        ),
+        pk=("account_id", "activity", "status_id"),
+    )

tests/fixtures.py

@@ -26,11 +26,17 @@
         " piñatas!"
     ),
     "account": ACCOUNT_ONE,
+    "bookmarked": False,
+    "favourited": True,
+    "replies_count": 1,
+    "reblogs_count": 3,
 }
 
 STATUS_TWO = {
     "id": "2",
     "created_at": "2021-12-20T19:46:29.073Z",
     "content": "Sleds are for suckers! Just ride on my gut!",
     "account": ACCOUNT_TWO,
+    "bookmarked": True,
+    "favourited": False,
 }

tests/test_services.py

@@ -74,3 +74,32 @@ def test_save_statuses(mock_db):
 
     assert mock_db["statuses"].exists() is True
     assert mock_db["statuses"].count == 2
+
+
+def test_save_activities(mock_db):
+    status_one = fixtures.STATUS_ONE.copy()
+    status_two = fixtures.STATUS_TWO.copy()
+
+    service.save_activities("bookmarked", mock_db, [status_one, status_two])
+
+    assert mock_db["statuses"].exists() is True
+    assert mock_db["statuses"].count == 2
+    assert mock_db["status_activities"].exists() is True
+    assert mock_db["status_activities"].count == 2
+
+
+def test_save_multiple_activity_types(mock_db):
+    status_one = fixtures.STATUS_ONE
+    status_two = fixtures.STATUS_TWO
+
+    service.save_activities(
+        "bookmarked", mock_db, [status_one.copy(), status_two.copy()]
+    )
+    service.save_activities(
+        "favourited", mock_db, [status_one.copy(), status_two.copy()]
+    )
+
+    assert mock_db["statuses"].exists() is True
+    assert mock_db["statuses"].count == 2
+    assert mock_db["status_activities"].exists() is True
+    assert mock_db["status_activities"].count == 4