Merge branch 'master' into auto-unassigner

.github/workflows/javascript_tests.yml

@@ -24,7 +24,8 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v3
         with:
-          node-version: '20' # Should match what's in our Dockerfile
+          # Should match what's in our Dockerfile
+          node-version: '20' # Also update the `key` in the `with` map, below
       - uses: actions/cache@v3
         id: npm-cache
         with:
@@ -35,7 +36,7 @@ jobs:
           # `node_modules` directly.
           path: 'node_modules'
           # Note the version number in this string -- update it when updating Node!
-          key: ${{ runner.os }}-node16-${{ hashFiles('**/package-lock.json') }}
+          key: ${{ runner.os }}-node20-${{ hashFiles('**/package-lock.json') }}
       - if: steps.npm-cache.outputs.cache-hit != 'true'
         run: npm install --no-audit
       - run: npm run lint

.github/workflows/new_comment_digest.yml

@@ -0,0 +1,21 @@
+name: new_comment_digest
+on:
+  schedule:  # 08:30 daily
+    - cron: '30 8 * * *'
+  workflow_dispatch:
+permissions:
+  contents: read
+
+jobs:
+  new_comment_digest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.x
+      - run: pip install requests
+      - run: scripts/gh_scripts/issue_comment_bot.py 24 "$SLACK_CHANNEL" "$SLACK_TOKEN"
+        env:
+          SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
+          SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL_ABC_TEAM_PLUS }}

.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
       - id: auto-walrus
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.13
+    rev: v0.1.14
     hooks:
       - id: ruff
 

conf/coverstore.yml

@@ -12,4 +12,5 @@ sentry:
     enabled: false
     # Dummy endpoint; where sentry logs are sent to
     dsn: 'https://[email protected]/0'
+    traces_sample_rate: 1.0
     environment: 'local'

conf/solr/conf/managed-schema.xml

@@ -514,8 +514,8 @@
     <fieldType name="text_title_sort" class="solr.TextField" sortMissingLast="false" omitNorms="true">
       <analyzer>
         <tokenizer name="keyword" />
-        <filter name="lowercase" />
         <filter name="trim" />
+        <filter name="icuFolding"/>
         <filter name="patternReplace"
                 pattern="^(an? |the |l[aeo]s? |l'|de la |el |il |un[ae]? |du |de[imrst]? |das |ein |eine[mnrs]? |bir )(.*)"
                 replacement="$2, $1"

openlibrary/core/lending.py

@@ -308,7 +308,22 @@ def update_availability_schema_to_v2(v1_resp, ocaid):
 
     url = '{}?{}={}'.format(config_ia_availability_api_v2_url, key, ','.join(ids))
     try:
-        response = requests.get(url, timeout=config_http_request_timeout)
+        client_ip = web.ctx.env.get('HTTP_X_FORWARDED_FOR', 'ol-internal')
+        params = {"scope": "printdisabled"}
+        headers = {
+            "x-preferred-client-id": client_ip,
+            "x-application-id": "openlibrary",
+        }
+        if config_ia_ol_metadata_write_s3:
+            headers["authorization"] = "LOW {s3_key}:{s3_secret}".format(
+                **config_ia_ol_metadata_write_s3
+            )
+
+        # Make authenticated request to Bulk Availability API
+        response = requests.get(
+            url, params=params, headers=headers, timeout=config_http_request_timeout
+        )
+
         items = response.json().get('responses', {})
         for pkey in items:
             ocaid = pkey if key == 'identifier' else items[pkey].get('identifier')

openlibrary/coverstore/code.py

@@ -239,25 +239,17 @@ def notfound():
             ):
                 return read_file(config.default_image)
             elif is_valid_url(i.default):
-                raise web.seeother(i.default)
+                return web.seeother(i.default)
             else:
-                raise web.notfound("")
-
-        def redirect(id):
-            size_part = size and ("-" + size) or ""
-            url = f"/{category}/id/{id}{size_part}.jpg"
-
-            if query := web.ctx.env.get('QUERY_STRING'):
-                url += '?' + query
-            raise web.found(url)
+                return web.notfound("")
 
         if key == 'isbn':
             value = value.replace("-", "").strip()  # strip hyphens from ISBN
             value = self.query(category, key, value)
         elif key == 'ia':
             url = self.get_ia_cover_url(value, size)
             if url:
-                raise web.found(url)
+                return web.found(url)
             else:
                 value = None  # notfound or redirect to default. handled later.
         elif key != 'id':
@@ -269,7 +261,7 @@ def redirect(id):
         # redirect to archive.org cluster for large size and original images whenever possible
         if size in ("L", "") and self.is_cover_in_cluster(value):
             url = zipview_url_from_id(int(value), size)
-            raise web.found(url)
+            return web.found(url)
 
         # covers_0008 batches [_00, _82] are tar'd / zip'd in archive.org items
         if isinstance(value, int) or value.isnumeric():  # noqa: SIM102
@@ -281,7 +273,7 @@ def redirect(id):
                 item_file = f"{pid}{'-' + size.upper() if size else ''}"
                 path = f"{item_id}/{item_tar}/{item_file}.jpg"
                 protocol = web.ctx.protocol
-                raise web.found(f"{protocol}://archive.org/download/{path}")
+                return web.found(f"{protocol}://archive.org/download/{path}")
 
         d = self.get_details(value, size.lower())
         if not d:
@@ -291,7 +283,7 @@ def redirect(id):
         if key == 'id':
             etag = f"{d.id}-{size.lower()}"
             if not web.modified(trim_microsecond(d.created), etag=etag):
-                raise web.notmodified()
+                return web.notmodified()
 
             web.header('Cache-Control', 'public')
             # this image is not going to expire in next 100 years.
@@ -306,14 +298,14 @@ def redirect(id):
             from openlibrary.coverstore import archive
 
             if d.id >= 8_820_000 and d.uploaded and '.zip' in d.filename:
-                raise web.found(
+                return web.found(
                     archive.Cover.get_cover_url(
                         d.id, size=size, protocol=web.ctx.protocol
                     )
                 )
             return read_image(d, size)
         except OSError:
-            raise web.notfound()
+            return web.notfound()
 
     def get_ia_cover_url(self, identifier, size="M"):
         url = "https://archive.org/metadata/%s/metadata" % identifier

openlibrary/i18n/pt/messages.po

@@ -638,7 +638,7 @@ msgstr "Pedir emprestado \"%(title)s\""
 
 #: ReadButton.html:15 type/list/embed.html:79 widget.html:39
 msgid "Borrow"
-msgstr "Emprestar"
+msgstr "Empréstimo"
 
 #: widget.html:45
 #, python-format
@@ -3172,7 +3172,7 @@ msgstr "Explorar o Centro de Desenvolvimento da Open Library"
 
 #: lib/nav_foot.html:37 lib/nav_head.html:25
 msgid "Developer Center"
-msgstr "Centro de Desenvolvedor"
+msgstr "Central de desenvolvedor"
 
 #: lib/nav_foot.html:38
 msgid "Explore Open Library APIs"
@@ -3204,7 +3204,7 @@ msgstr "Adicionar um novo livro à Open Library"
 
 #: lib/nav_foot.html:47
 msgid "Help Center"
-msgstr "Centro de ajuda"
+msgstr "Central de ajuda"
 
 #: lib/nav_foot.html:48
 msgid "Problems"

openlibrary/macros/ShareModal.html

@@ -6,7 +6,7 @@
 
 
 $def embed_iframe(page_url):
-  <iframe frameborder="0" width="165" height="400" src="$(page_url)/-/widget"></iframe>
+  <iframe frameborder="0" width="165" height="400" src="//$request.domain$page_url/widget"></iframe>
 
 <div class="$classes">
   $:link_markup

openlibrary/plugins/openlibrary/sentry.py

@@ -1,6 +1,6 @@
 import infogami
 from infogami.utils import delegate
-from openlibrary.utils.sentry import Sentry, SentryProcessor
+from openlibrary.utils.sentry import Sentry, InfogamiSentryProcessor
 
 sentry: Sentry | None = None
 
@@ -12,4 +12,4 @@ def setup():
     if sentry.enabled:
         sentry.init()
         delegate.add_exception_hook(lambda: sentry.capture_exception_webpy())
-        delegate.app.add_processor(SentryProcessor())
+        delegate.app.add_processor(InfogamiSentryProcessor(delegate.app))

openlibrary/utils/sentry.py

@@ -76,6 +76,7 @@ def capture_exception():
             return _internalerror()
 
         app.internalerror = capture_exception
+        app.add_processor(WebPySentryProcessor(app))
 
     def capture_exception_webpy(self):
         with sentry_sdk.push_scope() as scope:
@@ -97,12 +98,48 @@ def to_sentry_name(self) -> str:
         )
 
 
-class SentryProcessor:
+class WebPySentryProcessor:
+    def __init__(self, app: web.application):
+        self.app = app
+
+    def find_route_name(self) -> str:
+        handler, groups = self.app._match(self.app.mapping, web.ctx.path)
+        web.debug('ROUTE HANDLER', handler, groups)
+        return handler or '<other>'
+
+    def __call__(self, handler):
+        route_name = self.find_route_name()
+        hub = sentry_sdk.Hub.current
+        with sentry_sdk.Hub(hub) as hub:
+            with hub.configure_scope() as scope:
+                scope.clear_breadcrumbs()
+                scope.add_event_processor(add_web_ctx_to_event)
+
+            environ = dict(web.ctx.env)
+            # Don't forward cookies to Sentry
+            if 'HTTP_COOKIE' in environ:
+                del environ['HTTP_COOKIE']
+
+            transaction = Transaction.continue_from_environ(
+                environ,
+                op="http.server",
+                name=route_name,
+                source=TRANSACTION_SOURCE_ROUTE,
+            )
+
+            with hub.start_transaction(transaction):
+                try:
+                    return handler()
+                finally:
+                    transaction.set_http_status(int(web.ctx.status.split()[0]))
+
+
+class InfogamiSentryProcessor(WebPySentryProcessor):
     """
     Processor to profile the webpage and send a transaction to Sentry.
     """
 
-    def __call__(self, handler):
+    def find_route_name(self) -> str:
         def find_type() -> tuple[str, str] | None:
             return next(
                 (
@@ -134,27 +171,4 @@ def find_route() -> InfogamiRoute:
 
             return result
 
-        route = find_route()
-        hub = sentry_sdk.Hub.current
-        with sentry_sdk.Hub(hub) as hub:
-            with hub.configure_scope() as scope:
-                scope.clear_breadcrumbs()
-                scope.add_event_processor(add_web_ctx_to_event)
-
-            environ = dict(web.ctx.env)
-            # Don't forward cookies to Sentry
-            if 'HTTP_COOKIE' in environ:
-                del environ['HTTP_COOKIE']
-
-            transaction = Transaction.continue_from_environ(
-                environ,
-                op="http.server",
-                name=route.to_sentry_name(),
-                source=TRANSACTION_SOURCE_ROUTE,
-            )
-
-            with hub.start_transaction(transaction):
-                try:
-                    return handler()
-                finally:
-                    transaction.set_http_status(int(web.ctx.status.split()[0]))
+        return find_route().to_sentry_name()

scripts/gh_scripts/README.md

@@ -24,3 +24,27 @@ __Incorrect:__
 `node auto_unassigner.mjs --daysSince=21`
 
 The GitHub action that runs this script automatically sets `--repoOwner` to the owner of the repository that triggered the action.
+
+To quickly see a script's purpose and arguments, run the script with the `-h` or `--help` flag.
+
+## `issue_comment_bot.py`
+
+This script fetches issues that have new comments from contributors within the past number of hours, then posts a message to the team in our Slack channel.
+
+### Usage:
+This script has three positional arguments:
+```
+  hours        Fetch issues that have been updated since this many hours ago
+  channel      Issues will be published to this Slack channel
+  slack-token  Slack authentication token
+```
+
+__Running the script locally:__
+```
+docker compose exec -e PYTHONPATH=. web bash
+
+# Publish digest of new comments from the past day to #openlibrary-g:
+./scripts/gh_scripts/issue_comment_bot.py 24 "#openlibrary-g" "replace-with-slack-token"
+```
+
+__Note:__ When adding arguments, be sure to place any hyphenated values within double quotes.