Merge pull request #491 from JustAnotherArchivist/fix-stderr

Fix various messages getting printed on stdout instead of stderr

HISTORY.rst

@@ -32,6 +32,7 @@ Unreleased
 - Fixed uploading from a spreadsheet not correctly dropping the ``item`` column from metadata.
 - Fixed uploading from a spreadsheet with ``--checksum`` crashing on skipped files.
 - Fixed minor bug in S3 overload check on upload error retries.
+- Fixed various messages being printed to stdout instead of stderr.
 
 2.3.0 (2022-01-20)
 ++++++++++++++++++

internetarchive/cli/ia.py

@@ -104,7 +104,7 @@ def load_ia_module(cmd):
               file=sys.stderr)
         matches = '\t'.join(difflib.get_close_matches(cmd, cmd_aliases.values()))
         if matches:
-            print(f'\nDid you mean one of these?\n\t{matches}')
+            print(f'\nDid you mean one of these?\n\t{matches}', file=sys.stderr)
         sys.exit(127)
 
 

internetarchive/cli/ia_configure.py

@@ -69,22 +69,22 @@ def main(argv, session):
                                          args['--password'],
                                          config_file=session.config_file,
                                          host=session.host)
-            print(f'Config saved to: {config_file_path}')
+            print(f'Config saved to: {config_file_path}', file=sys.stderr)
 
         # Netrc
         elif args['--netrc']:
-            print("Configuring 'ia' with netrc file...")
+            print("Configuring 'ia' with netrc file...", file=sys.stderr)
             try:
                 n = netrc.netrc()
             except netrc.NetrcParseError as exc:
-                print('error: netrc.netrc() cannot parse your .netrc file.')
+                print('error: netrc.netrc() cannot parse your .netrc file.', file=sys.stderr)
                 sys.exit(1)
             username, _, password = n.hosts['archive.org']
             config_file_path = configure(username,
                                          password,
                                          config_file=session.config_file,
                                          host=session.host)
-            print(f'Config saved to: {config_file_path}')
+            print(f'Config saved to: {config_file_path}', file=sys.stderr)
 
         # Interactive input.
         else:
@@ -94,5 +94,5 @@ def main(argv, session):
             print(f'\nConfig saved to: {config_file_path}')
 
     except AuthenticationError as exc:
-        print(f'\nerror: {exc}')
+        print(f'\nerror: {exc}', file=sys.stderr)
         sys.exit(1)

internetarchive/cli/ia_copy.py

@@ -139,9 +139,9 @@ def main(argv, session, cmd='copy'):
             msg = get_s3_xml_text(r.text)
         except Exception as e:
             msg = r.text
-        print(f'error: failed to {cmd} "{src_path}" to "{dest_path}" - {msg}')
+        print(f'error: failed to {cmd} "{src_path}" to "{dest_path}" - {msg}', file=sys.stderr)
         sys.exit(1)
     elif cmd == 'copy':
-        print(f'success: copied "{src_path}" to "{dest_path}".')
+        print(f'success: copied "{src_path}" to "{dest_path}".', file=sys.stderr)
     else:
         return (r, SRC_FILE)

internetarchive/cli/ia_delete.py

@@ -82,7 +82,7 @@ def main(argv, session):
     verbose = True if not args['--quiet'] else False
     item = session.get_item(args['<identifier>'])
     if not item.exists:
-        print('{0}: skipping, item does\'t exist.')
+        print('{0}: skipping, item does\'t exist.', file=sys.stderr)
 
     # Files that cannot be deleted via S3.
     no_delete = ['_meta.xml', '_files.xml', '_meta.sqlite']
@@ -92,7 +92,7 @@ def main(argv, session):
         args['--header']['x-archive-keep-old-version'] = '1'
 
     if verbose:
-        print(f'Deleting files from {item.identifier}')
+        print(f'Deleting files from {item.identifier}', file=sys.stderr)
 
     if args['--all']:
         files = [f for f in item.get_files()]
@@ -124,7 +124,7 @@ def main(argv, session):
         if any(f.name.endswith(s) for s in no_delete):
             continue
         if args['--dry-run']:
-            print(f' will delete: {item.identifier}/{f.name}')
+            print(f' will delete: {item.identifier}/{f.name}', file=sys.stderr)
             continue
         try:
             resp = f.delete(verbose=verbose,

internetarchive/cli/ia_metadata.py

@@ -81,7 +81,7 @@ def modify_metadata(item, metadata, args):
             etype = 'error'
         print(f'{item.identifier} - {etype} ({r.status_code}): {error_msg}', file=sys.stderr)
         return r
-    print(f'{item.identifier} - success: {r.json()["log"]}')
+    print(f'{item.identifier} - success: {r.json()["log"]}', file=sys.stderr)
     return r
 
 
@@ -105,13 +105,15 @@ def remove_metadata(item, metadata, args):
                     r = item.remove_from_simplelist(c, 'holdings')
                     j = r.json()
                     if j.get('success'):
-                        print(f'{item.identifier} - success: {item.identifier} no longer in {c}')
+                        print(f'{item.identifier} - success: {item.identifier} no longer in {c}',
+                              file=sys.stderr)
                         sys.exit(0)
                     elif j.get('error', '').startswith('no row to delete for'):
-                        print(f'{item.identifier} - success: {item.identifier} no longer in {c}')
+                        print(f'{item.identifier} - success: {item.identifier} no longer in {c}',
+                              file=sys.stderr)
                         sys.exit(0)
                     else:
-                        print(f'{item.identifier} - error: {j.get("error")}')
+                        print(f'{item.identifier} - error: {j.get("error")}', file=sys.stderr)
                         sys.exit(1)
 
         if not isinstance(src_md, list):
@@ -188,7 +190,7 @@ def main(argv, session):
         if args['--exists']:
             if item.exists:
                 responses.append(True)
-                print(f'{identifier} exists')
+                print(f'{identifier} exists', file=sys.stderr)
             else:
                 responses.append(False)
                 print(f'{identifier} does not exist', file=sys.stderr)

internetarchive/cli/ia_move.py

@@ -76,6 +76,6 @@ def main(argv, session):
     r, src_file = ia_copy.main(argv, session, cmd='move')
     dr = src_file.delete(headers=args['--header'], cascade_delete=True)
     if dr.status_code == 204:
-        print(f'success: moved {src_path} to {dest_path}')
+        print(f'success: moved {src_path} to {dest_path}', file=sys.stderr)
         sys.exit(0)
-    print(f'error: {dr.content}')
+    print(f'error: {dr.content}', file=sys.stderr)

internetarchive/cli/ia_reviews.py

@@ -68,10 +68,11 @@ def main(argv, session):
     if j.get('success') or 'no change detected' in j.get('error', '').lower():
         task_id = j.get('value', {}).get('task_id')
         if task_id:
-            print(f'{item.identifier} - success: https://catalogd.archive.org/log/{task_id}')
+            print(f'{item.identifier} - success: https://catalogd.archive.org/log/{task_id}',
+                  file=sys.stderr)
         else:
-            print(f'{item.identifier} - warning: no changes detected!')
+            print(f'{item.identifier} - warning: no changes detected!', file=sys.stderr)
         sys.exit(0)
     else:
-        print(f'{item.identifier} - error: {j.get("error")}')
+        print(f'{item.identifier} - error: {j.get("error")}', file=sys.stderr)
         sys.exit(1)

internetarchive/cli/ia_tasks.py

@@ -95,13 +95,13 @@ def main(argv, session):
         j = r.json()
         if j.get('success'):
             task_log_url = j.get('value', {}).get('log')
-            print(f'success: {task_log_url}')
+            print(f'success: {task_log_url}', file=sys.stderr)
             sys.exit(0)
         elif 'already queued/running' in j.get('error', ''):
-            print(f'success: {args["--cmd"]} task already queued/running')
+            print(f'success: {args["--cmd"]} task already queued/running', file=sys.stderr)
             sys.exit(0)
         else:
-            print(f'error: {j.get("error")}')
+            print(f'error: {j.get("error")}', file=sys.stderr)
             sys.exit(1)
 
     # Tasks read API.

internetarchive/cli/ia_upload.py

@@ -84,27 +84,27 @@ def _upload_files(item, files, upload_kwargs, prev_identifier=None, archive_sess
     """Helper function for calling :meth:`Item.upload`"""
     responses = []
     if (upload_kwargs['verbose']) and (prev_identifier != item.identifier):
-        print(f'{item.identifier}:')
+        print(f'{item.identifier}:', file=sys.stderr)
 
     try:
         response = item.upload(files, **upload_kwargs)
         responses += response
     except HTTPError as exc:
         responses += [exc.response]
     except InvalidIdentifierException as exc:
-        print(str(exc))
+        print(str(exc), file=sys.stderr)
         sys.exit(1)
     finally:
         # Debug mode.
         if upload_kwargs['debug']:
             for i, r in enumerate(responses):
                 if i != 0:
-                    print('---')
+                    print('---', file=sys.stderr)
                 headers = '\n'.join(
                     [f' {k}:{v}' for (k, v) in r.headers.items()]
                 )
-                print(f'Endpoint:\n {r.url}\n')
-                print(f'HTTP Headers:\n{headers}')
+                print(f'Endpoint:\n {r.url}\n', file=sys.stderr)
+                print(f'HTTP Headers:\n{headers}', file=sys.stderr)
 
     return responses
 
@@ -167,7 +167,7 @@ def main(argv, session):
                   file=sys.stderr)
             sys.exit(1)
         else:
-            print(f'success: {args["<identifier>"]} is accepting requests.')
+            print(f'success: {args["<identifier>"]} is accepting requests.', file=sys.stderr)
             sys.exit()
 
     elif args['<identifier>']:

internetarchive/item.py

@@ -949,7 +949,7 @@ def upload_file(self, body,
             if (not self.tasks) and (ia_file) and (ia_file.md5 == md5_sum):
                 log.info(f'{key} already exists: {url}')
                 if verbose:
-                    print(f' {key} already exists, skipping.')
+                    print(f' {key} already exists, skipping.', file=sys.stderr)
                 if delete:
                     log.info(
                         f'{key} successfully uploaded to '
@@ -987,7 +987,7 @@ def _build_request():
                                               unit='MiB')
                     data = IterableToFileAdapter(progress_generator, size)
                 except:
-                    print(f' uploading {key}')
+                    print(f' uploading {key}', file=sys.stderr)
                     data = body
             else:
                 data = body

tests/cli/test_ia_metadata.py

@@ -11,7 +11,7 @@ def test_ia_metadata_exists(capsys):
         rsps.add_metadata_mock('nasa')
         ia_call(['ia', 'metadata', '--exists', 'nasa'], expected_exit_code=0)
         out, err = capsys.readouterr()
-        assert out == 'nasa exists\n'
+        assert err == 'nasa exists\n'
         rsps.reset()
         rsps.add_metadata_mock('nasa', '{}')
         sys.argv = ['ia', 'metadata', '--exists', 'nasa']
@@ -38,4 +38,4 @@ def test_ia_metadata_modify(capsys):
         valid_key = f'foo-{int(time())}'
         ia_call(['ia', 'metadata', '--modify', f'{valid_key}:test_value', 'nasa'])
         out, err = capsys.readouterr()
-        assert out == 'nasa - success: https://catalogd.archive.org/log/447613301\n'
+        assert err == 'nasa - success: https://catalogd.archive.org/log/447613301\n'

tests/cli/test_ia_upload.py

@@ -48,7 +48,7 @@ def test_ia_upload_status_check(capsys):
 
         ia_call(['ia', 'upload', 'nasa', '--status-check'])
         out, err = capsys.readouterr()
-        assert 'success: nasa is accepting requests.' in out
+        assert 'success: nasa is accepting requests.' in err
 
         j = json.loads(STATUS_CHECK_RESPONSE)
         j['over_limit'] = 1
@@ -69,13 +69,13 @@ def test_ia_upload_debug(capsys, tmpdir_ch, nasa_mocker):
 
     ia_call(['ia', 'upload', '--debug', 'nasa', 'test.txt'])
     out, err = capsys.readouterr()
-    assert 'User-Agent' in out
-    assert 's3.us.archive.org/nasa/test.txt' in out
-    assert 'Accept:*/*' in out
-    assert 'Authorization:LOW ' in out
-    assert 'Connection:close' in out
-    assert 'Content-Length:3' in out
-    assert 'Accept-Encoding:gzip, deflate' in out
+    assert 'User-Agent' in err
+    assert 's3.us.archive.org/nasa/test.txt' in err
+    assert 'Accept:*/*' in err
+    assert 'Authorization:LOW ' in err
+    assert 'Connection:close' in err
+    assert 'Content-Length:3' in err
+    assert 'Accept-Encoding:gzip, deflate' in err
 
 
 def test_ia_upload_403(capsys):
@@ -114,14 +114,14 @@ def test_ia_upload_size_hint(capsys, tmpdir_ch, nasa_mocker):
 
     ia_call(['ia', 'upload', '--debug', 'nasa', '--size-hint', '30', 'test.txt'])
     out, err = capsys.readouterr()
-    assert 'User-Agent' in out
-    assert 's3.us.archive.org/nasa/test.txt' in out
-    assert 'x-archive-size-hint:30' in out
-    assert 'Accept:*/*' in out
-    assert 'Authorization:LOW ' in out
-    assert 'Connection:close' in out
-    assert 'Content-Length:3' in out
-    assert 'Accept-Encoding:gzip, deflate' in out
+    assert 'User-Agent' in err
+    assert 's3.us.archive.org/nasa/test.txt' in err
+    assert 'x-archive-size-hint:30' in err
+    assert 'Accept:*/*' in err
+    assert 'Authorization:LOW ' in err
+    assert 'Connection:close' in err
+    assert 'Content-Length:3' in err
+    assert 'Accept-Encoding:gzip, deflate' in err
 
 
 def test_ia_upload_unicode(tmpdir_ch, caplog):