Skip to content

Commit

Permalink
Fix ssh filesystem when it's an easy case
Browse files Browse the repository at this point in the history
  • Loading branch information
mraspaud committed Nov 21, 2024
1 parent 8ae3381 commit f90589f
Show file tree
Hide file tree
Showing 3 changed files with 50 additions and 1 deletion.
16 changes: 16 additions & 0 deletions src/pytroll_watchers/local_watcher.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,22 @@
"""Watcher for non-remote file systems.
Either using OS-based envents (like inotify on linux), or polling.
An example configuration file to retrieve data from a directory.
.. code-block:: yaml
backend: local
fs_config:
directory: /data
file pattern: "H-000-{orig_platform_name:4s}__-{orig_platform_name:4s}_{service:3s}____-{channel_name:_<9s}-\
{segment:_<9s}-{start_time:%Y%m%d%H%M}-{compression:1s}_"
publisher_config:
name: hrit_watcher
message_config:
subject: /segment/hrit/l1b/
atype: file
"""
import logging
import os
Expand Down
12 changes: 11 additions & 1 deletion src/pytroll_watchers/publisher.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from contextlib import closing, suppress
from copy import deepcopy

from paramiko import SSHException
from posttroll.message import Message
from posttroll.publisher import create_publisher_from_dict_config
from trollsift import parse
Expand Down Expand Up @@ -105,8 +106,17 @@ def _build_file_location(file_item, include_dir=None):
uid = file_item.name
file_location["uid"] = uid
with suppress(AttributeError):
file_location["filesystem"] = json.loads(file_item.fs.to_json())
try:
file_location["filesystem"] = json.loads(file_item.fs.to_json())
except SSHException as ssh_exception:
if list(file_item.storage_options.keys()) == ["host"]:
file_location["filesystem"] = {"cls": "fsspec.implementations.sftp:SFTPFileSystem",
"protocol": "sftp", "args": [],
"host": file_item.storage_options["host"]}
else:
raise ssh_exception
file_location["path"] = file_item.path

return file_location


Expand Down
23 changes: 23 additions & 0 deletions tests/test_local_watcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,3 +118,26 @@ def test_publish_paths_forbids_passing_password(tmp_path, patched_local_events,
local_watcher.file_publisher(fs_config=local_settings,
publisher_config=publisher_settings,
message_config=message_settings)


def test_publish_paths_with_ssh(tmp_path, patched_local_events, caplog): # noqa
"""Test publishing paths with an ssh protocol."""
filename = os.fspath(tmp_path / "foo.txt")

host = "localhost"

local_settings = dict(directory=tmp_path, protocol="ssh",
storage_options=dict(host=host))
publisher_settings = dict(nameservers=False, port=1979)
message_settings = dict(subject="/segment/viirs/l1b/", atype="file", data=dict(sensor="viirs"))

caplog.set_level("INFO")
with patched_local_events([filename]):
with patched_publisher() as published_messages:
local_watcher.file_publisher(fs_config=local_settings,
publisher_config=publisher_settings,
message_config=message_settings)
assert len(published_messages) == 1
message = Message(rawstr=published_messages[0])
assert message.data["uri"].startswith("ssh://")
assert message.data["filesystem"]["host"] == host

0 comments on commit f90589f

Please sign in to comment.