s3_object should flush data to disk during download to avoid filling up the host's ram #2395
Open
1 task done
Labels
feature
This issue/PR relates to a feature request
Comments
abikouo
added
feature
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Summary
During object download, the
s3_objectmodule stores the entire file in memory before flushing data to disk.When operating with big files, this ends up occupying the host's entire RAM.
Though I can use workarounds, they all require me to not use this module, which I would prefer to use for standardization/consistency.
I feel the module should have an option to make the module flush to disk in small chucks.
It would be even better if the chunk's size could be defined in the module's arguments.
Issue Type
Bug Report
Component Name
s3_object
Ansible Version
Collection Versions
AWS SDK versions
Configuration
OS / Environment
{ "all_ipv4_addresses": [ "172.31.20.22" ], "all_ipv6_addresses": [ "abcd::abc:abcd:abcd:abcd" ], "ansible_local": {}, "apparmor": { "status": "disabled" }, "architecture": "x86_64", "bios_date": "10/16/2017", "bios_vendor": "Amazon EC2", "bios_version": "1.0", "board_asset_tag": "i-01234567890abcdef", "board_name": "NA", "board_serial": "NA", "board_vendor": "Amazon EC2", "board_version": "NA", "chassis_asset_tag": "Amazon EC2", "chassis_serial": "NA", "chassis_vendor": "Amazon EC2", "chassis_version": "NA", "cmdline": { "BOOT_IMAGE": "(hd0,gpt1)/boot/vmlinuz-6.1.115-126.197.amzn2023.x86_64", "console": "ttyS0,115200n8", "nvme_core.io_timeout": "4294967295", "quiet": true, "rd.emergency": "poweroff", "rd.shell": "0", "ro": true, "root": "UUID=01234567-abcd-0123-abcd-0123456789ab", "security": "selinux", "selinux": "1" }, "date_time": { "date": "2024-11-20", "day": "20", "epoch": "1732062188", "epoch_int": "1732062188", "hour": "00", "iso8601": "2024-11-20T00:23:08Z", "iso8601_basic": "20241120T002308511477", "iso8601_basic_short": "20241120T002308", "iso8601_micro": "2024-11-20T00:23:08.511477Z", "minute": "23", "month": "11", "second": "08", "time": "00:23:08", "tz": "UTC", "tz_dst": "UTC", "tz_offset": "+0000", "weekday": "Wednesday", "weekday_number": "3", "weeknumber": "47", "year": "2024" }, "default_ipv4": { "address": "172.31.20.220", "alias": "ens5", "broadcast": "", "gateway": "172.31.20.1", "interface": "ens5", "macaddress": "00:aa:bb:cc:dd:ee", "mtu": 9001, "netmask": "255.255.240.0", "network": "172.31.20.0", "prefix": "20", "type": "ether" }, "default_ipv6": {}, "device_links": { "ids": { "nvme0n1": [ "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef", "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef-ns-1", "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef_1", "nvme-uuid.01234567-abcd-0123-abcd-0123456789ab" ], "nvme0n1p1": [ "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef-ns-1-part1", "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef-part1", "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef_1-part1", "nvme-uuid.01234567-abcd-0123-abcd-0123456789ab-part1" ], "nvme0n1p127": [ "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef-ns-1-part127", "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef-part127", "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef_1-part127", "nvme-uuid.01234567-abcd-0123-abcd-0123456789ab-part127" ], "nvme0n1p128": [ "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef-ns-1-part128", "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef-part128", "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef_1-part128", "nvme-uuid.01234567-abcd-0123-abcd-0123456789ab-part128" ] }, "labels": { "nvme0n1p1": [ "\\x2f" ] }, "masters": {}, "uuids": { "nvme0n1p1": [ "01234567-abcd-0123-abcd-0123456789ab" ], "nvme0n1p128": [ "AAAA-0000" ] } }, "devices": { "loop0": { "holders": [], "host": "", "links": { "ids": [], "labels": [], "masters": [], "uuids": [] }, "model": null, "partitions": {}, "removable": "0", "rotational": "1", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "none", "sectors": "0", "sectorsize": "512", "size": "0.00 Bytes", "support_discard": "0", "vendor": null, "virtual": 1 }, "loop1": { "holders": [], "host": "", "links": { "ids": [], "labels": [], "masters": [], "uuids": [] }, "model": null, "partitions": {}, "removable": "0", "rotational": "1", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "none", "sectors": "0", "sectorsize": "512", "size": "0.00 Bytes", "support_discard": "0", "vendor": null, "virtual": 1 }, "loop2": { "holders": [], "host": "", "links": { "ids": [], "labels": [], "masters": [], "uuids": [] }, "model": null, "partitions": {}, "removable": "0", "rotational": "1", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "none", "sectors": "0", "sectorsize": "512", "size": "0.00 Bytes", "support_discard": "0", "vendor": null, "virtual": 1 }, "loop3": { "holders": [], "host": "", "links": { "ids": [], "labels": [], "masters": [], "uuids": [] }, "model": null, "partitions": {}, "removable": "0", "rotational": "1", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "none", "sectors": "0", "sectorsize": "512", "size": "0.00 Bytes", "support_discard": "0", "vendor": null, "virtual": 1 }, "loop4": { "holders": [], "host": "", "links": { "ids": [], "labels": [], "masters": [], "uuids": [] }, "model": null, "partitions": {}, "removable": "0", "rotational": "1", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "none", "sectors": "0", "sectorsize": "512", "size": "0.00 Bytes", "support_discard": "0", "vendor": null, "virtual": 1 }, "loop5": { "holders": [], "host": "", "links": { "ids": [], "labels": [], "masters": [], "uuids": [] }, "model": null, "partitions": {}, "removable": "0", "rotational": "1", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "none", "sectors": "0", "sectorsize": "512", "size": "0.00 Bytes", "support_discard": "0", "vendor": null, "virtual": 1 }, "loop6": { "holders": [], "host": "", "links": { "ids": [], "labels": [], "masters": [], "uuids": [] }, "model": null, "partitions": {}, "removable": "0", "rotational": "1", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "none", "sectors": "0", "sectorsize": "512", "size": "0.00 Bytes", "support_discard": "0", "vendor": null, "virtual": 1 }, "loop7": { "holders": [], "host": "", "links": { "ids": [], "labels": [], "masters": [], "uuids": [] }, "model": null, "partitions": {}, "removable": "0", "rotational": "1", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "none", "sectors": "0", "sectorsize": "512", "size": "0.00 Bytes", "support_discard": "0", "vendor": null, "virtual": 1 }, "nvme0n1": { "holders": [], "host": "Non-Volatile memory controller: Amazon.com, Inc. NVMe EBS Controller", "links": { "ids": [ "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef", "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef-ns-1", "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef_1", "nvme-uuid.01234567-abcd-0123-abcd-0123456789ab" ], "labels": [], "masters": [], "uuids": [] }, "model": "Amazon Elastic Block Store", "partitions": { "nvme0n1p1": { "holders": [], "links": { "ids": [ "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef-ns-1-part1", "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef-part1", "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef_1-part1", "nvme-uuid.01234567-abcd-0123-abcd-0123456789ab-part1" ], "labels": [ "\\x2f" ], "masters": [], "uuids": [ "01234567-abcd-0123-abcd-0123456789ab" ] }, "sectors": "419405791", "sectorsize": 512, "size": "199.99 GB", "start": "24576", "uuid": "01234567-abcd-0123-abcd-0123456789ab" }, "nvme0n1p127": { "holders": [], "links": { "ids": [ "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef-ns-1-part127", "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef-part127", "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef_1-part127", "nvme-uuid.01234567-abcd-0123-abcd-0123456789ab-part127" ], "labels": [], "masters": [], "uuids": [] }, "sectors": "2048", "sectorsize": 512, "size": "1.00 MB", "start": "22528", "uuid": null }, "nvme0n1p128": { "holders": [], "links": { "ids": [ "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef-ns-1-part128", "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef-part128", "nvme-Amazon_Elastic_Block_Store_vol01234567890abcdef_1-part128", "nvme-uuid.01234567-abcd-0123-abcd-0123456789ab-part128" ], "labels": [], "masters": [], "uuids": [ "AAAA-0000" ] }, "sectors": "20480", "sectorsize": 512, "size": "10.00 MB", "start": "2048", "uuid": "AAAA-0000" } }, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "none", "sectors": "419430400", "sectorsize": "512", "serial": "vol01234567890abcdef", "size": "200.00 GB", "support_discard": "0", "vendor": null, "virtual": 1 } }, "distribution": "Amazon", "distribution_file_parsed": true, "distribution_file_path": "/etc/os-release", "distribution_file_variety": "Amazon", "distribution_major_version": "2023", "distribution_minor_version": "NA", "distribution_release": "NA", "distribution_version": "2023", "dns": { "nameservers": [ "172.31.0.2" ], "search": [ "eu-west-1.compute.internal" ] }, "domain": "eu-west-1.compute.internal", "effective_group_id": 1001, "effective_user_id": 1001, "ens5": { "active": true, "device": "ens5", "features": { "esp_hw_offload": "off [fixed]", "esp_tx_csum_hw_offload": "off [fixed]", "fcoe_mtu": "off [fixed]", "generic_receive_offload": "on", "generic_segmentation_offload": "on", "highdma": "on", "hsr_dup_offload": "off [fixed]", "hsr_fwd_offload": "off [fixed]", "hsr_tag_ins_offload": "off [fixed]", "hsr_tag_rm_offload": "off [fixed]", "hw_tc_offload": "off [fixed]", "l2_fwd_offload": "off [fixed]", "large_receive_offload": "off [fixed]", "loopback": "off [fixed]", "macsec_hw_offload": "off [fixed]", "netns_local": "off [fixed]", "ntuple_filters": "off [fixed]", "receive_hashing": "on", "rx_all": "off [fixed]", "rx_checksumming": "on", "rx_fcs": "off [fixed]", "rx_gro_hw": "off [fixed]", "rx_gro_list": "off", "rx_udp_gro_forwarding": "off", "rx_udp_tunnel_port_offload": "off [fixed]", "rx_vlan_filter": "off [fixed]", "rx_vlan_offload": "off [fixed]", "rx_vlan_stag_filter": "off [fixed]", "rx_vlan_stag_hw_parse": "off [fixed]", "scatter_gather": "on", "tcp_segmentation_offload": "off", "tls_hw_record": "off [fixed]", "tls_hw_rx_offload": "off [fixed]", "tls_hw_tx_offload": "off [fixed]", "tx_checksum_fcoe_crc": "off [fixed]", "tx_checksum_ip_generic": "off [fixed]", "tx_checksum_ipv4": "on", "tx_checksum_ipv6": "on", "tx_checksum_sctp": "off [fixed]", "tx_checksumming": "on", "tx_esp_segmentation": "off [fixed]", "tx_fcoe_segmentation": "off [fixed]", "tx_gre_csum_segmentation": "off [fixed]", "tx_gre_segmentation": "off [fixed]", "tx_gso_list": "off [fixed]", "tx_gso_partial": "off [fixed]", "tx_gso_robust": "off [fixed]", "tx_ipxip4_segmentation": "off [fixed]", "tx_ipxip6_segmentation": "off [fixed]", "tx_lockless": "off [fixed]", "tx_nocache_copy": "off", "tx_scatter_gather": "on", "tx_scatter_gather_fraglist": "off [fixed]", "tx_sctp_segmentation": "off [fixed]", "tx_tcp6_segmentation": "off [fixed]", "tx_tcp_ecn_segmentation": "off [fixed]", "tx_tcp_mangleid_segmentation": "off [fixed]", "tx_tcp_segmentation": "off [fixed]", "tx_tunnel_remcsum_segmentation": "off [fixed]", "tx_udp_segmentation": "off [fixed]", "tx_udp_tnl_csum_segmentation": "off [fixed]", "tx_udp_tnl_segmentation": "off [fixed]", "tx_vlan_offload": "off [fixed]", "tx_vlan_stag_hw_insert": "off [fixed]", "vlan_challenged": "off [fixed]" }, "hw_timestamp_filters": [], "ipv4": { "address": "172.31.20.22", "broadcast": "", "netmask": "255.255.240.0", "network": "172.31.20.0", "prefix": "20" }, "ipv6": [ { "address": "aaaa::000:aaaa:0000:aaaa", "prefix": "64", "scope": "link" } ], "macaddress": "00:11:22:33:44:55", "module": "ena", "mtu": 9001, "pciid": "0000:00:05.0", "promisc": false, "timestamping": [], "type": "ether" }, "env": { "BASH_FUNC_which%%": "() { ( alias;\n eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot \"$@\"\n}", "HOME": "/home/ssm-user", "INVOCATION_ID": "ede4e91fd00d4b8db45b9abe37394ad8", "JOURNAL_STREAM": "8:43462", "LANG": "C.UTF-8", "LESSOPEN": "||/usr/bin/lesspipe.sh %s", "LS_COLORS": "rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.webp=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:", "PATH": "/home/ssm-user/.local/bin:/home/ssm-user/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin", "PWD": "/usr/bin", "SHLVL": "2", "SYSTEMD_COLORS": "false", "SYSTEMD_EXEC_PID": "25584", "S_COLORS": "auto", "TERM": "xterm-256color", "_": "/usr/bin/python3", "which_declare": "declare -f" }, "fibre_channel_wwn": [], "fips": false, "form_factor": "Other", "fqdn": "ip-172-31-20-22.eu-west-1.compute.internal", "gather_subset": [ "all" ], "hostname": "ip-172-31-20-22", "hostnqn": "", "interfaces": [ "ens5", "lo" ], "is_chroot": true, "iscsi_iqn": "", "kernel": "6.1.115-126.197.amzn2023.x86_64", "kernel_version": "#1 SMP PREEMPT_DYNAMIC Tue Nov 5 17:36:57 UTC 2024", "lo": { "active": true, "device": "lo", "features": { "esp_hw_offload": "off [fixed]", "esp_tx_csum_hw_offload": "off [fixed]", "fcoe_mtu": "off [fixed]", "generic_receive_offload": "on", "generic_segmentation_offload": "on", "highdma": "on [fixed]", "hsr_dup_offload": "off [fixed]", "hsr_fwd_offload": "off [fixed]", "hsr_tag_ins_offload": "off [fixed]", "hsr_tag_rm_offload": "off [fixed]", "hw_tc_offload": "off [fixed]", "l2_fwd_offload": "off [fixed]", "large_receive_offload": "off [fixed]", "loopback": "on [fixed]", "macsec_hw_offload": "off [fixed]", "netns_local": "on [fixed]", "ntuple_filters": "off [fixed]", "receive_hashing": "off [fixed]", "rx_all": "off [fixed]", "rx_checksumming": "on [fixed]", "rx_fcs": "off [fixed]", "rx_gro_hw": "off [fixed]", "rx_gro_list": "off", "rx_udp_gro_forwarding": "off", "rx_udp_tunnel_port_offload": "off [fixed]", "rx_vlan_filter": "off [fixed]", "rx_vlan_offload": "off [fixed]", "rx_vlan_stag_filter": "off [fixed]", "rx_vlan_stag_hw_parse": "off [fixed]", "scatter_gather": "on", "tcp_segmentation_offload": "on", "tls_hw_record": "off [fixed]", "tls_hw_rx_offload": "off [fixed]", "tls_hw_tx_offload": "off [fixed]", "tx_checksum_fcoe_crc": "off [fixed]", "tx_checksum_ip_generic": "on [fixed]", "tx_checksum_ipv4": "off [fixed]", "tx_checksum_ipv6": "off [fixed]", "tx_checksum_sctp": "on [fixed]", "tx_checksumming": "on", "tx_esp_segmentation": "off [fixed]", "tx_fcoe_segmentation": "off [fixed]", "tx_gre_csum_segmentation": "off [fixed]", "tx_gre_segmentation": "off [fixed]", "tx_gso_list": "on", "tx_gso_partial": "off [fixed]", "tx_gso_robust": "off [fixed]", "tx_ipxip4_segmentation": "off [fixed]", "tx_ipxip6_segmentation": "off [fixed]", "tx_lockless": "on [fixed]", "tx_nocache_copy": "off [fixed]", "tx_scatter_gather": "on [fixed]", "tx_scatter_gather_fraglist": "on [fixed]", "tx_sctp_segmentation": "on", "tx_tcp6_segmentation": "on", "tx_tcp_ecn_segmentation": "on", "tx_tcp_mangleid_segmentation": "on", "tx_tcp_segmentation": "on", "tx_tunnel_remcsum_segmentation": "off [fixed]", "tx_udp_segmentation": "on", "tx_udp_tnl_csum_segmentation": "off [fixed]", "tx_udp_tnl_segmentation": "off [fixed]", "tx_vlan_offload": "off [fixed]", "tx_vlan_stag_hw_insert": "off [fixed]", "vlan_challenged": "on [fixed]" }, "hw_timestamp_filters": [], "ipv4": { "address": "127.0.0.1", "broadcast": "", "netmask": "255.0.0.0", "network": "127.0.0.0", "prefix": "8" }, "ipv6": [ { "address": "::1", "prefix": "128", "scope": "host" } ], "mtu": 65536, "promisc": false, "timestamping": [], "type": "loopback" }, "loadavg": { "15m": 0.35, "1m": 0.77, "5m": 0.49 }, "locally_reachable_ips": { "ipv4": [ "127.0.0.0/8", "127.0.0.1", "172.31.20.22" ], "ipv6": [ "::1", "abcd::abcd:abcd:abcd:abcd" ] }, "lsb": {}, "lvm": "N/A", "machine": "x86_64", "machine_id": "abcdef0123456789abcdef0123456789", "memfree_mb": 5980, "memory_mb": { "nocache": { "free": 7160, "used": 567 }, "real": { "free": 5980, "total": 7727, "used": 1747 }, "swap": { "cached": 0, "free": 0, "total": 0, "used": 0 } }, "memtotal_mb": 7727, "module_setup": true, "mounts": [ { "block_available": 51438510, "block_size": 4096, "block_total": 52409339, "block_used": 970829, "device": "/dev/nvme0n1p1", "fstype": "xfs", "inode_available": 104753200, "inode_total": 104851440, "inode_used": 98240, "mount": "/", "options": "rw,seclabel,noatime,attr2,inode64,logbufs=8,logbsize=32k,sunit=1024,swidth=1024,noquota", "size_available": 210692136960, "size_total": 214668652544, "uuid": "01234567-abcd-0123-abcd-0123456789ab" }, { "block_available": 4446, "block_size": 2048, "block_total": 5101, "block_used": 655, "device": "/dev/nvme0n1p128", "fstype": "vfat", "inode_available": 0, "inode_total": 0, "inode_used": 0, "mount": "/boot/efi", "options": "rw,noatime,fmask=0077,dmask=0077,codepage=437,iocharset=ascii,shortname=winnt,errors=remount-ro", "size_available": 9105408, "size_total": 10446848, "uuid": "AAAA-0000" } ], "nodename": "ip-172-31-20-22.eu-west-1.compute.internal", "os_family": "RedHat", "pkg_mgr": "dnf", "proc_cmdline": { "BOOT_IMAGE": "(hd0,gpt1)/boot/vmlinuz-6.1.115-126.197.amzn2023.x86_64", "console": [ "tty0", "ttyS0,115200n8" ], "nvme_core.io_timeout": "4294967295", "quiet": true, "rd.emergency": "poweroff", "rd.shell": "0", "ro": true, "root": "UUID=01234567-abcd-0123-abcd-0123456789ab", "security": "selinux", "selinux": "1" }, "processor": [ "0", "AuthenticAMD", "AMD EPYC 7R13 Processor", "1", "AuthenticAMD", "AMD EPYC 7R13 Processor", "2", "AuthenticAMD", "AMD EPYC 7R13 Processor", "3", "AuthenticAMD", "AMD EPYC 7R13 Processor" ], "processor_cores": 2, "processor_count": 1, "processor_nproc": 4, "processor_threads_per_core": 2, "processor_vcpus": 4, "product_name": "c6a.xlarge", "product_serial": "NA", "product_uuid": "NA", "product_version": "NA", "python": { "executable": "/usr/bin/python3", "has_sslcontext": true, "type": "cpython", "version": { "major": 3, "micro": 16, "minor": 9, "releaselevel": "final", "serial": 0 }, "version_info": [ 3, 9, 16, "final", 0 ] }, "python_version": "3.9.16", "real_group_id": 1001, "real_user_id": 1001, "selinux": { "config_mode": "permissive", "mode": "permissive", "policyvers": 33, "status": "enabled", "type": "targeted" }, "selinux_python_present": true, "service_mgr": "systemd", "ssh_host_key_ecdsa_public": "AAAAE2VjZHNhLXNoYrItbmlz4HAyNTYAAAAIbmlzTYAyNTYAAABBBHF7OancTRHGG2OTMqNstUc9nJx0hUoEPpmFVYBHI5WCJlcQ8GB5jl/FIbGe04kGhhKHDiauXNGfNMLfAwWSQfI=", "ssh_host_key_ecdsa_public_keytype": "ecdsa-sha2-nistp256", "ssh_host_key_ed25519_public": "AAAAC3NzaC1lZDI1NTE5AAAAIFggCuPHK993MSZZMemI3Y32keMesdxdQ1UwwhZw5esL", "ssh_host_key_ed25519_public_keytype": "ssh-ed25519", "swapfree_mb": 0, "swaptotal_mb": 0, "system": "Linux", "system_capabilities": [ "" ], "system_capabilities_enforced": "True", "system_vendor": "Amazon EC2", "uptime_seconds": 3209, "user_dir": "/home/ssm-user", "user_gecos": "", "user_gid": 1001, "user_id": "ssm-user", "user_shell": "/bin/bash", "user_uid": 1001, "userspace_architecture": "x86_64", "userspace_bits": "64", "virtualization_role": "guest", "virtualization_tech_guest": [ "kvm" ], "virtualization_tech_host": [], "virtualization_type": "kvm" }Steps to Reproduce
Have an object in a S3 bucket bigger than the RAM of the executor
Download the object via the
s3_objectmodule:Look at the host crash
Expected Results
I expected the object to be downloaded in chunks, and those chucks to be concatenated in the resulting file as
curlwould do.I did not expect to have the whole object copied in memory.
Actual Results
The module runs, but the host stops responding when its memory is filled up completely.
Code of Conduct
The text was updated successfully, but these errors were encountered: